diff --git a/core/config.class.inc.php b/core/config.class.inc.php index 4c8ea89b5..e0c95bfc7 100644 --- a/core/config.class.inc.php +++ b/core/config.class.inc.php @@ -138,6 +138,12 @@ class Config */ protected $m_sEncryptionKey; + /** + * @var array Additional character sets to be supported by the interactive CSV import + * 'iconv_code' => 'display name' + */ + protected $m_aCharsets; + public function __construct($sConfigFile, $bLoadConfig = true) { $this->m_sFile = $sConfigFile; @@ -192,6 +198,7 @@ class Config $this->m_sAllowedLoginTypes = DEFAULT_ALLOWED_LOGIN_TYPES; $this->m_sExtAuthVariable = DEFAULT_EXT_AUTH_VARIABLE; $this->m_sEncryptionKey = DEFAULT_ENCRYPTION_KEY; + $this->m_aCharsets = array(); $this->m_aModuleSettings = array(); @@ -296,6 +303,7 @@ class Config $this->m_sAllowedLoginTypes = isset($MySettings['allowed_login_types']) ? trim($MySettings['allowed_login_types']) : DEFAULT_ALLOWED_LOGIN_TYPES; $this->m_sExtAuthVariable = isset($MySettings['ext_auth_variable']) ? trim($MySettings['ext_auth_variable']) : DEFAULT_EXT_AUTH_VARIABLE; $this->m_sEncryptionKey = isset($MySettings['encryption_key']) ? trim($MySettings['encryption_key']) : DEFAULT_ENCRYPTION_KEY; + $this->m_aCharsets = isset($MySettings['csv_import_charsets']) ? $MySettings['csv_import_charsets'] : array(); } protected function Verify() @@ -473,6 +481,11 @@ class Config return $this->m_sExtAuthVariable; } + public function GetCSVImportCharsets() + { + return $this->m_aCharsets; + } + public function SetDBHost($sDBHost) { $this->m_sDBHost = $sDBHost; @@ -568,6 +581,15 @@ class Config $this->m_sEncryptionKey = $sKey; } + public function SetCSVImportCharsets($aCharsets) + { + $this->m_aCharsets = $aCharsets; + } + + public function AddCSVImportCharset($sIconvCode, $sDisplayName) + { + $this->m_aCharsets[$sIconvCode] = $sDisplayName; + } public function FileIsWritable() { return is_writable($this->m_sFile); @@ -618,6 +640,9 @@ class Config fwrite($hFile, "\t'default_language' => '{$this->m_sDefaultLanguage}',\n"); fwrite($hFile, "\t'allowed_login_types' => '{$this->m_sAllowedLoginTypes}',\n"); fwrite($hFile, "\t'encryption_key' => '{$this->m_sEncryptionKey}',\n"); + $sExport = var_export($this->m_aCharsets, true); + fwrite($hFile, "\t'csv_import_charsets' => $sExport,\n"); + fwrite($hFile, ");\n"); fwrite($hFile, "\n"); diff --git a/dictionaries/dictionary.itop.ui.php b/dictionaries/dictionary.itop.ui.php index ac399abf0..d7fb9d5ea 100644 --- a/dictionaries/dictionary.itop.ui.php +++ b/dictionaries/dictionary.itop.ui.php @@ -496,7 +496,7 @@ Dict::Add('EN US', 'English', 'English', array( 'UI:CSVImport:HeaderSearch' => 'Search?', 'UI:CSVImport:AlertIncompleteMapping' => 'Please select a mapping for every field.', 'UI:CSVImport:AlertNoSearchCriteria' => 'Please select at least one search criteria', - + 'UI:CSVImport:Encoding' => 'Character encoding', 'UI:UniversalSearchTitle' => 'iTop - Universal Search', 'UI:UniversalSearch:Error' => 'Error: %1$s', 'UI:UniversalSearch:LabelSelectTheClass' => 'Select the class to search: ', diff --git a/dictionaries/fr.dictionary.itop.ui.php b/dictionaries/fr.dictionary.itop.ui.php index cbf576f00..182737318 100644 --- a/dictionaries/fr.dictionary.itop.ui.php +++ b/dictionaries/fr.dictionary.itop.ui.php @@ -499,6 +499,7 @@ Dict::Add('FR FR', 'French', 'Français', array( 'UI:CSVImport:HeaderSearch' => 'Recherche ?', 'UI:CSVImport:AlertIncompleteMapping' => 'Veuillez choisir le correspondance de chacun des champs.', 'UI:CSVImport:AlertNoSearchCriteria' => 'Veuillez choisir au moins une clef de recherche.', + 'UI:CSVImport:Encoding' => 'Encodage des caractères', 'UI:UniversalSearchTitle' => 'iTop - Recherche Universelle', 'UI:UniversalSearch:Error' => 'Erreur : %1$s', diff --git a/pages/ajax.csvimport.php b/pages/ajax.csvimport.php index 58594d6b8..5841f12b2 100644 --- a/pages/ajax.csvimport.php +++ b/pages/ajax.csvimport.php @@ -185,6 +185,7 @@ switch($sOperation) $sTextQualifier = utils::ReadParam('qualifier', '"'); $iLinesToSkip = utils::ReadParam('nb_lines_skipped', 0); $bFirstLineAsHeader = utils::ReadParam('header_line', true); + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); $sData = stripslashes(utils::ReadParam('csvdata', true)); $oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier); $aData = $oCSVParser->ToArray($iLinesToSkip); @@ -246,7 +247,7 @@ switch($sOperation) $sData = stripslashes(utils::ReadParam('csvdata', true)); $sClassName = utils::ReadParam('class_name', ''); $bAdvanced = utils::ReadParam('advanced', false); - + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); $oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier); $aData = $oCSVParser->ToArray($iLinesToSkip); $iTarget = count($aData); diff --git a/pages/csvimport.php b/pages/csvimport.php index 14bc4a010..36106081a 100644 --- a/pages/csvimport.php +++ b/pages/csvimport.php @@ -232,6 +232,7 @@ function ProcessCSVData(WebPage $oPage, $bSimulate = true) $aSearchFields = utils::ReadParam('search_field', array()); $iCurrentStep = $bSimulate ? 4 : 5; $bAdvanced = utils::ReadParam('advanced', 0); + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); // Parse the data set $oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier); @@ -476,6 +477,7 @@ function ProcessCSVData(WebPage $oPage, $bSimulate = true) $oPage->add(''); $oPage->add(''); $oPage->add(''); + $oPage->add(''); foreach($aFieldsMapping as $iNumber => $sAttCode) { $oPage->add(''); @@ -603,6 +605,7 @@ function SelectMapping(WebPage $oPage) } $sClassName = utils::ReadParam('class_name', ''); $bAdvanced = utils::ReadParam('advanced', 0); + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); $oPage->add('

'.Dict::S('UI:Title:CSVImportStep3').'

'); $oPage->add('
'); @@ -619,7 +622,7 @@ function SelectMapping(WebPage $oPage) $oPage->add(''); $oPage->add(''); $oPage->add(''); - $oPage->add(''); + $oPage->add(''); $oPage->add('

  '); $oPage->add('

'); $oPage->add(''); @@ -670,6 +673,7 @@ EOF var header_line = $('input[name=header_line]').val(); var nb_lines_skipped = $('input[name=nb_skipped_lines]').val(); var csv_data = $('input[name=csvdata]').val(); + var encoding = $('input[name=encoding]').val(); if (advanced != 1) { advanced = 0; @@ -687,7 +691,7 @@ EOF ajax_request = $.post('ajax.csvimport.php', { operation: 'display_mapping_form', enctype: 'multipart/form-data', csvdata: csv_data, separator: separator, qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line, class_name: class_name, - advanced: advanced }, + advanced: advanced, encoding: encoding }, function(data) { $('#mapping').empty(); $('#mapping').append(data); @@ -823,8 +827,19 @@ function SelectOptions(WebPage $oPage) default: $sCSVData = utils::ReadParam('csvdata', '', 'post'); } - - $aGuesses = GuessParameters($sCSVData); // Try to predict the parameters, based on the input data + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); + + // Compute a subset of the data set, now that we know the charset + if ($sEncoding == 'UTF-8') + { + $sUTF8Data = $sCSVData; + } + else + { + $sUTF8Data = iconv($sEncoding, 'UTF-8//IGNORE//TRANSLIT', $sCSVData); + } + + $aGuesses = GuessParameters($sUTF8Data); // Try to predict the parameters, based on the input data $sSeparator = utils::ReadParam('separator', ''); if ($sSeparator == '') // May be set to an empty value by the previous page @@ -848,23 +863,23 @@ function SelectOptions(WebPage $oPage) // Create a truncated version of the data used for the fast preview // Take about 20 lines of data... knowing that some lines may contain carriage returns $iMaxLines = 20; - $iMaxLen = strlen($sCSVData); + $iMaxLen = strlen($sUTF8Data); $iCurPos = true; while ( ($iCurPos > 0) && ($iMaxLines > 0)) { - $pos = strpos($sCSVData, "\n", $iCurPos); + $pos = strpos($sUTF8Data, "\n", $iCurPos); if ($pos !== false) { $iCurPos = 1+$pos; } else { - $iCurPos = strlen($sCSVData); + $iCurPos = strlen($sUTF8Data); $iMaxLines = 1; } $iMaxLines--; } - $sCSVDataTruncated = substr($sCSVData, 0, $iCurPos); + $sCSVDataTruncated = substr($sUTF8Data, 0, $iCurPos); $oPage->add('

'.Dict::S('UI:Title:CSVImportStep2').'

'); $oPage->add('
'); @@ -888,7 +903,7 @@ function SelectOptions(WebPage $oPage) $oPage->add('

'.Dict::Format('UI:CSVImport:Skip_N_LinesAtTheBeginning', '').'

'); $oPage->add(''); $oPage->add(''); - $oPage->add(''); + $oPage->add(''); $oPage->add(''); $oPage->add(''); $oPage->add(''); @@ -933,6 +948,7 @@ function SelectOptions(WebPage $oPage) { header_line = 1; } + var encoding = $('input[name=encoding]').val(); $('#preview').block(); @@ -945,7 +961,7 @@ function SelectOptions(WebPage $oPage) } ajax_request = $.post('ajax.csvimport.php', - { operation: 'parser_preview', enctype: 'multipart/form-data', csvdata: $("#csvdata_truncated").val(), separator: separator, qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line }, + { operation: 'parser_preview', enctype: 'multipart/form-data', csvdata: $("#csvdata_truncated").val(), separator: separator, qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line, encoding: encoding }, function(data) { $('#preview').empty(); $('#preview').append(data); @@ -965,6 +981,23 @@ EOF */ function Welcome(iTopWebPage $oPage) { + // Encodings supported: + // ICONV_CODE => Display Name + // Each iconv installation supports different encodings + // Some reasonably common and useful encodnings are listed here + $aPossibleEncodings = array( + 'UTF-8' => 'Unicode (UTF-8)', + 'ISO-8859-1' => 'Western (ISO-8859-1)', + 'WINDOWS-1251' => 'Cyrilic (Windows 1251)', + 'WINDOWS-1252' => 'Western (Windows 1252)', + 'ISO-8859-15' => 'Western (ISO-8859-15)', + ); + // Some more encodings can be specified in the config file + $aExtraCharsets = utils::GetConfig()->GetCSVImportCharsets(); + + $aPossibleEncodings = array_merge($aPossibleEncodings, $aExtraCharsets); + asort($aPossibleEncodings); + $oPage->add("

".Dict::S('UI:Title:BulkImport+')."

\n"); $oPage->AddTabContainer('tabs1'); @@ -974,12 +1007,26 @@ function Welcome(iTopWebPage $oPage) $iSkippedLines = utils::ReadParam('nb_skipped_lines', ''); $sClassName = utils::ReadParam('class_name', ''); $bAdvanced = utils::ReadParam('advanced', 0); + $sEncoding = utils::ReadParam('encoding', 'UTF-8'); $sFileLoadHtml = '

'.Dict::S('UI:CSVImport:SelectFile').'

'. - '

'. - '

'. - '

'. - '

'. + '

'; + + $sFileLoadHtml .= '

'.Dict::S('UI:CSVImport:Encoding').': '; + $sFileLoadHtml .= '

'; + $sFileLoadHtml .= '

'. + ''. + ''. ''. ''. ''. @@ -989,7 +1036,19 @@ function Welcome(iTopWebPage $oPage) $oPage->AddToTab('tabs1', Dict::S('UI:CSVImport:Tab:LoadFromFile'), $sFileLoadHtml); $sCSVData = utils::ReadParam('csvdata', ''); $sPasteDataHtml = '

'.Dict::S('UI:CSVImport:PasteData').'

'. - '

'. + '

'; + $sPasteDataHtml .= '

'.Dict::S('UI:CSVImport:Encoding').': '; + $sPasteDataHtml .= '

'. '

'. ''. ''.