diff --git a/core/attributedef.class.inc.php b/core/attributedef.class.inc.php index 300e70a0c..470458185 100644 --- a/core/attributedef.class.inc.php +++ b/core/attributedef.class.inc.php @@ -1,5 +1,5 @@ GetSQLColSpec() : ''); + return 'VARCHAR(255)' + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); } protected function GetSQLColSpec() { @@ -2092,7 +2114,13 @@ class AttributeString extends AttributeDBField } public function GetEditClass() {return "String";} - protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(255)".($bFullSpec ? $this->GetSQLColSpec() : '');} + + protected function GetSQLCol($bFullSpec = false) + { + return 'VARCHAR(255)' + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); + } public function GetValidationPattern() { @@ -2478,7 +2506,13 @@ class AttributePassword extends AttributeString } public function GetEditClass() {return "Password";} - protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(64)".($bFullSpec ? $this->GetSQLColSpec() : '');} + + protected function GetSQLCol($bFullSpec = false) + { + return "VARCHAR(64)" + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); + } public function GetMaxSize() { @@ -2604,8 +2638,11 @@ define('WIKI_OBJECT_REGEXP', '/\[\[(.+):(.+)\]\]/U'); class AttributeText extends AttributeString { public function GetEditClass() {return ($this->GetFormat() == 'text') ? 'Text' : "HTML";} - - protected function GetSQLCol($bFullSpec = false) {return "TEXT";} + + protected function GetSQLCol($bFullSpec = false) + { + return "TEXT".self::SQL_TEXT_COLUMNS_CHARSET; + } public function GetSQLColumns($bFullSpec = false) { @@ -2614,7 +2651,7 @@ class AttributeText extends AttributeString if ($this->GetOptional('format', null) != null ) { // Add the extra column only if the property 'format' is specified for the attribute - $aColumns[$this->Get('sql').'_format'] = "ENUM('text','html')"; + $aColumns[$this->Get('sql').'_format'] = "ENUM('text','html')".self::SQL_TEXT_COLUMNS_CHARSET; if ($bFullSpec) { $aColumns[$this->Get('sql').'_format'].= " DEFAULT 'text'"; // default 'text' is for migrating old records @@ -2936,7 +2973,10 @@ class AttributeText extends AttributeString */ class AttributeLongText extends AttributeText { - protected function GetSQLCol($bFullSpec = false) {return "LONGTEXT";} + protected function GetSQLCol($bFullSpec = false) + { + return "LONGTEXT".self::SQL_TEXT_COLUMNS_CHARSET; + } public function GetMaxSize() { @@ -3114,7 +3154,8 @@ class AttributeCaseLog extends AttributeLongText public function GetSQLColumns($bFullSpec = false) { $aColumns = array(); - $aColumns[$this->GetCode()] = 'LONGTEXT'; // 2^32 (4 Gb) + $aColumns[$this->GetCode()] = 'LONGTEXT' // 2^32 (4 Gb) + .self::SQL_TEXT_COLUMNS_CHARSET; $aColumns[$this->GetCode().'_index'] = 'BLOB'; return $aColumns; } @@ -3457,11 +3498,15 @@ class AttributeEnum extends AttributeString // In particular, I had to remove unnecessary spaces to // make sure that this string will match the field type returned by the DB // (used to perform a comparison between the current DB format and the data model) - return "ENUM(".implode(",", $aValues).")".($bFullSpec ? $this->GetSQLColSpec() : ''); + return "ENUM(".implode(",", $aValues).")" + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); } else { - return "VARCHAR(255)".($bFullSpec ? " DEFAULT ''" : ""); // ENUM() is not an allowed syntax! + return "VARCHAR(255)" + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? " DEFAULT ''" : ""); // ENUM() is not an allowed syntax! } } @@ -5190,7 +5235,12 @@ class AttributeURL extends AttributeString return array_merge(parent::ListExpectedParams(), array("target")); } - protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(2048)".($bFullSpec ? $this->GetSQLColSpec() : '');} + protected function GetSQLCol($bFullSpec = false) + { + return "VARCHAR(2048)" + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); + } public function GetMaxSize() { @@ -5363,8 +5413,8 @@ class AttributeBlob extends AttributeDefinition { $aColumns = array(); $aColumns[$this->GetCode().'_data'] = 'LONGBLOB'; // 2^32 (4 Gb) - $aColumns[$this->GetCode().'_mimetype'] = 'VARCHAR(255)'; - $aColumns[$this->GetCode().'_filename'] = 'VARCHAR(255)'; + $aColumns[$this->GetCode().'_mimetype'] = 'VARCHAR(255)'.self::SQL_TEXT_COLUMNS_CHARSET; + $aColumns[$this->GetCode().'_filename'] = 'VARCHAR(255)'.self::SQL_TEXT_COLUMNS_CHARSET; return $aColumns; } @@ -6501,7 +6551,7 @@ class AttributeOneWayPassword extends AttributeDefinition public function GetImportColumns() { $aColumns = array(); - $aColumns[$this->GetCode()] = 'TINYTEXT'; + $aColumns[$this->GetCode()] = 'TINYTEXT'.self::SQL_TEXT_COLUMNS_CHARSET; return $aColumns; } @@ -6569,7 +6619,11 @@ class AttributeOneWayPassword extends AttributeDefinition class AttributeTable extends AttributeDBField { public function GetEditClass() {return "Table";} - protected function GetSQLCol($bFullSpec = false) {return "LONGTEXT";} + + protected function GetSQLCol($bFullSpec = false) + { + return "LONGTEXT".self::SQL_TEXT_COLUMNS_CHARSET; + } public function GetMaxSize() { @@ -6970,7 +7024,9 @@ class AttributeRedundancySettings extends AttributeDBField public function GetEditClass() {return "RedundancySetting";} protected function GetSQLCol($bFullSpec = false) { - return "VARCHAR(20)".($bFullSpec ? $this->GetSQLColSpec() : ''); + return "VARCHAR(20)" + .self::SQL_TEXT_COLUMNS_CHARSET + .($bFullSpec ? $this->GetSQLColSpec() : ''); } diff --git a/core/cmdbsource.class.inc.php b/core/cmdbsource.class.inc.php index a37429c30..ac2423647 100644 --- a/core/cmdbsource.class.inc.php +++ b/core/cmdbsource.class.inc.php @@ -144,8 +144,8 @@ class CMDBSource self::Init($sServer, $sUser, $sPwd, $sSource, $sTlsKey, $sTlsCert, $sTlsCA, $sTlsCaPath, $sTlsCipher, $sTlsVerifyServerCert); - $sCharacterSet = $oConfig->Get('db_character_set'); - $sCollation = $oConfig->Get('db_collation'); + $sCharacterSet = DEFAULT_CHARACTER_SET; + $sCollation = DEFAULT_COLLATION; self::SetCharacterSet($sCharacterSet, $sCollation); } @@ -373,7 +373,7 @@ class CMDBSource return (!empty($sResult)); } - public static function SetCharacterSet($sCharset = 'utf8', $sCollation = 'utf8_general_ci') + public static function SetCharacterSet($sCharset = DEFAULT_CHARACTER_SET, $sCollation = DEFAULT_COLLATION) { if (strlen($sCharset) > 0) { @@ -455,7 +455,7 @@ class CMDBSource */ public static function CreateDB($sSource) { - self::Query("CREATE DATABASE `$sSource` CHARACTER SET utf8 COLLATE utf8_unicode_ci"); + self::Query("CREATE DATABASE `$sSource` CHARACTER SET ".DEFAULT_CHARACTER_SET." COLLATE ".DEFAULT_COLLATION); self::SelectDB($sSource); } @@ -893,6 +893,13 @@ class CMDBSource return ($aFieldData["Type"]); } + /** + * @param string $sTable + * @param string $sField + * + * @return bool|string + * @see \AttributeDefinition::GetSQLColumns() + */ public static function GetFieldSpec($sTable, $sField) { $aTableInfo = self::GetTableInfo($sTable); diff --git a/core/config.class.inc.php b/core/config.class.inc.php index 1059d8078..0f040fdc5 100644 --- a/core/config.class.inc.php +++ b/core/config.class.inc.php @@ -42,8 +42,12 @@ class ConfigException extends CoreException { } -define('DEFAULT_CHARACTER_SET', 'utf8'); -define('DEFAULT_COLLATION', 'utf8_unicode_ci'); +// was utf8 but it only supports BMP chars (https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html) +// so we switched to utf8mb4 in iTop 2.5, adding dependency to MySQL 5.5.3 +// The config params db_character_set and db_collation were introduced as a temporary workaround and removed in iTop 2.5 +// now everything uses those fixed value ! +define('DEFAULT_CHARACTER_SET', 'utf8mb4'); +define('DEFAULT_COLLATION', 'utf8mb4_unicode_ci'); define('DEFAULT_LOG_GLOBAL', true); define('DEFAULT_LOG_NOTIFICATION', true); @@ -194,19 +198,21 @@ class Config 'source_of_value' => '', 'show_in_conf_sample' => false, ), - 'db_character_set' => array( + 'db_character_set' => array( // @deprecated to remove in 2.7 ? #1001 utf8mb4 switch 'type' => 'string', - 'default' => null, + 'description' => 'Deprecated since iTop 2.5 : now using utf8mb4', + 'default' => 'DEPRECATED_2.5', 'value' => '', 'source_of_value' => '', - 'show_in_conf_sample' => true, + 'show_in_conf_sample' => false, ), - 'db_collation' => array( + 'db_collation' => array( // @deprecated to remove in 2.7 ? #1001 utf8mb4 switch 'type' => 'string', - 'default' => null, + 'description' => 'Deprecated since iTop 2.5 : now using utf8mb4_unicode_ci', + 'default' => 'DEPRECATED_2.5', 'value' => '', 'source_of_value' => '', - 'show_in_conf_sample' => true, + 'show_in_conf_sample' => false, ), 'skip_check_to_write' => array( 'type' => 'bool', @@ -1456,22 +1462,22 @@ class Config /** * @return string - * @deprecated 2.5 will be removed in 2.6 - * @see Config::Get() as a replacement + * @deprecated 2.5 will be removed in 2.6 #1001 utf8mb4 switch + * @see Config::DEFAULT_CHARACTER_SET */ public function GetDBCharacterSet() { - return $this->Get('db_character_set'); + return DEFAULT_CHARACTER_SET; } /** * @return string - * @deprecated 2.5 will be removed in 2.6 - * @see Config::Get() as a replacement + * @deprecated 2.5 will be removed in 2.6 #1001 utf8mb4 switch + * @see Config::DEFAULT_COLLATION */ public function GetDBCollation() { - return $this->Get('db_collation'); + return DEFAULT_COLLATION; } /** diff --git a/core/metamodel.class.php b/core/metamodel.class.php index 918358243..b3aaaa90b 100644 --- a/core/metamodel.class.php +++ b/core/metamodel.class.php @@ -4938,13 +4938,15 @@ abstract class MetaModel // $sTable = self::DBGetTable($sClass); $sKeyField = self::DBGetKey($sClass); + $sDbCharset = DEFAULT_CHARACTER_SET; + $sDbCollation = DEFAULT_COLLATION; $sAutoIncrement = (self::IsAutoIncrementKey($sClass) ? "AUTO_INCREMENT" : ""); $sKeyFieldDefinition = "`$sKeyField` INT(11) NOT NULL $sAutoIncrement PRIMARY KEY"; if (!CMDBSource::IsTable($sTable)) { - $aErrors[$sClass]['*'][] = "table '$sTable' could not be found into the DB"; - $aSugFix[$sClass]['*'][] = "CREATE TABLE `$sTable` ($sKeyFieldDefinition) ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci"; - $aCreateTable[$sTable] = "ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci"; + $aErrors[$sClass]['*'][] = "table '$sTable' could not be found in the DB"; + $aSugFix[$sClass]['*'][] = "CREATE TABLE `$sTable` ($sKeyFieldDefinition) ENGINE = ".MYSQL_ENGINE." CHARACTER SET $sDbCharset COLLATE $sDbCollation"; + $aCreateTable[$sTable] = "ENGINE = ".MYSQL_ENGINE." CHARACTER SET $sDbCharset COLLATE $sDbCollation"; $aCreateTableItems[$sTable][$sKeyField] = $sKeyFieldDefinition; } // Check that the key field exists diff --git a/datamodels/2.x/itop-backup/dbrestore.class.inc.php b/datamodels/2.x/itop-backup/dbrestore.class.inc.php index b8809d11a..d67f551f5 100644 --- a/datamodels/2.x/itop-backup/dbrestore.class.inc.php +++ b/datamodels/2.x/itop-backup/dbrestore.class.inc.php @@ -57,8 +57,8 @@ class DBRestore extends DBBackup } $sDataFileEscaped = self::EscapeShellArg($sDataFile); - $sCommand = "$sMySQLExe --default-character-set=utf8 --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sDBName <$sDataFileEscaped 2>&1"; - $sCommandDisplay = "$sMySQLExe --default-character-set=utf8 --host=$sHost $sPortOption --user=xxxx --password=xxxx $sDBName <$sDataFileEscaped 2>&1"; + $sCommand = "$sMySQLExe --default-character-set=".DEFAULT_CHARACTER_SET." --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sDBName <$sDataFileEscaped 2>&1"; + $sCommandDisplay = "$sMySQLExe --default-character-set=".DEFAULT_CHARACTER_SET." --host=$sHost $sPortOption --user=xxxx --password=xxxx $sDBName <$sDataFileEscaped 2>&1"; // Now run the command for real $this->LogInfo("Executing command: $sCommandDisplay"); diff --git a/setup/backup.class.inc.php b/setup/backup.class.inc.php index a189ef793..c00443c64 100644 --- a/setup/backup.class.inc.php +++ b/setup/backup.class.inc.php @@ -163,6 +163,13 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the " class DBBackup { + /** + * utf8mb4 was added in MySQL 5.5.3 but works with programs like mysqldump only since MySQL 5.5.33 + * + * @since 2.5 see #1001 + */ + const MYSQL_VERSION_WITH_UTF8MB4_IN_PROGRAMS = '5.5.33'; + // To be overriden depending on the expected usages protected function LogInfo($sMsg) { @@ -386,28 +393,25 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the " $this->LogInfo("Starting backup of $this->sDBHost/$this->sDBName(suffix:'$this->sDBSubName')"); - $sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true); - if (empty($sMySQLBinDir)) - { - $sMySQLDump = 'mysqldump'; - } - else - { - $sMySQLDump = '"'.$sMySQLBinDir.'/mysqldump"'; - } + $sMySQLDump = $this->GetMysqldumpCommand(); // Store the results in a temporary file $sTmpFileName = self::EscapeShellArg($sBackupFileName); - $sPortOption = self::GetMysqliCliSingleOption('port', $this->iDBPort); + $sPortOption = self::GetMysqliCliSingleOption('port', $this->iDBPort); $sTlsOptions = self::GetMysqlCliTlsOptions($this->oConfig); + $sMysqldumpVersion = self::GetMysqldumpVersion($sMySQLDump); + $bIsMysqldumpSupportUtf8mb4 = (version_compare($sMysqldumpVersion, + self::MYSQL_VERSION_WITH_UTF8MB4_IN_PROGRAMS) == -1); + $sMysqldumpCharset = $bIsMysqldumpSupportUtf8mb4 ? 'utf8' : DEFAULT_CHARACTER_SET; + // Delete the file created by tempnam() so that the spawned process can write into it (Windows/IIS) @unlink($sBackupFileName); // Note: opt implicitely sets lock-tables... which cancels the benefit of single-transaction! // skip-lock-tables compensates and allows for writes during a backup - $sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables 2>&1"; - $sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables"; + $sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=".$sMysqldumpCharset." --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables 2>&1"; + $sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=".$sMysqldumpCharset." --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables"; // Now run the command for real $this->LogInfo("Executing command: $sCommandDisplay"); @@ -631,6 +635,45 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the " return ' --'.$sCliArgName.'='.self::EscapeShellArg($sData); } + + /** + * @return string the command to launch mysqldump (without its params) + */ + private function GetMysqldumpCommand() + { + $sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true); + if (empty($sMySQLBinDir)) + { + $sMysqldumpCommand = 'mysqldump'; + } + else + { + $sMysqldumpCommand = '"'.$sMySQLBinDir.'/mysqldump"'; + } + + return $sMysqldumpCommand; + } + + /** + * @param string $sMysqldumpCommand + * + * @return string version of the mysqldump program, as parsed from program return + * + * @uses mysqldump -V Sample return value : mysqldump Ver 10.13 Distrib 5.7.19, for Win64 (x86_64) + * @since 2.5 needed to check compatibility with utf8mb4 (N°1001) + */ + private static function GetMysqldumpVersion($sMysqldumpCommand) + { + $sCommand = $sMysqldumpCommand.' -V'; + $aOutput = array(); + exec($sCommand, $aOutput, $iRetCode); + + $sMysqldumpOutput = $aOutput[0]; + $aDumpVersionMatchResults = array(); + preg_match('/Distrib (\d\.\d+\.\d+)/', $sMysqldumpOutput, $aDumpVersionMatchResults); + + return $aDumpVersionMatchResults[1]; + } } } diff --git a/synchro/synchrodatasource.class.inc.php b/synchro/synchrodatasource.class.inc.php index 3cd493e11..b3e59b824 100644 --- a/synchro/synchrodatasource.class.inc.php +++ b/synchro/synchrodatasource.class.inc.php @@ -787,7 +787,9 @@ EOF $aFieldDefs[] = "INDEX (primary_key)"; $sFieldDefs = implode(', ', $aFieldDefs); - $sCreateTable = "CREATE TABLE `$sTable` ($sFieldDefs) ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci;"; + $sDbCharset = DEFAULT_CHARACTER_SET; + $sDbCollation = DEFAULT_COLLATION; + $sCreateTable = "CREATE TABLE `$sTable` ($sFieldDefs) ENGINE = ".MYSQL_ENGINE." CHARACTER SET ".$sDbCharset." COLLATE ".$sDbCollation.";"; CMDBSource::Query($sCreateTable); $aTriggers = $this->GetTriggersDefinition();