N°1001 switch DB charset from utf8 to utf8mb4 to allow characters outside of the BMP

* use centralized constants instead of literal values in code * remove config parameters 'db_character_set' and 'db_collation' * always fix charset when creating/altering column * backup : use utf8mb4 only for mysqldump >= 5.5.33 (was introduced in 5.5.3 but only available in 5.5.33 for programs) SVN:trunk[5443]
2026-02-13 07:24:13 +01:00 · 2018-03-16 09:59:16 +00:00
parent fd7d30333f
commit b219161011
7 changed files with 171 additions and 55 deletions
--- a/core/attributedef.class.inc.php
+++ b/core/attributedef.class.inc.php
@@ -1,5 +1,5 @@
 <?php
-// Copyright (C) 2010-2017 Combodo SARL
+// Copyright (C) 2010-2018 Combodo SARL
 //
 //   This file is part of iTop.
 //
@@ -20,7 +20,7 @@
 /**
 * Typology for the attributes
 *
- * @copyright   Copyright (C) 2010-2017 Combodo SARL
+ * @copyright   Copyright (C) 2010-2018 Combodo SARL
 * @license	 http://opensource.org/licenses/AGPL-3.0
 */

@@ -111,6 +111,15 @@ define('LINKSET_EDITMODE_ADDREMOVE', 4); // The "linked" objects can be added/re
 */
 abstract class AttributeDefinition
 {
+	/**
+	 * SQL charset & collation declaration for text columns
+	 *
+	 * @see https://dev.mysql.com/doc/refman/5.7/en/charset-column.html
+	 * @since 2.5 #1001 switch to utf8mb4
+	 */
+	const SQL_TEXT_COLUMNS_CHARSET = ' CHARACTER SET '.DEFAULT_CHARACTER_SET.' COLLATE '.DEFAULT_COLLATION;
+
+
 	public function GetType()
 	{
 		return Dict::S('Core:'.get_class($this));
@@ -473,7 +482,18 @@ abstract class AttributeDefinition

 	public function GetSQLExpressions($sPrefix = '') {return array();} // returns suffix/expression pairs (1 in most of the cases), for READING (Select)
 	public function FromSQLToValue($aCols, $sPrefix = '') {return null;} // returns a value out of suffix/value pairs, for SELECT result interpretation
-	public function GetSQLColumns($bFullSpec = false) {return array();} // returns column/spec pairs (1 in most of the cases), for STRUCTURING (DB creation)
+
+	/**
+	 * @param bool $bFullSpec
+	 *
+	 * @return array column/spec pairs (1 in most of the cases), for STRUCTURING (DB creation)
+	 * @see \CMDBSource::GetFieldSpec()
+	 */
+	public function GetSQLColumns($bFullSpec = false)
+	{
+		return array();
+	}
+
 	public function GetSQLValues($value) {return array();} // returns column/value pairs (1 in most of the cases), for WRITING (Insert, Update)
 	public function RequiresIndex() {return false;}
 	public function CopyOnAllTables() {return false;}
@@ -1482,7 +1502,9 @@ class AttributeDBFieldVoid extends AttributeDefinition
 	// To be overriden, used in GetSQLColumns
 	protected function GetSQLCol($bFullSpec = false)
 	{
-		return "VARCHAR(255)".($bFullSpec ? $this->GetSQLColSpec() : '');
+		return 'VARCHAR(255)'
+			.self::SQL_TEXT_COLUMNS_CHARSET
+			.($bFullSpec ? $this->GetSQLColSpec() : '');
 	}
 	protected function GetSQLColSpec()
 	{
@@ -2092,7 +2114,13 @@ class AttributeString extends AttributeDBField
 	}

 	public function GetEditClass() {return "String";}
-	protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(255)".($bFullSpec ? $this->GetSQLColSpec() : '');}
+
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return 'VARCHAR(255)'
+			.self::SQL_TEXT_COLUMNS_CHARSET
+			.($bFullSpec ? $this->GetSQLColSpec() : '');
+	}

 	public function GetValidationPattern()
 	{
@@ -2478,7 +2506,13 @@ class AttributePassword extends AttributeString
 	}

 	public function GetEditClass() {return "Password";}
-	protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(64)".($bFullSpec ? $this->GetSQLColSpec() : '');}
+
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return "VARCHAR(64)"
+			.self::SQL_TEXT_COLUMNS_CHARSET
+			.($bFullSpec ? $this->GetSQLColSpec() : '');
+	}

 	public function GetMaxSize()
 	{
@@ -2604,8 +2638,11 @@ define('WIKI_OBJECT_REGEXP', '/\[\[(.+):(.+)\]\]/U');
 class AttributeText extends AttributeString
 {
 	public function GetEditClass() {return ($this->GetFormat() == 'text') ? 'Text' : "HTML";}
-	
-	protected function GetSQLCol($bFullSpec = false) {return "TEXT";}
+
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return "TEXT".self::SQL_TEXT_COLUMNS_CHARSET;
+	}

 	public function GetSQLColumns($bFullSpec = false)
 	{
@@ -2614,7 +2651,7 @@ class AttributeText extends AttributeString
 		if ($this->GetOptional('format', null) != null )
 		{
 			// Add the extra column only if the property 'format' is specified for the attribute
-			$aColumns[$this->Get('sql').'_format'] = "ENUM('text','html')";
+			$aColumns[$this->Get('sql').'_format'] = "ENUM('text','html')".self::SQL_TEXT_COLUMNS_CHARSET;
 			if ($bFullSpec)
 			{
 				$aColumns[$this->Get('sql').'_format'].= " DEFAULT 'text'"; // default 'text' is for migrating old records
@@ -2936,7 +2973,10 @@ class AttributeText extends AttributeString
 */
 class AttributeLongText extends AttributeText
 {
-	protected function GetSQLCol($bFullSpec = false) {return "LONGTEXT";}
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return "LONGTEXT".self::SQL_TEXT_COLUMNS_CHARSET;
+	}

 	public function GetMaxSize()
 	{
@@ -3114,7 +3154,8 @@ class AttributeCaseLog extends AttributeLongText
 	public function GetSQLColumns($bFullSpec = false)
 	{
 		$aColumns = array();
-		$aColumns[$this->GetCode()] = 'LONGTEXT'; // 2^32 (4 Gb)
+		$aColumns[$this->GetCode()] = 'LONGTEXT' // 2^32 (4 Gb)
+			.self::SQL_TEXT_COLUMNS_CHARSET;
 		$aColumns[$this->GetCode().'_index'] = 'BLOB';
 		return $aColumns;
 	}
@@ -3457,11 +3498,15 @@ class AttributeEnum extends AttributeString
 			// In particular, I had to remove unnecessary spaces to
 			// make sure that this string will match the field type returned by the DB
 			// (used to perform a comparison between the current DB format and the data model)
-			return "ENUM(".implode(",", $aValues).")".($bFullSpec ? $this->GetSQLColSpec() : '');
+			return "ENUM(".implode(",", $aValues).")"
+				.self::SQL_TEXT_COLUMNS_CHARSET
+				.($bFullSpec ? $this->GetSQLColSpec() : '');
 		}
 		else
 		{
-			return "VARCHAR(255)".($bFullSpec ? " DEFAULT ''" : ""); // ENUM() is not an allowed syntax!
+			return "VARCHAR(255)"
+				.self::SQL_TEXT_COLUMNS_CHARSET
+				.($bFullSpec ? " DEFAULT ''" : ""); // ENUM() is not an allowed syntax!
 		}
 	}
 	
@@ -5190,7 +5235,12 @@ class AttributeURL extends AttributeString
 		return array_merge(parent::ListExpectedParams(), array("target"));
 	}

-	protected function GetSQLCol($bFullSpec = false) {return "VARCHAR(2048)".($bFullSpec ? $this->GetSQLColSpec() : '');}
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return "VARCHAR(2048)"
+			.self::SQL_TEXT_COLUMNS_CHARSET
+			.($bFullSpec ? $this->GetSQLColSpec() : '');
+	}

 	public function GetMaxSize()
 	{
@@ -5363,8 +5413,8 @@ class AttributeBlob extends AttributeDefinition
 	{
 		$aColumns = array();
 		$aColumns[$this->GetCode().'_data'] = 'LONGBLOB'; // 2^32 (4 Gb)
-		$aColumns[$this->GetCode().'_mimetype'] = 'VARCHAR(255)';
-		$aColumns[$this->GetCode().'_filename'] = 'VARCHAR(255)';
+		$aColumns[$this->GetCode().'_mimetype'] = 'VARCHAR(255)'.self::SQL_TEXT_COLUMNS_CHARSET;
+		$aColumns[$this->GetCode().'_filename'] = 'VARCHAR(255)'.self::SQL_TEXT_COLUMNS_CHARSET;
 		return $aColumns;
 	}

@@ -6501,7 +6551,7 @@ class AttributeOneWayPassword extends AttributeDefinition
 	public function GetImportColumns()
 	{
 		$aColumns = array();
-		$aColumns[$this->GetCode()] = 'TINYTEXT';
+		$aColumns[$this->GetCode()] = 'TINYTEXT'.self::SQL_TEXT_COLUMNS_CHARSET;
 		return $aColumns;
 	}

@@ -6569,7 +6619,11 @@ class AttributeOneWayPassword extends AttributeDefinition
 class AttributeTable extends AttributeDBField
 {
 	public function GetEditClass() {return "Table";}
-	protected function GetSQLCol($bFullSpec = false) {return "LONGTEXT";}
+
+	protected function GetSQLCol($bFullSpec = false)
+	{
+		return "LONGTEXT".self::SQL_TEXT_COLUMNS_CHARSET;
+	}

 	public function GetMaxSize()
 	{
@@ -6970,7 +7024,9 @@ class AttributeRedundancySettings extends AttributeDBField
 	public function GetEditClass() {return "RedundancySetting";}
 	protected function GetSQLCol($bFullSpec = false)
 	{
-		return "VARCHAR(20)".($bFullSpec ? $this->GetSQLColSpec() : '');
+		return "VARCHAR(20)"
+			.self::SQL_TEXT_COLUMNS_CHARSET
+			.($bFullSpec ? $this->GetSQLColSpec() : '');
 	}


--- a/core/cmdbsource.class.inc.php
+++ b/core/cmdbsource.class.inc.php
@@ -144,8 +144,8 @@ class CMDBSource
 		self::Init($sServer, $sUser, $sPwd, $sSource, $sTlsKey, $sTlsCert, $sTlsCA, $sTlsCaPath, $sTlsCipher,
 			$sTlsVerifyServerCert);

-		$sCharacterSet = $oConfig->Get('db_character_set');
-		$sCollation = $oConfig->Get('db_collation');
+		$sCharacterSet = DEFAULT_CHARACTER_SET;
+		$sCollation = DEFAULT_COLLATION;
 		self::SetCharacterSet($sCharacterSet, $sCollation);
 	}

@@ -373,7 +373,7 @@ class CMDBSource
 		return (!empty($sResult));
 	}

-	public static function SetCharacterSet($sCharset = 'utf8', $sCollation = 'utf8_general_ci')
+	public static function SetCharacterSet($sCharset = DEFAULT_CHARACTER_SET, $sCollation = DEFAULT_COLLATION)
 	{
 		if (strlen($sCharset) > 0)
 		{
@@ -455,7 +455,7 @@ class CMDBSource
 	 */
 	public static function CreateDB($sSource)
 	{
-		self::Query("CREATE DATABASE `$sSource` CHARACTER SET utf8 COLLATE utf8_unicode_ci");
+		self::Query("CREATE DATABASE `$sSource` CHARACTER SET ".DEFAULT_CHARACTER_SET." COLLATE ".DEFAULT_COLLATION);
 		self::SelectDB($sSource);
 	}

@@ -893,6 +893,13 @@ class CMDBSource
 		return ($aFieldData["Type"]);
 	}

+	/**
+	 * @param string $sTable
+	 * @param string $sField
+	 *
+	 * @return bool|string
+	 * @see \AttributeDefinition::GetSQLColumns()
+	 */
 	public static function GetFieldSpec($sTable, $sField)
 	{
 		$aTableInfo = self::GetTableInfo($sTable);
--- a/core/config.class.inc.php
+++ b/core/config.class.inc.php
@@ -42,8 +42,12 @@ class ConfigException extends CoreException
 {
 }

-define('DEFAULT_CHARACTER_SET', 'utf8');
-define('DEFAULT_COLLATION', 'utf8_unicode_ci');
+// was utf8 but it only supports BMP chars (https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)
+// so we switched to utf8mb4 in iTop 2.5, adding dependency to MySQL 5.5.3
+// The config params db_character_set and db_collation were introduced as a temporary workaround and removed in iTop 2.5
+// now everything uses those fixed value !
+define('DEFAULT_CHARACTER_SET', 'utf8mb4');
+define('DEFAULT_COLLATION', 'utf8mb4_unicode_ci');

 define('DEFAULT_LOG_GLOBAL', true);
 define('DEFAULT_LOG_NOTIFICATION', true);
@@ -194,19 +198,21 @@ class Config
 			'source_of_value' => '',
 			'show_in_conf_sample' => false,
 		),
-		'db_character_set' => array(
+		'db_character_set' => array( // @deprecated to remove in 2.7 ? #1001 utf8mb4 switch
 			'type' => 'string',
-			'default' => null,
+			'description' => 'Deprecated since iTop 2.5 : now using utf8mb4',
+			'default' => 'DEPRECATED_2.5',
 			'value' => '',
 			'source_of_value' => '',
-			'show_in_conf_sample' => true,
+			'show_in_conf_sample' => false,
 		),
-		'db_collation' => array(
+		'db_collation' => array( // @deprecated to remove in 2.7 ? #1001 utf8mb4 switch
 			'type' => 'string',
-			'default' => null,
+			'description' => 'Deprecated since iTop 2.5 : now using utf8mb4_unicode_ci',
+			'default' => 'DEPRECATED_2.5',
 			'value' => '',
 			'source_of_value' => '',
-			'show_in_conf_sample' => true,
+			'show_in_conf_sample' => false,
 		),
 		'skip_check_to_write' => array(
 			'type' => 'bool',
@@ -1456,22 +1462,22 @@ class Config

 	/**
 	 * @return string
-	 * @deprecated 2.5 will be removed in 2.6
-	 * @see Config::Get() as a replacement
+	 * @deprecated 2.5 will be removed in 2.6 #1001 utf8mb4 switch
+	 * @see Config::DEFAULT_CHARACTER_SET
 	 */
 	public function GetDBCharacterSet()
 	{
-		return $this->Get('db_character_set');
+		return DEFAULT_CHARACTER_SET;
 	}

 	/**
 	 * @return string
-	 * @deprecated 2.5 will be removed in 2.6
-	 * @see Config::Get() as a replacement
+	 * @deprecated 2.5 will be removed in 2.6 #1001 utf8mb4 switch
+	 * @see Config::DEFAULT_COLLATION
 	 */
 	public function GetDBCollation()
 	{
-		return $this->Get('db_collation');
+		return DEFAULT_COLLATION;
 	}

 	/**
--- a/core/metamodel.class.php
+++ b/core/metamodel.class.php
@@ -4938,13 +4938,15 @@ abstract class MetaModel
 			//
 			$sTable = self::DBGetTable($sClass);
 			$sKeyField = self::DBGetKey($sClass);
+			$sDbCharset = DEFAULT_CHARACTER_SET;
+			$sDbCollation = DEFAULT_COLLATION;
 			$sAutoIncrement = (self::IsAutoIncrementKey($sClass) ? "AUTO_INCREMENT" : "");
 			$sKeyFieldDefinition = "`$sKeyField` INT(11) NOT NULL $sAutoIncrement PRIMARY KEY";
 			if (!CMDBSource::IsTable($sTable))
 			{
-				$aErrors[$sClass]['*'][] = "table '$sTable' could not be found into the DB";
-				$aSugFix[$sClass]['*'][] = "CREATE TABLE `$sTable` ($sKeyFieldDefinition) ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci";
-				$aCreateTable[$sTable] = "ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci";
+				$aErrors[$sClass]['*'][] = "table '$sTable' could not be found in the DB";
+				$aSugFix[$sClass]['*'][] = "CREATE TABLE `$sTable` ($sKeyFieldDefinition) ENGINE = ".MYSQL_ENGINE." CHARACTER SET $sDbCharset COLLATE $sDbCollation";
+				$aCreateTable[$sTable] = "ENGINE = ".MYSQL_ENGINE." CHARACTER SET $sDbCharset COLLATE $sDbCollation";
 				$aCreateTableItems[$sTable][$sKeyField] = $sKeyFieldDefinition;
 			}
 			// Check that the key field exists
--- a/datamodels/2.x/itop-backup/dbrestore.class.inc.php
+++ b/datamodels/2.x/itop-backup/dbrestore.class.inc.php
@@ -57,8 +57,8 @@ class DBRestore extends DBBackup
 		}

 		$sDataFileEscaped = self::EscapeShellArg($sDataFile);
-		$sCommand = "$sMySQLExe --default-character-set=utf8 --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sDBName <$sDataFileEscaped 2>&1";
-		$sCommandDisplay = "$sMySQLExe --default-character-set=utf8 --host=$sHost $sPortOption --user=xxxx --password=xxxx $sDBName <$sDataFileEscaped 2>&1";
+		$sCommand = "$sMySQLExe --default-character-set=".DEFAULT_CHARACTER_SET." --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sDBName <$sDataFileEscaped 2>&1";
+		$sCommandDisplay = "$sMySQLExe --default-character-set=".DEFAULT_CHARACTER_SET." --host=$sHost $sPortOption --user=xxxx --password=xxxx $sDBName <$sDataFileEscaped 2>&1";

 		// Now run the command for real
 		$this->LogInfo("Executing command: $sCommandDisplay");
--- a/setup/backup.class.inc.php
+++ b/setup/backup.class.inc.php
@@ -163,6 +163,13 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "

 	class DBBackup
 	{
+		/**
+		 * utf8mb4 was added in MySQL 5.5.3 but works with programs like mysqldump only since MySQL 5.5.33
+		 *
+		 * @since 2.5 see #1001
+		 */
+		const MYSQL_VERSION_WITH_UTF8MB4_IN_PROGRAMS = '5.5.33';
+
 		// To be overriden depending on the expected usages
 		protected function LogInfo($sMsg)
 		{
@@ -386,28 +393,25 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "

 			$this->LogInfo("Starting backup of $this->sDBHost/$this->sDBName(suffix:'$this->sDBSubName')");

-			$sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true);
-			if (empty($sMySQLBinDir))
-			{
-				$sMySQLDump = 'mysqldump';
-			}
-			else
-			{
-				$sMySQLDump = '"'.$sMySQLBinDir.'/mysqldump"';
-			}
+			$sMySQLDump = $this->GetMysqldumpCommand();

 			// Store the results in a temporary file
 			$sTmpFileName = self::EscapeShellArg($sBackupFileName);
-			$sPortOption = self::GetMysqliCliSingleOption('port', $this->iDBPort);

+			$sPortOption = self::GetMysqliCliSingleOption('port', $this->iDBPort);
 			$sTlsOptions = self::GetMysqlCliTlsOptions($this->oConfig);

+			$sMysqldumpVersion = self::GetMysqldumpVersion($sMySQLDump);
+			$bIsMysqldumpSupportUtf8mb4 = (version_compare($sMysqldumpVersion,
+					self::MYSQL_VERSION_WITH_UTF8MB4_IN_PROGRAMS) == -1);
+			$sMysqldumpCharset = $bIsMysqldumpSupportUtf8mb4 ? 'utf8' : DEFAULT_CHARACTER_SET;
+
 			// Delete the file created by tempnam() so that the spawned process can write into it (Windows/IIS)
 			@unlink($sBackupFileName);
 			// Note: opt implicitely sets lock-tables... which cancels the benefit of single-transaction!
 			//       skip-lock-tables compensates and allows for writes during a backup
-			$sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables 2>&1";
-			$sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables";
+			$sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=".$sMysqldumpCharset." --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables 2>&1";
+			$sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=".$sMysqldumpCharset." --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx $sTlsOptions --result-file=$sTmpFileName $sDBName $sTables";

 			// Now run the command for real
 			$this->LogInfo("Executing command: $sCommandDisplay");
@@ -631,6 +635,45 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "

 			return ' --'.$sCliArgName.'='.self::EscapeShellArg($sData);
 		}
+
+		/**
+		 * @return string the command to launch mysqldump (without its params)
+		 */
+		private function GetMysqldumpCommand()
+		{
+			$sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true);
+			if (empty($sMySQLBinDir))
+			{
+				$sMysqldumpCommand = 'mysqldump';
+			}
+			else
+			{
+				$sMysqldumpCommand = '"'.$sMySQLBinDir.'/mysqldump"';
+			}
+
+			return $sMysqldumpCommand;
+		}
+
+		/**
+		 * @param string $sMysqldumpCommand
+		 *
+		 * @return string version of the mysqldump program, as parsed from program return
+		 *
+		 * @uses mysqldump -V Sample return value : mysqldump  Ver 10.13 Distrib 5.7.19, for Win64 (x86_64)
+		 * @since 2.5 needed to check compatibility with utf8mb4 (N°1001)
+		 */
+		private static function GetMysqldumpVersion($sMysqldumpCommand)
+		{
+			$sCommand = $sMysqldumpCommand.' -V';
+			$aOutput = array();
+			exec($sCommand, $aOutput, $iRetCode);
+
+			$sMysqldumpOutput = $aOutput[0];
+			$aDumpVersionMatchResults = array();
+			preg_match('/Distrib (\d\.\d+\.\d+)/', $sMysqldumpOutput, $aDumpVersionMatchResults);
+
+			return $aDumpVersionMatchResults[1];
+		}
 	}
 }

--- a/synchro/synchrodatasource.class.inc.php
+++ b/synchro/synchrodatasource.class.inc.php
@@ -787,7 +787,9 @@ EOF
 		$aFieldDefs[] = "INDEX (primary_key)";
 		$sFieldDefs = implode(', ', $aFieldDefs);

-		$sCreateTable = "CREATE TABLE `$sTable` ($sFieldDefs) ENGINE = ".MYSQL_ENGINE." CHARACTER SET utf8 COLLATE utf8_unicode_ci;";
+		$sDbCharset = DEFAULT_CHARACTER_SET;
+		$sDbCollation = DEFAULT_COLLATION;
+		$sCreateTable = "CREATE TABLE `$sTable` ($sFieldDefs) ENGINE = ".MYSQL_ENGINE." CHARACTER SET ".$sDbCharset." COLLATE ".$sDbCollation.";";
 		CMDBSource::Query($sCreateTable);

 		$aTriggers = $this->GetTriggersDefinition();