Improved import.php and synchro_import.php: added 'date_format' (example: %d/%m/%Y %H:%i:%s)

SVN:trunk[1272]
2026-04-23 10:38:45 +02:00 · 2011-05-23 13:42:33 +00:00
parent f8d794bc93
commit 9704dd8e35
5 changed files with 253 additions and 39 deletions
--- a/application/utils.inc.php
+++ b/application/utils.inc.php
@@ -264,12 +264,59 @@ class utils
        return $iReturn;
    }

-    /**
-     * Returns an absolute URL to the current page
-     * @param $bQueryString bool True to also get the query string, false otherwise
-     * @param $bForceHTTPS bool True to force HTTPS, false otherwise
-     * @return string The absolute URL to the current page
-     */                   
+	/**
+	 * Helper function to convert a string to a date, given a format specification. It replaces strtotime which does not allow for specifying a date in a french format (for instance)
+	 * Example: StringToTime('01/05/11 12:03:45', '%d/%m/%y %H:%i:%s')
+	 * @param string $sDate
+	 * @param string $sFormat
+	 * @return timestamp or false if the input format is not correct
+	 */	 	  
+	public static function StringToTime($sDate, $sFormat)
+	{
+	   // Source: http://php.net/manual/fr/function.strftime.php
+		// (alternative: http://www.php.net/manual/fr/datetime.formats.date.php)
+		static $aDateTokens = null;
+		static $aDateRegexps = null;
+		if (is_null($aDateTokens))
+		{
+		   $aSpec = array(
+				'%d' =>'(?<day>[0-9]{2})',
+				'%m' => '(?<month>[0-9]{2})',
+				'%y' => '(?<year>[0-9]{2})',
+				'%Y' => '(?<year>[0-9]{4})',
+				'%H' => '(?<hour>[0-2][0-9])',
+				'%i' => '(?<minute>[0-5][0-9])',
+				'%s' => '(?<second>[0-5][0-9])',
+				);
+			$aDateTokens = array_keys($aSpec);
+			$aDateRegexps = array_values($aSpec);
+		}
+	   
+	   $sDateRegexp = str_replace($aDateTokens, $aDateRegexps, $sFormat);
+	   
+	   if (preg_match('!^(?<head>)'.$sDateRegexp.'(?<tail>)$!', $sDate, $aMatches))
+	   {
+			$sYear = isset($aMatches['year']) ? $aMatches['year'] : 0;
+			$sMonth = isset($aMatches['month']) ? $aMatches['month'] : 1;
+			$sDay = isset($aMatches['day']) ? $aMatches['day'] : 1;
+			$sHour = isset($aMatches['hour']) ? $aMatches['hour'] : 0;
+			$sMinute = isset($aMatches['minute']) ? $aMatches['minute'] : 0;
+			$sSecond = isset($aMatches['second']) ? $aMatches['second'] : 0;
+			return strtotime("$sYear-$sMonth-$sDay $sHour:$sMinute:$sSecond");
+		}
+	   else
+	   {
+	   	return false;
+	   }
+	   // http://www.spaweditor.com/scripts/regex/index.php
+	}
+
+	/**
+	 * Returns an absolute URL to the current page
+	 * @param $bQueryString bool True to also get the query string, false otherwise
+	 * @param $bForceHTTPS bool True to force HTTPS, false otherwise
+	 * @return string The absolute URL to the current page
+	 */                   
 	static public function GetAbsoluteUrl($bQueryString = true, $bForceHTTPS = false)
 	{
 		// Build an absolute URL to this page on this server/port
@@ -313,16 +360,16 @@ class utils
 			}
 			$_SERVER['REQUEST_URI'] = $sPath;
 		}
-   		$sPath = $_SERVER['REQUEST_URI'];
-        if (!$bQueryString)
-        {
-            // remove all the parameters from the query string
-            $iQuestionMarkPos = strpos($sPath, '?');
-            if ($iQuestionMarkPos !== false)
-            {
-                $sPath = substr($sPath, 0, $iQuestionMarkPos);
-            }
-        } 
+		$sPath = $_SERVER['REQUEST_URI'];
+		if (!$bQueryString)
+		{
+			// remove all the parameters from the query string
+			$iQuestionMarkPos = strpos($sPath, '?');
+			if ($iQuestionMarkPos !== false)
+			{
+				$sPath = substr($sPath, 0, $iQuestionMarkPos);
+			}
+		} 
 		$sUrl = "$sProtocol://{$sServerName}{$sPort}{$sPath}";
 		
 		return $sUrl;
--- a/core/bulkchange.class.inc.php
+++ b/core/bulkchange.class.inc.php
@@ -252,8 +252,9 @@ class BulkChange
 	protected $m_aReconcilKeys; // attcode (attcode = 'id' for the pkey)
 	protected $m_sSynchroScope; // OQL - if specified, then the missing items will be reported
 	protected $m_aOnDisappear; // array of attcode => value, values to be set when an object gets out of scope (ignored if no scope has been defined)
+	protected $m_sDateFormat; // Date format specification, see utils::StringToTime()

-	public function __construct($sClass, $aData, $aAttList, $aExtKeys, $aReconcilKeys, $sSynchroScope = null, $aOnDisappear = null)
+	public function __construct($sClass, $aData, $aAttList, $aExtKeys, $aReconcilKeys, $sSynchroScope = null, $aOnDisappear = null, $sDateFormat = null)
 	{
 		$this->m_sClass = $sClass;
 		$this->m_aData = $aData;
@@ -262,6 +263,7 @@ class BulkChange
 		$this->m_aExtKeys = $aExtKeys;
 		$this->m_sSynchroScope = $sSynchroScope;
 		$this->m_aOnDisappear = $aOnDisappear;
+		$this->m_sDateFormat = $sDateFormat;
 	}

 	protected $m_bReportHtml = false;
@@ -412,7 +414,7 @@ class BulkChange
 				}
 			}
 			else
-			{ 
+			{
 				$res = $oTargetObj->CheckValue($sAttCode, $aRowData[$iCol]);
 				if ($res === true)
 				{
@@ -702,6 +704,35 @@ class BulkChange
 			exit;
 		}

+		$aResult = array();
+
+		if (!is_null($this->m_sDateFormat) && (strlen($this->m_sDateFormat) > 0))
+		{
+			// Translate dates from the source data
+			//
+			foreach ($this->m_aAttList as $sAttCode => $iCol)
+			{
+				$oAttDef = MetaModel::GetAttributeDef($this->m_sClass, $sAttCode);
+				if ($oAttDef instanceof AttributeDateTime)
+				{
+					foreach($this->m_aData as $iRow => $aRowData)
+					{
+						$sNewDate = utils::StringToTime($this->m_aData[$iRow][$iCol], $this->m_sDateFormat);
+						if ($sNewDate !== false)
+						{
+							// Todo - improve the reporting
+							$this->m_aData[$iRow][$iCol] = $sNewDate;
+						}
+						else
+						{
+							// Leave the cell unchanged
+							$aResult[$iRow]["__STATUS__"]= new RowStatus_Issue("wrong date format");
+							$aResult[$iRow][$sAttCode] = new CellStatus_Issue(null, $this->m_aData[$iRow][$iCol], 'Wrong date format');
+						}
+					}
+				}
+			}
+		}

 		// Compute the results
 		//
@@ -709,9 +740,13 @@ class BulkChange
 		{
 			$aVisited = array();
 		}
-		$aResult = array();
 		foreach($this->m_aData as $iRow => $aRowData)
 		{
+			if (isset($aResult[$iRow]["__STATUS__"]))
+			{
+				// An issue at the earlier steps - skip the rest
+				continue;
+			}
 			$oReconciliationFilter = new CMDBSearchFilter($this->m_sClass);
 			$bSkipQuery = false;
 			foreach($this->m_aReconcilKeys as $sAttCode)
@@ -798,8 +833,28 @@ class BulkChange
 					$aResult[$iRow]["finalclass"]= 'n/a';
 				}
 			}
-	
-			// Whatever happened, do report the reconciliation values
+		}
+
+		if (!is_null($this->m_sSynchroScope))
+		{
+			// Compute the delta between the scope and visited objects
+			$oScopeSearch = DBObjectSearch::FromOQL($this->m_sSynchroScope);
+			$oScopeSet = new DBObjectSet($oScopeSearch);
+			while ($oObj = $oScopeSet->Fetch())
+			{
+				$iObj = $oObj->GetKey();
+				if (!in_array($iObj, $aVisited))
+				{
+					$iRow++;
+					$this->UpdateMissingObject($aResult, $iRow, $oObj, $oChange);
+				}
+			}
+		}
+
+		// Fill in the blanks - the result matrix is expected to be 100% complete
+		//
+		foreach($this->m_aData as $iRow => $aRowData)
+		{
 			foreach($this->m_aAttList as $iCol)
 			{
 				if (!array_key_exists($iCol, $aResult[$iRow]))
@@ -824,22 +879,6 @@ class BulkChange
 			}
 		}

-		if (!is_null($this->m_sSynchroScope))
-		{
-			// Compute the delta between the scope and visited objects
-			$oScopeSearch = DBObjectSearch::FromOQL($this->m_sSynchroScope);
-			$oScopeSet = new DBObjectSet($oScopeSearch);
-			while ($oObj = $oScopeSet->Fetch())
-			{
-				$iObj = $oObj->GetKey();
-				if (!in_array($iObj, $aVisited))
-				{
-					$iRow++;
-					$this->UpdateMissingObject($aResult, $iRow, $oObj, $oChange);
-				}
-			}
-		}
-
 		return $aResult;
 	}

--- a/synchro/synchro_import.php
+++ b/synchro/synchro_import.php
@@ -91,6 +91,13 @@ $aPageParams = array
 		'default' => 'UTF-8',
 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',
 	),
+	'date_format' => array
+	(
+		'mandatory' => false,
+		'modes' => 'http,cli',
+		'default' => '',
+		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',
+	),
 	'separator' => array
 	(
 		'mandatory' => false,
@@ -192,6 +199,22 @@ function ReadMandatoryParam($oP, $sParam)
 	return trim($sValue);
 }

+function ChangeDateFormat($sProposedDate, $sDateFormat)
+{
+	// Make sure this is a valid MySQL datetime
+	$iTime = utils::StringToTime($sProposedDate, $sDateFormat);
+	if ($iTime !== false)
+	{
+		$sDate = date('Y-m-d H:i:s', $iTime);
+		return $sDate;
+	}
+	else
+	{
+		return false;
+	}
+}
+
+
 /////////////////////////////////
 // Main program

@@ -264,6 +287,7 @@ try
 	$sSep = ReadParam($oP, 'separator');
 	$sQualifier = ReadParam($oP, 'qualifier');
 	$sCharSet = ReadParam($oP, 'charset');
+	$sDateFormat = ReadParam($oP, 'date_format');
 	$sOutput = ReadParam($oP, 'output');
 //	$sReportLevel = ReadParam($oP, 'reportlevel');
 	$sSimulate = ReadParam($oP, 'simulate');
@@ -271,6 +295,12 @@ try

 	$oLoadStartDate = new DateTime(); // Now

+   // Note about date formatting: These MySQL settings are read-only... and in fact unused :-(
+	// SET SESSION date_format = '%d/%m/%Y';
+   // SET SESSION datetime_format = '%d/%m/%Y %H:%i:%s';
+   // Therefore, we have to allow users to transform the format according to a given specification: date_format
+
+
 	//////////////////////////////////////////////////
 	//
 	// Statistics
@@ -358,8 +388,21 @@ try

 	// Check columns
 	$aColumns = $oDataSource->GetSQLColumns();
+	$aDateColumns = $oDataSource->GetDateSQLColumns();
+	$aIsDateToTransform = array();
+	$aDateToTransformReport = array();
 	foreach($aInputColumns as $iFieldId => $sInputColumn)
 	{
+		if ((strlen($sDateFormat) > 0) && (array_key_exists($sInputColumn, $aDateColumns)))
+		{
+			$aIsDateToTransform[$iFieldId] = true;
+			$aDateToTransformReport[] = $sInputColumn;
+		}
+		else
+		{
+			$aIsDateToTransform[$iFieldId] = false;
+		}
+
 		if ($sInputColumn == 'primary_key')
 		{
 			$iPrimaryKeyCol = $iFieldId;
@@ -420,6 +463,22 @@ try
 					{
 						$aValues[] = 'NULL';
 					}
+					elseif ($aIsDateToTransform[$iCol])
+					{
+						$sDate = ChangeDateFormat($value, $sDateFormat);
+						if ($sDate === false)
+						{
+							$aValues[] = CMDBSource::Quote('');
+							if ($sOutput == 'details')
+							{
+								$oP->add("$iRow: Wrong format for date field: '$value' (skipped column)\n");
+							}
+						}
+						else
+						{
+							$aValues[] = CMDBSource::Quote($sDate);
+						}
+					}
 					else
 					{
 						$aValues[] = CMDBSource::Quote($value);
@@ -446,7 +505,26 @@ try
 					if ($iCol == $iPrimaryKeyCol) continue;
 		
 					$sCol = $aInputColumns[$iCol];
-					$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]);
+					if ($aIsDateToTransform[$iCol])
+					{
+						$sDate = ChangeDateFormat($aRow[$iCol], $sDateFormat);
+						if ($sDate === false)
+						{
+							// Skip this column spec
+							if ($sOutput == 'details')
+							{
+								$oP->add("$iRow: Wrong format for date field: '".$aRow[$iCol]."' (skipped column)\n");
+							}
+						}
+						else
+						{
+							$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($sDate);
+						}
+					}
+					else
+					{
+						$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]);
+					}
 				}
 				$sValuePairs = implode(', ', $aValuePairs);
 				$sUpdateQuery = "UPDATE `$sTable` SET $sValuePairs WHERE $sReconciliationCondition";
@@ -469,6 +547,14 @@ try
 			$oP->add_comment("Separator: ".$sSep);
 			$oP->add_comment("Qualifier: ".$sQualifier);
 			$oP->add_comment("Charset Encoding:".$sCharSet);
+			if (strlen($sDateFormat) > 0)
+			{
+				$oP->add_comment("Date format: '$sDateFormat', applied to columns {".implode(', ', $aDateToTransformReport)."}");
+			}
+			else
+			{
+				$oP->add_comment("Date format: <none>");
+			}
 			$oP->add_comment("Data Size: ".strlen($sCSVData));
 			$oP->add_comment("Data Lines: ".$iLineCount);
 			$oP->add_comment("Columns: ".implode(', ', $aInputColumns));
--- a/synchro/synchrodatasource.class.inc.php
+++ b/synchro/synchrodatasource.class.inc.php
@@ -1095,6 +1095,24 @@ EOF
 		return $aColumns;
 	}
 	
+	/**
+	 * Get the list of Date and Datetime SQL columns
+	 */
+	public function GetDateSQLColumns()
+	{
+		$aDateAttributes = array();
+		
+		$sClass = $this->GetTargetClass();
+		foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
+		{
+			if ($oAttDef instanceof AttributeDateTime)
+			{
+				$aDateAttributes[] = $sAttCode;
+			}
+		}
+		return $this->GetSQLColumns($aDateAttributes);
+	}
+
 	public function IsRunning()
 	{
 		$sOQL = "SELECT SynchroLog WHERE sync_source_id = :source_id AND status='running'";
--- a/webservices/import.php
+++ b/webservices/import.php
@@ -90,6 +90,13 @@ $aPageParams = array
 		'default' => 'UTF-8',
 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',
 	),
+	'date_format' => array
+	(
+		'mandatory' => false,
+		'modes' => 'http,cli',
+		'default' => '',
+		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',
+	),
 	'separator' => array
 	(
 		'mandatory' => false,
@@ -269,6 +276,7 @@ try
 	$sSep = ReadParam($oP, 'separator');
 	$sQualifier = ReadParam($oP, 'qualifier');
 	$sCharSet = ReadParam($oP, 'charset');
+	$sDateFormat = ReadParam($oP, 'date_format');
 	$sOutput = ReadParam($oP, 'output');
 //	$sReportLevel = ReadParam($oP, 'reportlevel');
 	$sReconcKeys = ReadParam($oP, 'reconciliationkeys');
@@ -304,6 +312,11 @@ try
 		throw new BulkLoadException("Unknown output format: '$sOutput'");
 	}

+	if (strlen($sDateFormat) == 0)
+	{
+		$sDateFormat = null;
+	}
+	
 /*
 	$aReportLevels = explode('|', $sReportLevel);
 	foreach($aReportLevels as $sLevel)
@@ -331,6 +344,14 @@ try
 		$oP->add_comment("Separator: ".$sSep);
 		$oP->add_comment("Qualifier: ".$sQualifier);
 		$oP->add_comment("Charset Encoding:".$sCharSet);
+		if (strlen($sDateFormat) > 0)
+		{
+			$oP->add_comment("Date format: '$sDateFormat'");
+		}
+		else
+		{
+			$oP->add_comment("Date format: <none>");
+		}
 		$oP->add_comment("Data Size: ".strlen($sCSVData));
 	}
 	//////////////////////////////////////////////////
@@ -592,7 +613,10 @@ try
 		$aData,
 		$aAttList,
 		$aExtKeys,
-		$aFinalReconcilKeys
+		$aFinalReconcilKeys,
+		null, // synchro scope
+		null, // on delete
+		$sDateFormat
 	);

 	if ($bSimulate)