New import web service (extracted the bulk load from csvimport (interactive) page - this is still being updated to reflect this factorization)

SVN:code[34]

core/bulkchange.class.inc.php

@@ -0,0 +1,409 @@
+<?php
+
+/**
+ * BulkChange
+ * Interpret a given data set and update the DB accordingly (fake mode avail.) 
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @author      Denis Flaven <denisflave@free.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     1.1.1.1 $
+ */
+
+class BulkChangeException extends CoreException
+{
+}
+
+/**
+ * CellChangeSpec
+ * A series of classes, keeping the information about a given cell: could it be changed or not (and why)?  
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     $itopversion$
+ */
+abstract class CellChangeSpec
+{
+	protected $m_proposedValue;
+
+	public function __construct($proposedValue)
+	{
+		$this->m_proposedValue = $proposedValue;
+	}
+
+	public function GetValue()
+	{
+		return $this->m_proposedValue; 
+	}
+
+	abstract public function GetDescription();
+}
+
+
+class CellChangeSpec_Void extends CellChangeSpec
+{
+	public function GetDescription()
+	{
+		return $this->GetValue();
+	}
+}
+
+class CellChangeSpec_Unchanged extends CellChangeSpec
+{
+	public function GetDescription()
+	{
+		return $this->GetValue()." (unchanged)";
+	}
+}
+
+class CellChangeSpec_Modify extends CellChangeSpec
+{
+	protected $m_previousValue;
+
+	public function __construct($proposedValue, $previousValue)
+	{
+		$this->m_previousValue = $previousValue;
+		parent::__construct($proposedValue);
+	}
+
+	public function GetDescription()
+	{
+		return $this->GetValue()." (previous: ".$this->m_previousValue.")";
+	}
+}
+
+class CellChangeSpec_Issue extends CellChangeSpec_Modify
+{
+	protected $m_sReason;
+
+	public function __construct($proposedValue, $previousValue, $sReason)
+	{
+		$this->m_sReason = $sReason;
+		parent::__construct($proposedValue, $previousValue);
+	}
+
+	public function GetDescription()
+	{
+		if (is_null($this->m_proposedValue))
+		{
+			return 'Could not be changed - reason: '.$this->m_sReason;
+		}
+		return 'Could not be changed to "'.$this->GetValue().' - reason:'.$this->m_sReason.' (previous: '.$this->m_previousValue.')';
+	}
+}
+
+
+/**
+ * RowStatus
+ * A series of classes, keeping the information about a given row: could it be changed or not (and why)?  
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     $itopversion$
+ */
+abstract class RowStatus
+{
+	public function __construct()
+	{
+	}
+
+	abstract public function GetDescription();
+}
+
+class RowStatus_NoChange extends RowStatus
+{
+	public function GetDescription()
+	{
+		return "unchanged";
+	}
+}
+
+class RowStatus_NewObj extends RowStatus
+{
+	protected $m_iObjKey;
+
+	public function __construct($iObjKey = null)
+	{
+		$this->m_iObjKey = $iObjKey;
+	}
+
+	public function GetDescription()
+	{
+		if (is_null($this->m_iObjKey))
+		{
+			return "Create";
+		}
+		else
+		{
+			return 'Created ('.$this->m_iObjKey.')';
+		}	
+	}
+}
+
+class RowStatus_Modify extends RowStatus
+{
+	protected $m_iChanged;
+
+	public function __construct($iChanged)
+	{
+		$this->m_iChanged = $iChanged;
+	}
+
+	public function GetDescription()
+	{
+		return "update ".$this->m_iChanged." cols";
+	}
+}
+
+class RowStatus_Issue extends RowStatus
+{
+	protected $m_sReason;
+
+	public function __construct($proposedValue, $previousValue, $sReason)
+	{
+		$this->m_sReason = $sReason;
+	}
+
+	public function GetDescription()
+	{
+		return 'Skipped - reason:'.$this->m_sReason;
+	}
+}
+
+
+/**
+ * BulkChange
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     $itopversion$
+ */
+class BulkChange
+{
+	protected $m_sClass;
+	protected $m_aData;
+	// Note: hereafter, iCol maybe actually be any acceptable key (string)
+	// #@# todo: rename the variables to sColIndex
+	protected $m_aAttList; // attcode => iCol
+	protected $m_aReconcilKeys;// iCol => attcode
+	protected $m_aExtKeys;	// aExtKeys[sExtKeyAttCode][sExtReconcKeyAttCode] = iCol;
+
+	public function __construct($sClass, $aData, $aAttList, $aReconcilKeys, $aExtKeys)
+	{
+		$this->m_sClass = $sClass;
+		$this->m_aData = $aData;
+		$this->m_aAttList = $aAttList;
+		$this->m_aReconcilKeys = $aReconcilKeys;
+		$this->m_aExtKeys = $aExtKeys;
+	}
+
+	protected function PrepareObject(&$oTargetObj, $aRowData, &$aErrors)
+	{
+		$aResults = array();
+		$aErrors = array();
+	
+		// External keys reconciliation
+		//
+		foreach($this->m_aExtKeys as $sAttCode => $aKeyConfig)
+		{
+			$oExtKey = MetaModel::GetAttributeDef(get_class($oTargetObj), $sAttCode);
+			$oReconFilter = new CMDBSearchFilter($oExtKey->GetTargetClass());
+			foreach ($aKeyConfig as $sForeignAttCode => $iCol)
+			{
+				// The foreign attribute is one of our reconciliation key
+				$sFieldId = MakeExtFieldSelectValue($sAttCode, $sForeignAttCode);
+				$oReconFilter->AddCondition($sForeignAttCode, $aRowData[$iCol], '=');
+				$aResults["col$iCol"] = new CellChangeSpec_Void($aRowData[$iCol]);
+			}
+			$oExtObjects = new CMDBObjectSet($oReconFilter);
+			switch($oExtObjects->Count())
+			{
+			case 0:
+				$aErrors[$sAttCode] = "Object not found";
+				$aResults[$sAttCode]= new CellChangeSpec_Issue(null, $oTargetObj->Get($sAttCode), 'Object not found');
+				break;
+			case 1:
+				// Do change the external key attribute
+				$oForeignObj = $oExtObjects->Fetch();
+				$oTargetObj->Set($sAttCode, $oForeignObj->GetKey());
+	
+				// Report it
+				if (array_key_exists($sAttCode, $oTargetObj->ListChanges(false)))
+				{
+					$aResults[$sAttCode]= new CellChangeSpec_Modify($oForeignObj->GetKey(), $oTargetObj->Get($sAttCode), $oTargetObj->GetOriginal($sAttCode));
+				}
+				else
+				{
+					$aResults[$sAttCode]= new CellChangeSpec_Unchanged($oTargetObj->Get($sAttCode));
+				}
+				break;
+			default:
+				$aErrors[$sAttCode] = "Found ".$oExtObjects->Count()." matches";
+				$aResults[$sAttCode]= new CellChangeSpec_Issue(null, $oTargetObj->Get($sAttCode), "Found ".$oExtObjects->Count()." matches");
+			}
+		}	
+	
+		// Set the object attributes
+		//
+		foreach ($this->m_aAttList as $sAttCode => $iCol)
+		{
+			$oTargetObj->Set($sAttCode, $aRowData[$iCol]);
+		}
+	
+		// Reporting on fields
+		//
+		$aChangedFields = $oTargetObj->ListChanges();
+		foreach ($this->m_aAttList as $sAttCode => $iCol)
+		{
+			if (isset($aErrors[$sAttCode]))
+			{
+				$aResults["col$iCol"]= new CellChangeSpec_Issue($aRowData[$iCol], $previousValue, $sReason);
+			}
+			elseif (array_key_exists($sAttCode, $aChangedFields))
+			{
+				$aResults["col$iCol"]= new CellChangeSpec_Modify($aRowData[$iCol], $oTargetObj->Get($sAttCode), $oTargetObj->GetOriginal($sAttCode));
+			}
+			else
+			{
+				// By default... nothing happens
+				$aResults["col$iCol"]= new CellChangeSpec_Void($aRowData[$iCol]);
+			}
+		}
+	
+		// Checks
+		//
+		if (!$oTargetObj->CheckConsistency())
+		{
+			$aErrors["GLOBAL"] = "Attributes not consistent with each others";
+		}
+		return $aResults;
+	}
+	
+	
+	protected function CreateObject(&$aResult, $iRow, $aRowData, CMDBChange $oChange = null)
+	{
+		$oTargetObj = MetaModel::NewObject($this->m_sClass);
+		$aResult[$iRow] = $this->PrepareObject($oTargetObj, $aRowData, $aErrors);
+	
+		if (count($aErrors) > 0)
+		{
+			$sErrors = implode(', ', $aErrors);
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_Issue("Unexpected attribute value(s)");
+			return;
+		}
+	
+		// Check that any external key will have a value proposed
+		// Could be said once for all rows !!!
+		foreach(MetaModel::ListAttributeDefs($this->m_sClass) as $sAttCode=>$oAtt)
+		{
+			if (!$oAtt->IsExternalKey()) continue;
+		}
+	
+		// Optionaly record the results
+		//
+		if ($oChange)
+		{
+			$newID = $oTargetObj->DBInsertTracked($oChange);
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_NewObj($newID);
+		}
+		else
+		{
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_NewObj();
+		}
+	
+	}
+	
+	protected function UpdateObject(&$aResult, $iRow, $oTargetObj, $aRowData, CMDBChange $oChange = null)
+	{
+		$aResult[$iRow] = $this->PrepareObject($oTargetObj, $aRowData, $aErrors);
+	
+		// Reporting
+		//
+		if (count($aErrors) > 0)
+		{
+			$sErrors = implode(', ', $aErrors);
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_Issue("Unexpected attribute value(s)");
+			return;
+		}
+	
+		$aChangedFields = $oTargetObj->ListChanges();
+		if (count($aChangedFields) > 0)
+		{
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_Modify(count($aChangedFields));
+	
+			// Optionaly record the results
+			//
+			if ($oChange)
+			{
+				$oTargetObj->DBUpdateTracked($oChange);
+			}
+		}
+		else
+		{
+			$aResult[$iRow]["__STATUS__"] = new RowStatus_NoChange();
+		}
+	}
+	
+	public function Process(CMDBChange $oChange = null)
+	{
+		// Note: $oChange can be null, in which case the aim is to check what would be done
+	
+		// Compute the results
+		//
+		$aResult = array();
+		foreach($this->m_aData as $iRow => $aRowData)
+		{
+			$oReconciliationFilter = new CMDBSearchFilter($this->m_sClass);
+			foreach($this->m_aReconcilKeys as $sAttCode)
+			{
+				$iCol = $this->m_aAttList[$sAttCode];
+				$oReconciliationFilter->AddCondition($sAttCode, $aRowData[$iCol], '=');
+			}
+			$oReconciliationSet = new CMDBObjectSet($oReconciliationFilter);
+			switch($oReconciliationSet->Count())
+			{
+			case 0:
+				$this->CreateObject($aResult, $iRow, $aRowData, $oChange);
+				// $aResult[$iRow]["__STATUS__"]=> set in CreateObject
+				$aResult[$iRow]["__RECONCILIATION__"] = "Object not found";
+				break;
+			case 1:
+				$oTargetObj = $oReconciliationSet->Fetch();
+				$this->UpdateObject($aResult, $iRow, $oTargetObj, $aRowData, $oChange);
+				$aResult[$iRow]["__RECONCILIATION__"] = "Found a match ".$oTargetObj->GetKey();
+				// $aResult[$iRow]["__STATUS__"]=> set in UpdateObject
+				break;
+			default:
+				foreach ($this->m_aAttList as $sAttCode => $iCol)
+				{
+					$aResult[$iRow]["col$iCol"]= $aRowData[$iCol];
+				}
+				$aResult[$iRow]["__RECONCILIATION__"] = "Found ".$oReconciliationSet->Count()." matches";
+				$aResult[$iRow]["__STATUS__"]= new RowStatus_Issue("ambiguous reconciliation");
+			}
+	
+			// Whatever happened, do report the reconciliation values
+			foreach($this->m_aReconcilKeys as $sAttCode)
+			{
+				$iCol = $this->m_aAttList[$sAttCode];
+				$aResult[$iRow]["col$iCol"] = new CellChangeSpec_Void($aRowData[$iCol]);
+			}
+		}
+		return $aResult;
+	}
+}
+
+
+?>

core/cmdbobject.class.inc.php

@@ -40,6 +40,9 @@ require_once('dbobjectset.class.php');
 require_once('cmdbchange.class.inc.php');
 require_once('cmdbchangeop.class.inc.php');
 
+require_once('csvparser.class.inc.php');
+require_once('bulkchange.class.inc.php');
+
 require_once('userrights.class.inc.php');
 
 //

core/csvparser.class.inc.php

@@ -0,0 +1,192 @@
+<?php
+
+/**
+ * CSVParser
+ * CSV interpreter helper, optionaly tries to guess column mapping and the separator, check the consistency 
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @author      Denis Flaven <denisflave@free.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     1.1.1.1 $
+ */
+
+class CSVParserException extends CoreException
+{
+}
+
+
+
+
+/**
+ * CSVParser
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     $itopversion$
+ */
+class CSVParser
+{
+	private $m_sCSVData;
+	private $m_sSep;
+	private $m_iSkip;
+
+	public function __construct($sTxt)
+	{
+		$this->m_sCSVData = $sTxt;
+	}
+
+	public function SetSeparator($sSep)
+	{
+		$this->m_sSep = $sSep;
+	}
+	public function GetSeparator()
+	{
+		return $this->m_sSep;
+	}
+
+	public function SetSkipLines($iSkip)
+	{
+		$this->m_iSkip = $iSkip;
+	}
+	public function GetSkipLines()
+	{
+		return $this->m_iSkip;
+	}
+
+	public function GuessSeparator()
+	{
+		// Note: skip the first line anyway
+	
+		$aKnownSeps = array(';', ',', "\t"); // Use double quote for special chars!!!
+		$aStatsBySeparator = array();
+		foreach ($aKnownSeps as $sSep)
+		{
+			$aStatsBySeparator[$sSep] = array();
+		}
+	
+		foreach(split("\n", $this->m_sCSVData) as $sLine)
+		{
+			$sLine = trim($sLine);
+			if (substr($sLine, 0, 1) == '#') continue;
+			if (empty($sLine)) continue;
+	
+			$aLineCharsCount = count_chars($sLine, 0);
+			foreach ($aKnownSeps as $sSep)
+			{
+				$aStatsBySeparator[$sSep][] = $aLineCharsCount[ord($sSep)];
+			}
+		}
+	
+		// Default to ','
+		$this->SetSeparator(",");
+
+		foreach ($aKnownSeps as $sSep)
+		{
+			// Note: this function is NOT available :-( 
+			// stats_variance($aStatsBySeparator[$sSep]);
+			$iMin = min($aStatsBySeparator[$sSep]);
+			$iMax = max($aStatsBySeparator[$sSep]);
+			if (($iMin == $iMax) && ($iMax > 0))
+			{
+				$this->SetSeparator($sSep);
+				break;
+			}
+		}
+		return $this->GetSeparator();
+	}
+
+	public function GuessSkipLines()
+	{
+		// Take the FIRST -valuable- LINE ONLY
+		// If there is a number, then for sure this is not a header line
+		// Otherwise, we may consider that there is one line to skip
+		foreach(split("\n", $this->m_sCSVData) as $sLine)
+		{
+			$sLine = trim($sLine);
+			if (substr($sLine, 0, 1) == '#') continue;
+			if (empty($sLine)) continue;
+	
+			foreach (split($this->m_sSep, $sLine) as $value)
+			{
+				if (is_numeric($value))
+				{
+					$this->SetSkipLines(0);
+					return 0;
+				}
+			}
+			$this->SetSkipLines(1);
+			return 1;
+		}
+	}
+
+	function ToArray($aFieldMap = null, $iMax = 0)
+	{
+		// $aFieldMap is an array of col_index=>col_name
+		// $iMax is to limit the count of rows computed
+		$aRes = array();
+	
+		$iCount = 0;
+		$iSkipped = 0;
+		foreach(split("\n", $this->m_sCSVData) as $sLine)
+		{
+			$sLine = trim($sLine);
+			if (substr($sLine, 0, 1) == '#') continue;
+			if (empty($sLine)) continue;
+	
+			if ($iSkipped < $this->m_iSkip)
+			{
+				$iSkipped++;
+				continue;
+			}
+	
+			foreach (split($this->m_sSep, $sLine) as $iCol=>$sValue)
+			{
+				if (is_array($aFieldMap)) $sColRef = $aFieldMap[$iCol];
+				else                      $sColRef = $iCol;
+				$aRes[$iCount][$sColRef] = $sValue;
+			}
+	
+			$iCount++;
+			if (($iMax > 0) && ($iCount >= $iMax)) break;
+		}
+		return $aRes;
+	}
+
+	public function ListFields()
+	{
+		// Take the first valuable line
+		foreach(explode("\n", $this->m_sCSVData) as $sLine)
+		{
+			$sLine = trim($sLine);
+			if (substr($sLine, 0, 1) == '#') continue;
+			if (empty($sLine)) continue;
+			// We've got the first valuable line, that's it!
+			break;
+		}
+
+		$aRet = array();
+		foreach (explode($this->m_sSep, $sLine) as $iCol=>$value)
+		{
+			if ($this->m_iSkip == 0)
+			{
+				// No header to help us
+				$sLabel = "field $iCol";
+			}
+			else
+			{
+				$sLabel = "$value";
+			}
+			$aRet[] = $sLabel;
+		}
+		return $aRet;
+	}
+}
+
+
+?>

pages/export.php

@@ -1,73 +0,0 @@
-<?php
-require_once('../application/application.inc.php');
-require_once('../application/webpage.class.inc.php');
-require_once('../application/csvpage.class.inc.php');
-require_once('../application/xmlpage.class.inc.php');
-
-require_once('../application/startup.inc.php');
-
-require_once('../application/loginwebpage.class.inc.php');
-login_web_page::DoLogin(); // Check user rights and prompt if needed
-
-$sOperation = utils::ReadParam('operation', 'menu');
-$oContext = new UserContext();
-$oAppContext = new ApplicationContext();
-$iActiveNodeId = utils::ReadParam('menu', -1);
-$currentOrganization = utils::ReadParam('org_id', '');
-
-// Main program
-$sExpression = utils::ReadParam('expression', '');
-$sFormat = strtolower(utils::ReadParam('format', 'html'));
-$oP = null;
-
-if (!empty($sExpression))
-{
-	try
-	{
-		$oFilter = DBObjectSearch::FromOQL($sExpression);
-		if ($oFilter)
-		{
-			$oSet = new CMDBObjectSet($oFilter);
-			switch($sFormat)
-			{
-				case 'html':
-				$oP = new web_page("iTop - Export");
-				cmdbAbstractObject::DisplaySet($oP, $oSet);
-				break;
-				
-				case 'csv':
-				$oP = new CSVPage("iTop - Export");
-				cmdbAbstractObject::DisplaySetAsCSV($oP, $oSet);
-				break;
-				
-				case 'xml':
-				$oP = new XMLPage("iTop - Export");
-				cmdbAbstractObject::DisplaySetAsXML($oP, $oSet);
-				break;
-				
-				default:
-				$oP = new web_page("iTop - Export");
-				$oP->add("Unsupported format '$sFormat'. Possible values are: html, csv or xml.");
-			}
-		}
-	}
-	catch(Exception $e)
-	{
-		$oP = new web_page("iTop - Export");
-		$oP->p("Error the query can not be executed.");		
-		$oP->p($e->GetHtmlDesc());		
-	}
-}
-if (!$oP)
-{
-	// Display a short message about how to use this page
-	$oP = new web_page("iTop - Export");
-	$oP->p("<strong>General purpose export page.</strong>");
-	$oP->p("<strong>Parameters:</strong>");
-	$oP->p("<ul><li>expression: an OQL expression (URL encoded if needed)</li>
-			    <li>format: (optional, default is html) the desired output format. Can be one of 'html', 'csv' or 'xml'</li>
-		    </ul>");
-}
-
-$oP->output();
-?>

webservices/import.php

@@ -1,5 +1,17 @@
 <?php
 
+/**
+ * Import web service 
+ *
+ * @package     iTopORM
+ * @author      Romain Quetiez <romainquetiez@yahoo.fr>
+ * @author      Denis Flaven <denisflave@free.fr>
+ * @license     http://www.opensource.org/licenses/lgpl-license.php LGPL
+ * @link        www.itop.com
+ * @since       1.0
+ * @version     1.1.1.1 $
+ */
+
 //
 // Test URLs
 //
@@ -20,7 +32,7 @@
 //
 // Known issues
 // - ALMOST impossible to troubleshoot when an externl key has a wrong value
-// - no character escaping in the xml output
+// - no character escaping in the xml output (yes !?!?!)
 // - not outputing xml when a wrong input is given (class, attribute names)
 // - for a bizIncidentTicket you may use the name as the reconciliation key,
 //   but that attribute is in fact recomputed by the application! An error should be raised somewhere
@@ -49,8 +61,7 @@ try
 {
 	$sClass = utils::ReadParam('class', '');
 	$sSep = utils::ReadParam('separator', ';');
-	//$sCSVData = utils::ReadPostedParam('csvdata');
-	$sCSVData = utils::ReadParam('csvdata');
+	$sCSVData = utils::ReadPostedParam('csvdata');
 
 	$oCSVParser = new CSVParser($sCSVData); 
 	$oCSVParser->SetSeparator($sSep);
@@ -82,7 +93,15 @@ try
 
 	$oMyChange = MetaModel::NewObject("CMDBChange");
 	$oMyChange->Set("date", time());
-	$oMyChange->Set("userinfo", "CSV Import Web Service");
+	if (UserRights::GetUser() != UserRights::GetRealUser())
+	{
+		$sUserString = UserRights::GetRealUser()." on behalf of ".UserRights::GetUser();
+	}
+	else
+	{
+		$sUserString = UserRights::GetUser();
+	}
+	$oMyChange->Set("userinfo", $sUserString.' (bulk load by web service)');
 	$iChangeId = $oMyChange->DBInsert();
 
 	$aRes = $oBulk->Process($oMyChange);