diff --git a/core/dbobject.class.php b/core/dbobject.class.php index 10c7037d1..19327ebfc 100644 --- a/core/dbobject.class.php +++ b/core/dbobject.class.php @@ -669,6 +669,8 @@ abstract class DBObject // a displayable error is returned public function DoCheckToWrite() { + $this->DoComputeValues(); + $this->m_aCheckIssues = array(); foreach(MetaModel::ListAttributeDefs(get_class($this)) as $sAttCode=>$oAttDef) diff --git a/setup/xmldataloader.class.inc.php b/setup/xmldataloader.class.inc.php index de28bfb4f..8da02b661 100644 --- a/setup/xmldataloader.class.inc.php +++ b/setup/xmldataloader.class.inc.php @@ -41,7 +41,11 @@ class XMLDataLoader protected $m_bSessionActive; protected $m_oChange; protected $m_sCacheFileName; - + + protected $m_aErrors; + protected $m_aWarnings; + protected $m_iCountCreated; + public function __construct($sConfigFileName) { $this->m_aKeys = array(); @@ -51,7 +55,9 @@ class XMLDataLoader $this->InitDataModel($sConfigFileName); $this->LoadKeysCache(); $this->m_bSessionActive = true; - + $this->m_aErrors = array(); + $this->m_aWarnings = array(); + $this->m_iCountCreated = 0; } public function StartSession($oChange) @@ -63,10 +69,38 @@ class XMLDataLoader $this->m_bSessionActive = true; } - public function EndSession() + public function EndSession($bStrict = false) { $this->ResolveExternalKeys(); $this->m_bSessionActive = false; + + if (count($this->m_aErrors) > 0) + { + return false; + } + elseif ($bStrict && count($this->m_aWarnings) > 0) + { + return false; + } + else + { + return true; + } + } + + public function GetErrors() + { + return $this->m_aErrors; + } + + public function GetWarnings() + { + return $this->m_aWarnings; + } + + public function GetCountCreated() + { + return $this->m_iCountCreated; } public function __destruct() @@ -116,7 +150,10 @@ class XMLDataLoader { $sData = serialize( array('keys' => $this->m_aKeys, 'objects' => $this->m_aObjectsCache, - 'change' => $this->m_oChange)); + 'change' => $this->m_oChange, + 'errors' => $this->m_aErrors, + 'warnings' => $this->m_aWarnings, + )); fwrite($hFile, $sData); fclose($hFile); } @@ -137,7 +174,9 @@ class XMLDataLoader $aCache = unserialize($sFileContent); $this->m_aKeys = $aCache['keys']; $this->m_aObjectsCache = $aCache['objects']; - $this->m_oChange = $aCache['change']; + $this->m_oChange = $aCache['change']; + $this->m_aErrors = $aCache['errors']; + $this->m_aWarnings = $aCache['warnings']; } } @@ -170,6 +209,12 @@ class XMLDataLoader $aReplicas = array(); foreach($oXml as $sClass => $oXmlObj) { + if (!MetaModel::IsValidClass($sClass)) + { + SetupWebPage::log_error("Unknown class - $sClass"); + throw(new Exception("Unknown class - $sClass")); + } + $iSrcId = (integer)$oXmlObj['id']; // Mandatory to cast // Import algorithm @@ -177,40 +222,84 @@ class XMLDataLoader // for all attribute that is neither an external field // not an external key, assign it // Store all external keys for further reference - // Create the object an store the correspondence between its newly created Id + // Create the object an store the correspondance between its newly created Id // and its original Id // Once all the objects have been created re-assign all the external keys to // their actual Ids $oTargetObj = MetaModel::NewObject($sClass); - foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode=>$oAttDef) + foreach($oXmlObj as $sAttCode => $oSubNode) { + if (!MetaModel::IsValidAttCode($sClass, $sAttCode)) + { + $sMsg = "Unknown attribute code - $sClass/$sAttCode"; + SetupWebPage::log_error($sMsg); + throw(new Exception($sMsg)); + } + + $oAttDef = MetaModel::GetAttributeDef($sClass, $sAttCode); if (($oAttDef->IsWritable()) && ($oAttDef->IsScalar())) { if ($oAttDef->IsExternalKey()) { - $iDstObj = (integer)($oXmlObj->$sAttCode); - // Attempt to find the object in the list of loaded objects - $iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj); - if ($iExtKey == 0) + if (substr(trim($oSubNode), 0, 6) == 'SELECT') { - $iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative ! - $oTargetObj->RegisterAsDirty(); + $sQuery = trim($oSubNode); + $oSet = new DBObjectSet(DBObjectSearch::FromOQL($sQuery)); + $iMatches = $oSet->Count(); + if ($iMatches == 1) + { + $oFoundObject = $oSet->Fetch(); + $iExtKey = $oFoundObject->GetKey(); + } + else + { + $sMsg = "Ext key not reconcilied - $sClass/$iSrcId - $sAttCode: '".$sQuery."' - found $iMatches matche(s)"; + SetupWebPage::log_error($sMsg); + $this->m_aErrors[] = $sMsg; + $iExtKey = 0; + } + } + else + { + $iDstObj = (integer)($oSubNode); + // Attempt to find the object in the list of loaded objects + $iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj); + if ($iExtKey == 0) + { + $iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative ! + $oTargetObj->RegisterAsDirty(); + } + // here we allow external keys to be invalid because we will resolve them later on... } - // here we allow external keys to be invalid because we will resolve them later on... //$oTargetObj->CheckValue($sAttCode, $iExtKey); $oTargetObj->Set($sAttCode, $iExtKey); } + elseif ($oAttDef instanceof AttributeBlob) + { + $sMimeType = (string) $oSubNode->mimetype; + $sFileName = (string) $oSubNode->filename; + $data = base64_decode((string) $oSubNode->data); + $oDoc = new ormDocument($data, $sMimeType, $sFileName); + $oTargetObj->Set($sAttCode, $oDoc); + } else { - // tested by Romain, little impact on perf (not significant on the intial setup) - $res = $oTargetObj->CheckValue($sAttCode, (string)$oXmlObj->$sAttCode); + $value = (string)$oSubNode; + + if ($value == '') + { + $value = $oAttDef->GetNullValue(); + } + + $res = $oTargetObj->CheckValue($sAttCode, $value); if ($res !== true) { // $res contains the error description - SetupWebPage::log_error("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res"); - throw(new Exception("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res")); + $sMsg = "Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oSubNode."' ; $res"; + SetupWebPage::log_error($sMsg); + $this->m_aErrors[] = $sMsg; } - $oTargetObj->Set($sAttCode, (string)$oXmlObj->$sAttCode); + $oTargetObj->Set($sAttCode, $value); } } } @@ -283,12 +372,13 @@ class XMLDataLoader { $iObjId = $oTargetObj->DBInsertNoReload(); } + $this->m_iCountCreated++; } } catch(Exception $e) { - SetupWebPage::log_error("An object could not be loaded - $sClass/$iSrcId - ".$e->getMessage()); - echo $e->GetHtmlDesc(); + SetupWebPage::log_error("An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage()); + $this->m_aErrors[] = "An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage(); } $aParentClasses = MetaModel::EnumParentClasses($sClass); $aParentClasses[] = $sClass; @@ -323,6 +413,7 @@ class XMLDataLoader { $sMsg = "unresolved extkey in $sClass::".$oTargetObj->GetKey()."(".$oTargetObj->GetName().")::$sAttCode=$sTargetClass::$iTempKey"; SetupWebPage::log_warning($sMsg); + $this->m_aWarnings[] = $sMsg; //echo "
aKeys[".$sTargetClass."]:\n";
 							//print_r($this->m_aKeys[$sTargetClass]);
 							//echo "
\n"; @@ -349,7 +440,7 @@ class XMLDataLoader } catch(Exception $e) { - echo $e->GetHtmlDesc(); + $this->m_aErrors[] = "The object changes could not be tracked - $sClass/$iSrcId - ".$e->getMessage(); } } } diff --git a/webservices/backoffice.dataloader.php b/webservices/backoffice.dataloader.php new file mode 100644 index 000000000..f4056d5bd --- /dev/null +++ b/webservices/backoffice.dataloader.php @@ -0,0 +1,172 @@ + + * @author Romain Quetiez + * @author Denis Flaven + * @license http://www.opensource.org/licenses/gpl-3.0.html LGPL + */ + +/** + * This page is called to load an XML file into the database + * parameters + * 'file' string Name of the file to load + */ +define('SAFE_MINIMUM_MEMORY', 256*1024*1024); +require_once('../application/utils.inc.php'); +require_once("../application/nicewebpage.class.inc.php"); +// required because the class xmldataloader is reporting errors in the setup.log file +require_once('../setup/setuppage.class.inc.php'); + + +function SetMemoryLimit($oP) +{ + $sMemoryLimit = trim(ini_get('memory_limit')); + if (empty($sMemoryLimit)) + { + // On some PHP installations, memory_limit does not exist as a PHP setting! + // (encountered on a 5.2.0 under Windows) + // In that case, ini_set will not work, let's keep track of this and proceed with the data load + $oP->p("No memory limit has been defined in this instance of PHP"); + } + else + { + // Check that the limit will allow us to load the data + // + $iMemoryLimit = utils::ConvertToBytes($sMemoryLimit); + if ($iMemoryLimit < SAFE_MINIMUM_MEMORY) + { + if (ini_set('memory_limit', SAFE_MINIMUM_MEMORY) === FALSE) + { + $oP->p("memory_limit is too small: $iMemoryLimit and can not be increased by the script itself."); + } + else + { + $oP->p("memory_limit increased from $iMemoryLimit to ".SAFE_MINIMUM_MEMORY."."); + } + } + } +} + + +//////////////////////////////////////////////////////////////////////////////// +// +// Main +// +//////////////////////////////////////////////////////////////////////////////// + +require_once('../core/config.class.inc.php'); +require_once('../core/log.class.inc.php'); +require_once('../core/kpi.class.inc.php'); +require_once('../core/cmdbsource.class.inc.php'); +require_once('../setup/xmldataloader.class.inc.php'); + +define('FINAL_CONFIG_FILE', '../config-itop.php'); + +// Never cache this page +header("Cache-Control: no-cache, must-revalidate"); // HTTP/1.1 +header("Expires: Fri, 17 Jul 1970 05:00:00 GMT"); // Date in the past + +Utils::SpecifyConfigFile(FINAL_CONFIG_FILE); + +/** + * Main program + */ +$sFileName = Utils::ReadParam('file', ''); + +$oP = new WebPage("iTop - Backoffice data loader"); + + +try +{ + // Note: the data model must be loaded first + $oDataLoader = new XMLDataLoader(FINAL_CONFIG_FILE); // When called by the wizard, the final config is not yet there + + if (empty($sFileName)) + { + throw(new Exception("Missing argument 'file'")); + } + if (!file_exists($sFileName)) + { + throw(new Exception("File $sFileName does not exist")); + } + + SetMemoryLimit($oP); + + + // The XMLDataLoader constructor has initialized the DB, let's start a transaction + CMDBSource::Query('SET AUTOCOMMIT=0'); + CMDBSource::Query('BEGIN WORK'); + + $oChange = MetaModel::NewObject("CMDBChange"); + $oChange->Set("date", time()); + $oChange->Set("userinfo", "Initialization"); + $iChangeId = $oChange->DBInsert(); + $oP->p("Starting data load."); + $oDataLoader->StartSession($oChange); + + $oDataLoader->LoadFile($sFileName); + + $oP->p("Ending data load session"); + if ($oDataLoader->EndSession(true /* strict */)) + { + $iCountCreated = $oDataLoader->GetCountCreated(); + CMDBSource::Query('COMMIT'); + + $oP->p("Data successfully written into the DB: $iCountCreated objects created"); + } + else + { + CMDBSource::Query('ROLLBACK'); + $oP->p("Some issues have been encountered, changes will not be recorded, please review the source data"); + $aErrors = $oDataLoader->GetErrors(); + if (count($aErrors) > 0) + { + $oP->p('Errors ('.count($aErrors).')'); + foreach ($aErrors as $sMsg) + { + $oP->p(' * '.$sMsg); + } + } + $aWarnings = $oDataLoader->GetWarnings(); + if (count($aWarnings) > 0) + { + $oP->p('Warnings ('.count($aWarnings).')'); + foreach ($aWarnings as $sMsg) + { + $oP->p(' * '.$sMsg); + } + } + } + +} +catch(Exception $e) +{ + $oP->p("An error happened while loading the data: ".$e->getMessage()); + $oP->p("Aborting (no data written)..."); + CMDBSource::Query('ROLLBACK'); +} + +if (function_exists('memory_get_peak_usage')) +{ + $oP->p("Information: memory peak usage: ".memory_get_peak_usage()); +} + +$oP->Output(); +?>