composer/iTop
6
0
Fork 0
mirror of https://github.com/Combodo/iTop.git synced 2026-02-12 23:14:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

#111 Improved the data loader, and added a REST service to load data from a file

Browse Source 
SVN:trunk[892]
This commit is contained in:
Romain Quetiez
2010-10-13 16:36:51 +00:00
parent 0fff433e90
commit 1e1e10aa00
3 changed files with 287 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/dbobject.class.php
View File

@@ -669,6 +669,8 @@ abstract class DBObject
// a displayable error is returned
public function DoCheckToWrite()
{
$this->DoComputeValues();
$this->m_aCheckIssues = array();
foreach(MetaModel::ListAttributeDefs(get_class($this)) as $sAttCode=>$oAttDef)

135
setup/xmldataloader.class.inc.php
View File

@@ -41,7 +41,11 @@ class XMLDataLoader
protected $m_bSessionActive;
protected $m_oChange;
protected $m_sCacheFileName;
protected $m_aErrors;
protected $m_aWarnings;
protected $m_iCountCreated;
public function __construct($sConfigFileName)
{
$this->m_aKeys = array();
@@ -51,7 +55,9 @@ class XMLDataLoader
$this->InitDataModel($sConfigFileName);
$this->LoadKeysCache();
$this->m_bSessionActive = true;
$this->m_aErrors = array();
$this->m_aWarnings = array();
$this->m_iCountCreated = 0;
}
public function StartSession($oChange)
@@ -63,10 +69,38 @@ class XMLDataLoader
$this->m_bSessionActive = true;
}
public function EndSession()
public function EndSession($bStrict = false)
{
$this->ResolveExternalKeys();
$this->m_bSessionActive = false;
if (count($this->m_aErrors) > 0)
{
return false;
}
elseif ($bStrict && count($this->m_aWarnings) > 0)
{
return false;
}
else
{
return true;
}
}
public function GetErrors()
{
return $this->m_aErrors;
}
public function GetWarnings()
{
return $this->m_aWarnings;
}
public function GetCountCreated()
{
return $this->m_iCountCreated;
}
public function __destruct()
@@ -116,7 +150,10 @@ class XMLDataLoader
{
$sData = serialize( array('keys' => $this->m_aKeys,
'objects' => $this->m_aObjectsCache,
'change' => $this->m_oChange));
'change' => $this->m_oChange,
'errors' => $this->m_aErrors,
'warnings' => $this->m_aWarnings,
));
fwrite($hFile, $sData);
fclose($hFile);
}
@@ -137,7 +174,9 @@ class XMLDataLoader
$aCache = unserialize($sFileContent);
$this->m_aKeys = $aCache['keys'];
$this->m_aObjectsCache = $aCache['objects'];
$this->m_oChange = $aCache['change'];
$this->m_oChange = $aCache['change'];
$this->m_aErrors = $aCache['errors'];
$this->m_aWarnings = $aCache['warnings'];
}
}
@@ -170,6 +209,12 @@ class XMLDataLoader
$aReplicas = array();
foreach($oXml as $sClass => $oXmlObj)
{
if (!MetaModel::IsValidClass($sClass))
{
SetupWebPage::log_error("Unknown class - $sClass");
throw(new Exception("Unknown class - $sClass"));
}
$iSrcId = (integer)$oXmlObj['id']; // Mandatory to cast
// Import algorithm
@@ -177,40 +222,84 @@ class XMLDataLoader
// for all attribute that is neither an external field
// not an external key, assign it
// Store all external keys for further reference
// Create the object an store the correspondence between its newly created Id
// Create the object an store the correspondance between its newly created Id
// and its original Id
// Once all the objects have been created re-assign all the external keys to
// their actual Ids
$oTargetObj = MetaModel::NewObject($sClass);
foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode=>$oAttDef)
foreach($oXmlObj as $sAttCode => $oSubNode)
{
if (!MetaModel::IsValidAttCode($sClass, $sAttCode))
{
$sMsg = "Unknown attribute code - $sClass/$sAttCode";
SetupWebPage::log_error($sMsg);
throw(new Exception($sMsg));
}
$oAttDef = MetaModel::GetAttributeDef($sClass, $sAttCode);
if (($oAttDef->IsWritable()) && ($oAttDef->IsScalar()))
{
if ($oAttDef->IsExternalKey())
{
$iDstObj = (integer)($oXmlObj->$sAttCode);
// Attempt to find the object in the list of loaded objects
$iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj);
if ($iExtKey == 0)
if (substr(trim($oSubNode), 0, 6) == 'SELECT')
{
$iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative !
$oTargetObj->RegisterAsDirty();
$sQuery = trim($oSubNode);
$oSet = new DBObjectSet(DBObjectSearch::FromOQL($sQuery));
$iMatches = $oSet->Count();
if ($iMatches == 1)
{
$oFoundObject = $oSet->Fetch();
$iExtKey = $oFoundObject->GetKey();
}
else
{
$sMsg = "Ext key not reconcilied - $sClass/$iSrcId - $sAttCode: '".$sQuery."' - found $iMatches matche(s)";
SetupWebPage::log_error($sMsg);
$this->m_aErrors[] = $sMsg;
$iExtKey = 0;
}
}
else
{
$iDstObj = (integer)($oSubNode);
// Attempt to find the object in the list of loaded objects
$iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj);
if ($iExtKey == 0)
{
$iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative !
$oTargetObj->RegisterAsDirty();
}
// here we allow external keys to be invalid because we will resolve them later on...
}
// here we allow external keys to be invalid because we will resolve them later on...
//$oTargetObj->CheckValue($sAttCode, $iExtKey);
$oTargetObj->Set($sAttCode, $iExtKey);
}
elseif ($oAttDef instanceof AttributeBlob)
{
$sMimeType = (string) $oSubNode->mimetype;
$sFileName = (string) $oSubNode->filename;
$data = base64_decode((string) $oSubNode->data);
$oDoc = new ormDocument($data, $sMimeType, $sFileName);
$oTargetObj->Set($sAttCode, $oDoc);
}
else
{
// tested by Romain, little impact on perf (not significant on the intial setup)
$res = $oTargetObj->CheckValue($sAttCode, (string)$oXmlObj->$sAttCode);
$value = (string)$oSubNode;
if ($value == '')
{
$value = $oAttDef->GetNullValue();
}
$res = $oTargetObj->CheckValue($sAttCode, $value);
if ($res !== true)
{
// $res contains the error description
SetupWebPage::log_error("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res");
throw(new Exception("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res"));
$sMsg = "Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oSubNode."' ; $res";
SetupWebPage::log_error($sMsg);
$this->m_aErrors[] = $sMsg;
}
$oTargetObj->Set($sAttCode, (string)$oXmlObj->$sAttCode);
$oTargetObj->Set($sAttCode, $value);
}
}
}
@@ -283,12 +372,13 @@ class XMLDataLoader
{
$iObjId = $oTargetObj->DBInsertNoReload();
}
$this->m_iCountCreated++;
}
}
catch(Exception $e)
{
SetupWebPage::log_error("An object could not be loaded - $sClass/$iSrcId - ".$e->getMessage());
echo $e->GetHtmlDesc();
SetupWebPage::log_error("An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage());
$this->m_aErrors[] = "An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage();
}
$aParentClasses = MetaModel::EnumParentClasses($sClass);
$aParentClasses[] = $sClass;
@@ -323,6 +413,7 @@ class XMLDataLoader
{
$sMsg = "unresolved extkey in $sClass::".$oTargetObj->GetKey()."(".$oTargetObj->GetName().")::$sAttCode=$sTargetClass::$iTempKey";
SetupWebPage::log_warning($sMsg);
$this->m_aWarnings[] = $sMsg;
//echo "<pre>aKeys[".$sTargetClass."]:\n";
//print_r($this->m_aKeys[$sTargetClass]);
//echo "</pre>\n";
@@ -349,7 +440,7 @@ class XMLDataLoader
}
catch(Exception $e)
{
echo $e->GetHtmlDesc();
$this->m_aErrors[] = "The object changes could not be tracked - $sClass/$iSrcId - ".$e->getMessage();
}
}
}

172
webservices/backoffice.dataloader.php Normal file
View File

@@ -0,0 +1,172 @@
<?php
// Copyright (C) 2010 Combodo SARL
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; version 3 of the License.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
/**
* Does load data from XML files (currently used in the setup and the backoffice data loader utility)
*
* @author Erwan Taloc <erwan.taloc@combodo.com>
* @author Romain Quetiez <romain.quetiez@combodo.com>
* @author Denis Flaven <denis.flaven@combodo.com>
* @license http://www.opensource.org/licenses/gpl-3.0.html LGPL
*/
/**
* This page is called to load an XML file into the database
* parameters
* 'file' string Name of the file to load
*/
define('SAFE_MINIMUM_MEMORY', 256*1024*1024);
require_once('../application/utils.inc.php');
require_once("../application/nicewebpage.class.inc.php");
// required because the class xmldataloader is reporting errors in the setup.log file
require_once('../setup/setuppage.class.inc.php');
function SetMemoryLimit($oP)
{
$sMemoryLimit = trim(ini_get('memory_limit'));
if (empty($sMemoryLimit))
{
// On some PHP installations, memory_limit does not exist as a PHP setting!
// (encountered on a 5.2.0 under Windows)
// In that case, ini_set will not work, let's keep track of this and proceed with the data load
$oP->p("No memory limit has been defined in this instance of PHP");
}
else
{
// Check that the limit will allow us to load the data
//
$iMemoryLimit = utils::ConvertToBytes($sMemoryLimit);
if ($iMemoryLimit < SAFE_MINIMUM_MEMORY)
{
if (ini_set('memory_limit', SAFE_MINIMUM_MEMORY) === FALSE)
{
$oP->p("memory_limit is too small: $iMemoryLimit and can not be increased by the script itself.");
}
else
{
$oP->p("memory_limit increased from $iMemoryLimit to ".SAFE_MINIMUM_MEMORY.".");
}
}
}
}
////////////////////////////////////////////////////////////////////////////////
//
// Main
//
////////////////////////////////////////////////////////////////////////////////
require_once('../core/config.class.inc.php');
require_once('../core/log.class.inc.php');
require_once('../core/kpi.class.inc.php');
require_once('../core/cmdbsource.class.inc.php');
require_once('../setup/xmldataloader.class.inc.php');
define('FINAL_CONFIG_FILE', '../config-itop.php');
// Never cache this page
header("Cache-Control: no-cache, must-revalidate"); // HTTP/1.1
header("Expires: Fri, 17 Jul 1970 05:00:00 GMT"); // Date in the past
Utils::SpecifyConfigFile(FINAL_CONFIG_FILE);
/**
* Main program
*/
$sFileName = Utils::ReadParam('file', '');
$oP = new WebPage("iTop - Backoffice data loader");
try
{
// Note: the data model must be loaded first
$oDataLoader = new XMLDataLoader(FINAL_CONFIG_FILE); // When called by the wizard, the final config is not yet there
if (empty($sFileName))
{
throw(new Exception("Missing argument 'file'"));
}
if (!file_exists($sFileName))
{
throw(new Exception("File $sFileName does not exist"));
}
SetMemoryLimit($oP);
// The XMLDataLoader constructor has initialized the DB, let's start a transaction
CMDBSource::Query('SET AUTOCOMMIT=0');
CMDBSource::Query('BEGIN WORK');
$oChange = MetaModel::NewObject("CMDBChange");
$oChange->Set("date", time());
$oChange->Set("userinfo", "Initialization");
$iChangeId = $oChange->DBInsert();
$oP->p("Starting data load.");
$oDataLoader->StartSession($oChange);
$oDataLoader->LoadFile($sFileName);
$oP->p("Ending data load session");
if ($oDataLoader->EndSession(true /* strict */))
{
$iCountCreated = $oDataLoader->GetCountCreated();
CMDBSource::Query('COMMIT');
$oP->p("Data successfully written into the DB: $iCountCreated objects created");
}
else
{
CMDBSource::Query('ROLLBACK');
$oP->p("Some issues have been encountered, changes will not be recorded, please review the source data");
$aErrors = $oDataLoader->GetErrors();
if (count($aErrors) > 0)
{
$oP->p('Errors ('.count($aErrors).')');
foreach ($aErrors as $sMsg)
{
$oP->p(' * '.$sMsg);
}
}
$aWarnings = $oDataLoader->GetWarnings();
if (count($aWarnings) > 0)
{
$oP->p('Warnings ('.count($aWarnings).')');
foreach ($aWarnings as $sMsg)
{
$oP->p(' * '.$sMsg);
}
}
}
}
catch(Exception $e)
{
$oP->p("An error happened while loading the data: ".$e->getMessage());
$oP->p("Aborting (no data written)...");
CMDBSource::Query('ROLLBACK');
}
if (function_exists('memory_get_peak_usage'))
{
$oP->p("Information: memory peak usage: ".memory_get_peak_usage());
}
$oP->Output();
?>