diff --git a/application/csvpage.class.inc.php b/application/csvpage.class.inc.php index f4746d56f..9ef3655ae 100644 --- a/application/csvpage.class.inc.php +++ b/application/csvpage.class.inc.php @@ -45,12 +45,51 @@ class CSVPage extends WebPage echo trim($this->s_content); } - public function small_p($sText) - { + public function small_p($sText) + { } - + + public function add($sText) + { + $this->s_content .= $sText; + } + + public function p($sText) + { + $this->s_content .= $sText."\n"; + } + + public function add_comment($sText) + { + $this->s_content .= "#".$sText."\n"; + } + public function table($aConfig, $aData, $aParams = array()) { + $aCells = array(); + foreach($aConfig as $sName=>$aDef) + { + if (strlen($aDef['description']) > 0) + { + $aCells[] = $aDef['label'].' ('.$aDef['description'].')'; + } + else + { + $aCells[] = $aDef['label']; + } + } + $this->s_content .= implode(';', $aCells)."\n"; + + foreach($aData as $aRow) + { + $aCells = array(); + foreach($aConfig as $sName=>$aAttribs) + { + $sValue = $aRow["$sName"]; + $aCells[] = $sValue; + } + $this->s_content .= implode(';', $aCells)."\n"; + } } } diff --git a/application/utils.inc.php b/application/utils.inc.php index 26922f4a4..db655a948 100644 --- a/application/utils.inc.php +++ b/application/utils.inc.php @@ -42,9 +42,40 @@ class utils private static $m_sConfigFile = ITOP_CONFIG_FILE; private static $m_oConfig = null; - public static function ReadParam($sName, $defaultValue = "") + + public static function IsModeCLI() { - return isset($_REQUEST[$sName]) ? $_REQUEST[$sName] : $defaultValue; + global $argv; + if (isset($argv)) + { + return true; + } + else + { + return false; + } + } + + + public static function ReadParam($sName, $defaultValue = "", $bAllowCLI = false) + { + global $argv; + $retValue = $defaultValue; + if (isset($_REQUEST[$sName])) + { + $retValue = $_REQUEST[$sName]; + } + elseif ($bAllowCLI && isset($argv)) + { + foreach($argv as $iArg => $sArg) + { + if (preg_match('/^--'.$sName.'=(.*)$/', $sArg, $aMatches)) + { + $retValue = $aMatches[1]; + } + } + } + return $retValue; } public static function ReadPostedParam($sName, $defaultValue = "") diff --git a/core/bulkchange.class.inc.php b/core/bulkchange.class.inc.php index b7519bb67..df074ed11 100644 --- a/core/bulkchange.class.inc.php +++ b/core/bulkchange.class.inc.php @@ -71,7 +71,7 @@ abstract class CellChangeSpec public function GetOql() { - return $this->m_proposedValue; + return $this->m_sOql; } abstract public function GetDescription(); @@ -289,7 +289,7 @@ class BulkChange break; default: $aErrors[$sAttCode] = "Found ".$oExtObjects->Count()." matches"; - $aResults[$sAttCode]= new CellStatus_Ambiguous($oTargetObj->Get($sAttCode), $oExtObjects->Count(), $oExtObjects->ToOql()); + $aResults[$sAttCode]= new CellStatus_Ambiguous($oTargetObj->Get($sAttCode), $oExtObjects->Count(), $oReconFilter->ToOql()); } // Report diff --git a/core/test.class.inc.php b/core/test.class.inc.php index b8baa4ebf..c7c97686b 100644 --- a/core/test.class.inc.php +++ b/core/test.class.inc.php @@ -95,6 +95,13 @@ abstract class TestHandler abstract protected function DoExecute(); protected function DoCleanup() {return true;} + protected static function DumpVariable($var) + { + echo "
\n"; 
+		print_r($var);
+		echo "
\n"; + } + protected function ReportSuccess($sMessage, $sSubtestId = '') { $this->m_aSuccesses[] = $sMessage; diff --git a/pages/test.php b/pages/test.php index cf6f31042..076045b16 100644 --- a/pages/test.php +++ b/pages/test.php @@ -23,7 +23,14 @@ * @license http://www.opensource.org/licenses/gpl-3.0.html LGPL */ - +?> + +Parse(); // Not expecting a given format, otherwise use ParseExpression/ParseObjectQuery/ParseValueSetQuery - MyHelpers::var_dump_html($oTrash, true); + self::DumpVariable($oTrash); } catch (OQLException $OqlException) { @@ -236,9 +236,7 @@ a2?;?b?;?c? ?a?;?b?;?ouf !? Espace sur la fin ; 1234; e@taloc.com '; - echo "
\n";
-		print_r($sDataFile);
-		echo "
\n"; + self::DumpVariable($sDataFile); $aExpectedResult = array( //array('field1', 'field2', 'field3'), @@ -375,13 +373,13 @@ class TestMyBizModel extends TestBizModel function test_linksinfo() { echo "

Enum links

"; - MyHelpers::var_dump_html(MetaModel::EnumReferencedClasses("cmdbTeam")); - MyHelpers::var_dump_html(MetaModel::EnumReferencingClasses("Organization")); + self::DumpVariable(MetaModel::EnumReferencedClasses("cmdbTeam")); + self::DumpVariable(MetaModel::EnumReferencingClasses("Organization")); - MyHelpers::var_dump_html(MetaModel::EnumLinkingClasses()); - MyHelpers::var_dump_html(MetaModel::EnumLinkingClasses("cmdbContact")); - MyHelpers::var_dump_html(MetaModel::EnumLinkingClasses("cmdWorkshop")); - MyHelpers::var_dump_html(MetaModel::GetLinkLabel("Liens_entre_contacts_et_workshop", "toworkshop")); + self::DumpVariable(MetaModel::EnumLinkingClasses()); + self::DumpVariable(MetaModel::EnumLinkingClasses("cmdbContact")); + self::DumpVariable(MetaModel::EnumLinkingClasses("cmdWorkshop")); + self::DumpVariable(MetaModel::GetLinkLabel("Liens_entre_contacts_et_workshop", "toworkshop")); } function test_list_attributes() @@ -423,7 +421,7 @@ class TestMyBizModel extends TestBizModel echo "

Reload

"; $team = MetaModel::GetObject("cmdbContact", "2"); echo "Chargement de l'attribut headcount: {$team->Get("headcount")}
\n"; - MyHelpers::var_dump_html($team); + self::DumpVariable($team); } function test_setattribute() @@ -432,7 +430,7 @@ class TestMyBizModel extends TestBizModel $team = MetaModel::GetObject("cmdbTeam", "2"); $team->Set("headcount", rand(1,1000)); $team->Set("email", "Luis ".rand(9,250)); - MyHelpers::var_dump_html($team->ListChanges()); + self::DumpVariable($team->ListChanges()); echo "New headcount = {$team->Get("headcount")}
\n"; echo "Computed name = {$team->Get("name")}
\n"; @@ -447,7 +445,7 @@ class TestMyBizModel extends TestBizModel echo "

Check the modified team

"; $oTeam = MetaModel::GetObject("cmdbTeam", "2"); - MyHelpers::var_dump_html($oTeam); + self::DumpVariable($oTeam); } function test_newobject() { @@ -469,7 +467,7 @@ class TestMyBizModel extends TestBizModel $oTeam = MetaModel::GetObject("cmdbTeam", $iId); $oTeam->DBDeleteTracked($oMyChange); echo "Deleted team: $iId
"; - MyHelpers::var_dump_html($oTeam); + self::DumpVariable($oTeam); } @@ -507,7 +505,7 @@ class TestMyBizModel extends TestBizModel $oMyChange->Set("userinfo", "Made by robot #".rand(1,100)); $iChangeId = $oMyChange->DBInsert(); echo "Created new change: $iChangeId
"; - MyHelpers::var_dump_html($oMyChange); + self::DumpVariable($oMyChange); echo "

Create a new object (team)

"; $oNewTeam = MetaModel::NewObject("cmdbTeam"); @@ -522,7 +520,7 @@ class TestMyBizModel extends TestBizModel $oTeam = MetaModel::GetObject("cmdbTeam", $iId); $oTeam->DBDeleteTracked($oMyChange); echo "Deleted team: $iId
"; - MyHelpers::var_dump_html($oTeam); + self::DumpVariable($oTeam); } function test_zlist() @@ -579,8 +577,8 @@ class TestMyBizModel extends TestBizModel { echo "

Test relations

"; - //MyHelpers::var_dump_html(MetaModel::EnumRelationQueries("cmdbObjectHomeMade", "Potes")); - MyHelpers::var_dump_html(MetaModel::EnumRelationQueries("cmdbContact", "Potes")); + //self::DumpVariable(MetaModel::EnumRelationQueries("cmdbObjectHomeMade", "Potes")); + self::DumpVariable(MetaModel::EnumRelationQueries("cmdbContact", "Potes")); $iMaxDepth = 9; echo "Max depth = $iMaxDepth
\n"; @@ -640,24 +638,24 @@ class TestMyBizModel extends TestBizModel echo "

Test object lifecycle

"; - MyHelpers::var_dump_html(MetaModel::GetStateAttributeCode("cmdbContact")); - MyHelpers::var_dump_html(MetaModel::EnumStates("cmdbContact")); - MyHelpers::var_dump_html(MetaModel::EnumStimuli("cmdbContact")); + self::DumpVariable(MetaModel::GetStateAttributeCode("cmdbContact")); + self::DumpVariable(MetaModel::EnumStates("cmdbContact")); + self::DumpVariable(MetaModel::EnumStimuli("cmdbContact")); foreach(MetaModel::EnumStates("cmdbContact") as $sStateCode => $aStateDef) { echo "

Transition from $sStateCode

\n"; - MyHelpers::var_dump_html(MetaModel::EnumTransitions("cmdbContact", $sStateCode)); + self::DumpVariable(MetaModel::EnumTransitions("cmdbContact", $sStateCode)); } $oObj = MetaModel::GetObject("cmdbContact", 18); echo "Current state: ".$oObj->GetState()."... let's go to school..."; - MyHelpers::var_dump_html($oObj->EnumTransitions()); + self::DumpVariable($oObj->EnumTransitions()); $oObj->ApplyStimulus("toschool"); echo "New state: ".$oObj->GetState()."... let's get older..."; - MyHelpers::var_dump_html($oObj->EnumTransitions()); + self::DumpVariable($oObj->EnumTransitions()); $oObj->ApplyStimulus("raise"); echo "New state: ".$oObj->GetState()."... let's try to go further... (should give an error)"; - MyHelpers::var_dump_html($oObj->EnumTransitions()); + self::DumpVariable($oObj->EnumTransitions()); $oObj->ApplyStimulus("raise"); // should give an error } @@ -774,7 +772,7 @@ class TestQueriesOnFarm extends MyFarm { //$oOql = new OqlInterpreter($sQuery); //$oTrash = $oOql->ParseObjectQuery(); - //MyHelpers::var_dump_html($oTrash, true); + //self::DumpVariable($oTrash, true); $oMyFilter = DBObjectSearch::FromOQL($sQuery); } catch (OQLException $oOqlException) @@ -809,10 +807,10 @@ class TestQueriesOnFarm extends MyFarm $this->search_and_show_list($oMyFilter); //echo "

first pass

\n"; - //MyHelpers::var_dump_html($oMyFilter, true); + //self::DumpVariable($oMyFilter, true); $sQuery1 = MetaModel::MakeSelectQuery($oMyFilter); //echo "

second pass

\n"; - //MyHelpers::var_dump_html($oMyFilter, true); + //self::DumpVariable($oMyFilter, true); //$sQuery1 = MetaModel::MakeSelectQuery($oMyFilter); $sSerialize = $oMyFilter->serialize(); @@ -991,7 +989,7 @@ class TestBulkChangeOnFarm extends TestBizModel chita,456, "); $aData = $oParser->ToArray(array('_name', '_height', '_birth'), ','); - MyHelpers::var_dump_html($aData); + self::DumpVariable($aData); $oBulk = new BulkChange( 'Mammal', @@ -1012,10 +1010,10 @@ class TestBulkChangeOnFarm extends TestBizModel echo "

Planned for loading...

"; $aRes = $oBulk->Process(); - print_r($aRes); + self::DumpVariable($aRes); echo "

Go for loading...

"; $aRes = $oBulk->Process($oMyChange); - print_r($aRes); + self::DumpVariable($aRes); return; @@ -1165,11 +1163,13 @@ class TestItopEfficiency extends TestBizModel 'SELECT CMDBChangeOpSetAttribute WHERE id=10', 'SELECT CMDBChangeOpSetAttribute WHERE id=123456789', 'SELECT CMDBChangeOpSetAttribute WHERE CMDBChangeOpSetAttribute.id=10', - 'SELECT bizIncidentTicket', - 'SELECT bizIncidentTicket WHERE id=1', - 'SELECT bizPerson', - 'SELECT bizPerson WHERE id=1', - 'SELECT bizIncidentTicket JOIN bizPerson ON bizIncidentTicket.agent_id = bizPerson.id WHERE bizPerson.id = 5', + 'SELECT Ticket', + 'SELECT Ticket WHERE id=1', + 'SELECT Person', + 'SELECT Person WHERE id=1', + 'SELECT Server', + 'SELECT Server WHERE id=1', + 'SELECT Incident JOIN Person ON Incident.agent_id = Person.id WHERE Person.id = 5', ); $aStats = array(); foreach ($aQueries as $sOQL) @@ -1197,28 +1197,51 @@ class TestItopEfficiency extends TestBizModel // Test data load /////////////////////////////////////////////////////////////////////////// -class TestItopWebServices extends TestWebServices +class TestImportREST extends TestWebServices { static public function GetName() { - return 'Itop - web services'; + return 'CSV import (REST)'; } static public function GetDescription() { - return 'Bulk load and ???'; + return 'Bulk load in the background'; } protected function DoExecSingleLoad($aLoadSpec) { - $sTitle = 'Load: '.$aLoadSpec['class']; - $sClass = $aLoadSpec['class']; $sCsvData = $aLoadSpec['csvdata']; - $aPostData = array('class' => $sClass, 'csvdata' => $sCsvData); - $sRes = self::DoPostRequestAuth('../webservices/import.php', $aPostData); + echo "
\n"; + echo "

{$aLoadSpec['desc']}

\n"; - echo "

$sTitle

$sCsvData
$sRes
"; + $aPostData = array('csvdata' => $sCsvData); + + $aGetParams = array(); + $aGetParamReport = array(); + foreach($aLoadSpec['args'] as $sArg => $sValue) + { + $aGetParams[] = $sArg.'='.urlencode($sValue); + $aGetParamReport[] = $sArg.'='.$sValue; + } + $sGetParams = implode('&', $aGetParams); + $sRes = self::DoPostRequestAuth('../webservices/import.php?'.$sGetParams, $aPostData); + + $sArguments = implode('
', $aGetParamReport); + + echo "
\n"; + echo "
\n"; + echo " $sArguments\n"; + echo "
\n"; + echo "
\n"; + echo "
$sCsvData
\n"; + echo "
\n"; + echo "
\n"; + + echo "
$sRes
\n"; + + echo "
\n"; } protected function DoExecute() @@ -1226,30 +1249,244 @@ class TestItopWebServices extends TestWebServices $aLoads = array( array( - 'class' => 'bizOrganization', - 'csvdata' => "name;code\nWorldCompany;WCY" + 'desc' => 'Missing class', + 'args' => array( + ), + 'csvdata' => "xxx", ), array( - 'class' => 'bizLocation', - 'csvdata' => "name;org_id;address\nParis;1;Centre de la Franca" + 'desc' => 'Wrong class', + 'args' => array( + 'class' => 'toto', + ), + 'csvdata' => "xxx", ), array( - 'class' => 'bizPerson', - 'csvdata' => "email;name;first_name;org_id;phone\njohn.foo@starac.com;Foo;John;1;+33(1)23456789" + 'desc' => 'Wrong output type', + 'args' => array( + 'class' => 'NetworkDevice', + 'output' => 'onthefly', + ), + 'csvdata' => "xxx", ), array( - 'class' => 'bizTeam', - 'csvdata' => "name;org_id;location_name\nSquadra Azzura2;1;Paris" + 'desc' => 'Wrong report level', + 'args' => array( + 'class' => 'NetworkDevice', + 'reportlevel' => 'errors|ouarnings|changed', + ), + 'csvdata' => "xxx", ), array( - 'class' => 'bizWorkgroup', - 'csvdata' => "name;org_id;team_id\ntravailleurs alpins;1;6" + 'desc' => 'Weird format, working anyhow...', + 'args' => array( + 'class' => 'Server', + 'output' => 'details', + 'separator' => '*', + 'qualifier' => '@', + 'reconciliationkeys' => 'org_id,name', + ), + 'csvdata' => 'name*org_id + server01*2 + @server02@@combodo@* 2 + server45*99', ), array( - 'class' => 'bizIncidentTicket', - 'csvdata' => "name;title;type;org_id;initial_situation;start_date;next_update;caller_id;workgroup_id;agent_id\nOVSD-12345;server down;Network;1;server was found down;2009-04-10 12:00;2009-04-10 15:00;3;317;5" + 'desc' => 'Load an organization', + 'args' => array( + 'class' => 'Organization', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "name;code\nWorldCompany;WCY", ), - ); + array( + 'desc' => 'Load a location', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "name;org_id;address\nParis;1;Centre de la Franca", + ), + array( + 'desc' => 'Load a person', + 'args' => array( + 'class' => 'Person', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "email;name;first_name;org_id;phone\njohn.foo@starac.com;Foo;John;1;+33(1)23456789", + ), + array( + 'desc' => 'Load a person - wrong email format', + 'args' => array( + 'class' => 'Person', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "email;name;first_name;org_id\nemailPASbon;Foo;John;1", + ), + array( + 'desc' => 'Load a team', + 'args' => array( + 'class' => 'Team', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "name;org_id;location_name\nSquadra Azzura2;1;Paris", + ), + array( + 'desc' => 'Load server', + 'args' => array( + 'class' => 'Server', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "name;status;owner_name;location_name;os_family;os_version;management_ip;cpu;ram;brand;model;serial_number\nlocalhost.;production;Demo;Grenoble;Ubuntu 9.10;2.6.31-19-generic-#56-Ubuntu SMP Thu Jan 28 01:26:53 UTC 2010;16.16.230.232;Intel(R) Core(TM)2 Duo CPU T7100 @ 1.80GHz;2005;Hewlett-Packard;HP Compaq 6510b (GM108UC#ABF);CNU7370BNP", + ), + array( + 'desc' => 'Load NW if', + 'args' => array( + 'class' => 'NetworkInterface', + 'output' => 'details', + 'reconciliationkeys' => '', + ), + 'csvdata' => "name;status;org_id;device_name;physical_type;ip_address;ip_mask;mac_address;speed\neth0;implementation;2;localhost.;ethernet;16.16.230.232;255.255.240.0;00:1a:4b:68:e3:97;\nlo;implementation;2;localhost.;ethernet;127.0.0.1;255.0.0.0;;", + ), + // Data Bruno + array( + 'desc' => 'Load NW devices from real life', + 'args' => array( + 'class' => 'NetworkDevice', + 'output' => 'details', + 'reconciliationkeys' => 'org_id->name,name', + ), + 'csvdata' => 'name;management_ip;importance;org_id->name;type + truc-machin-bidule;172.15.255.150;high;My Company/Department;switch + 10.15.255.222;10.15.255.222;high;My Company/Department;switch', + ), + array( + 'desc' => 'Load NW ifs', + 'args' => array( + 'class' => 'NetworkInterface', + 'output' => 'details', + 'reconciliationkeys' => 'device_id->name,name', + ), + 'csvdata' => 'device_id->name;org_id->name;name;ip_address;ip_mask;speed;link_type;mac_address;physical_type + truc-machin-bidule;My Company/Department;"GigabitEthernet44";;;0;downlink;00 12 F2 CB C4 EB ;ethernet + truc-machin-bidule;My Company/Department;"GigabitEthernet38";;;0;downlink;00 12 F2 CB C4 E5 ;ethernet + un-autre;My Company/Department;"GigabitEthernet2/3";;;1000000000;uplink;00 12 F2 20 0F 1A ;ethernet', + ), + array( + 'desc' => 'The simplest data load', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name\nParis", + ), + array( + 'desc' => 'The simplest data load + org', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org_id\nParis;2", + ), + array( + 'desc' => 'The simplest data load + org (name)', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org_name\nParis;Demo", + ), + array( + 'desc' => 'The simplest data load + org (code)', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org_id->code\nParis;DEMO", + ), + array( + 'desc' => 'Ouput: summary', + 'args' => array( + 'class' => 'Location', + 'output' => 'summary', + ), + 'csvdata' => "name;org_id->code\nParis;DEMO", + ), + array( + 'desc' => 'Ouput: retcode', + 'args' => array( + 'class' => 'Location', + 'output' => 'retcode', + ), + 'csvdata' => "name;org_id->code\nParis;DEMO", + ), + array( + 'desc' => 'Error in reconciliation list', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + 'reconciliationkeys' => 'org_id', + ), + 'csvdata' => "name\nParis", + ), + array( + 'desc' => 'Error in attribute list that does not allow to compute reconciliation scheme', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "country\nFrance", + ), + array( + 'desc' => 'Error in attribute list - case A', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org\nParis;2", + ), + array( + 'desc' => 'Error in attribute list - case B1 (key->attcode)', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org->code\nParis;DEMO", + ), + array( + 'desc' => 'Error in attribute list - case B2 (key->attcode)', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + ), + 'csvdata' => "name;org_id->duns\nParis;DEMO", + ), + array( + 'desc' => 'Always changing... special comment in change tracking', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + 'comment' => 'automated testing' + ), + 'csvdata' => "name;address\nLe pantheon;Addresse bidon:".((string)microtime(true)), + ), + array( + 'desc' => 'Always changing... but "simulate"', + 'args' => array( + 'class' => 'Location', + 'output' => 'details', + 'simulate' => '1', + 'comment' => 'SHOULD NEVER APPEAR IN THE HISTORY' + ), + 'csvdata' => "name;address\nLe pantheon;restore address?", + ), + ); foreach ($aLoads as $aLoadSpec) { @@ -1534,9 +1771,7 @@ class TestSoap extends TestSoapWebService if (false) { - print "
\n"; 
-			print_r($this->m_SoapClient->__getTypes());
-			print "
\n"; + self::DumpVariable($this->m_SoapClient->__getTypes()); } global $aWebServices; @@ -1559,9 +1794,7 @@ class TestSoap extends TestSoapWebService throw $e; } - echo "
\n";
-			print_r($oRes);
-			echo "
\n"; + self::DumpVariable($oRes); print "
\n"; 
 			print "Request: \n".htmlspecialchars($this->m_SoapClient->__getLastRequest()) ."\n"; 
@@ -1605,9 +1838,7 @@ class TestWebServicesDirect extends TestBizModel
 			echo "

SOAP call #$iPos - {$aWebService['verb']}

\n"; echo "

{$aWebService['explain result']}

\n"; $oRes = call_user_func_array(array($oWebServices, $aWebService['verb']), $aWebService['args']); - echo "
\n";
-			print_r($oRes);
-			echo "
\n"; + self::DumpVariable($oRes); if ($oRes instanceof SOAPResult) { diff --git a/setup/benchmark.php b/setup/benchmark.php index 3ebd62be0..53d57524a 100644 --- a/setup/benchmark.php +++ b/setup/benchmark.php @@ -145,7 +145,7 @@ class BenchmarkDataCreation $oMyObject->Set($sProp, $value); } - $iId = $oMyObject->DBInsertTrackedNoReload($this->m_oChange); + $iId = $oMyObject->DBInsertTrackedNoReload($this->m_oChange, true /* skip security */); $sClassId = "$sClass ($sClassDesc)"; $this->m_aCreatedByDesc[$sClassId][] = $iId; diff --git a/webservices/import.php b/webservices/import.php index ff531be18..a0436130c 100644 --- a/webservices/import.php +++ b/webservices/import.php @@ -25,17 +25,12 @@ // // Known limitations -// - output still in html, because the errors are not displayed in xml // - reconciliation is made on the first column -// - no option to force 'always create' or 'never create' -// - text qualifier hardcoded to " // // Known issues // - ALMOST impossible to troubleshoot when an externl key has a wrong value // - no character escaping in the xml output (yes !?!?!) // - not outputing xml when a wrong input is given (class, attribute names) -// - for a bizIncidentTicket you may use the name as the reconciliation key, -// but that attribute is in fact recomputed by the application! An error should be raised somewhere // require_once('../application/application.inc.php'); @@ -45,147 +40,583 @@ require_once('../application/xmlpage.class.inc.php'); require_once('../application/startup.inc.php'); -require_once('../application/loginwebpage.class.inc.php'); - -class WebServiceException extends Exception +class BulkLoadException extends Exception { } -LoginWebPage::DoLogin(); // Check user rights and prompt if needed +$aPageParams = array +( + 'class' => array + ( + 'mandatory' => true, + 'default' => null, + 'description' => 'class of loaded objects', + ), + 'csvdata' => array + ( + 'mandatory' => true, + 'default' => null, + 'description' => 'data', + ), + 'charset' => array + ( + 'mandatory' => false, + 'default' => 'UTF-8', + 'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15', + ), +// 'csvfile' => array +// ( +// 'mandatory' => false, +// 'default' => '', +// 'description' => 'local data file, replaces csvdata if specified', +// ), + 'separator' => array + ( + 'mandatory' => false, + 'default' => ';', + 'description' => 'column separator in CSV data', + ), + 'qualifier' => array + ( + 'mandatory' => false, + 'default' => '"', + 'description' => 'test qualifier in CSV data', + ), + 'output' => array + ( + 'mandatory' => false, + 'default' => 'summary', + 'description' => '[retcode] to return the count of lines in error, [summary] to return a concise report, [details] to get a detailed report (each line listed)', + ), +/* + 'reportlevel' => array + ( + 'mandatory' => false, + 'default' => 'errors|warnings|created|changed|unchanged', + 'description' => 'combination of flags to limit the detailed output', + ), +*/ + 'reconciliationkeys' => array + ( + 'mandatory' => false, + 'default' => '', + 'description' => 'name of the columns used to identify existing objects and update them, or create a new one', + ), + 'simulate' => array + ( + 'mandatory' => false, + 'default' => '0', + 'description' => 'If set to 1, then the load will not be executed, but the expected report will be produced', + ), + 'comment' => array + ( + 'mandatory' => false, + 'default' => '', + 'description' => 'Comment to be added into the change log', + ), +); -$oAppContext = new ApplicationContext(); -//$iActiveNodeId = utils::ReadParam('menu', -1); -//$currentOrganization = utils::ReadParam('org_id', ''); +function UsageAndExit($oP) +{ + global $aPageParams; + $oP->p("USAGE:\n"); + foreach($aPageParams as $sParam => $aParamData) + { + $sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']'); + $oP->p("$sParam = $sDesc\n"); + } + $oP->output(); + exit; +} + +function ReadParam($oP, $sParam) +{ + global $aPageParams; + assert(isset($aPageParams[$sParam])); + assert(!$aPageParams[$sParam]['mandatory']); + $sValue = utils::ReadParam($sParam, $aPageParams[$sParam]['default'], true /* Allow CLI */); + return trim($sValue); +} + +function ReadMandatoryParam($oP, $sParam) +{ + global $aPageParams; + assert(isset($aPageParams[$sParam])); + assert($aPageParams[$sParam]['mandatory']); + + $sValue = utils::ReadParam($sParam, null, true /* Allow CLI */); + if (is_null($sValue)) + { + $oP->p("ERROR: Missing argument '$sParam'\n"); + UsageAndExit($oP); + } + return trim($sValue); +} + +///////////////////////////////// // Main program -//$oP = new XMLPage("iTop - Bulk import"); -$oP = new WebPage("iTop - Bulk import"); -$oP->add('This is a prototype, I repeat: PRO-TO-TYPE, therefore it suffers bugs and limitations, documented in the code. Next step: specify...'); +if (false && utils::IsModeCLI()) +{ + // Mode CLI under construction... sorry! + // Next steps: + // implement a new page: CLIPage, that does a very clean output + // branch if in CLI mode + // enable the new argument 'csvfile' + $sAuthUser = ReadMandatoryParam($oP, 'auth_user'); + $sAuthPwd = ReadMandatoryParam($oP, 'auth_user'); + $sCsvFile = ReadMandatoryParam($oP, 'csv_file'); + if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd)) + { + UserRights::Login($sAuthUser); // Login & set the user's language + } + else + { + $oP->p("Access restricted or wrong credentials ('$sAuthUser')"); + exit; + } +} +else +{ + require_once('../application/loginwebpage.class.inc.php'); + LoginWebPage::DoLogin(); // Check user rights and prompt if needed + + $oP = new CSVPage("iTop - Bulk import"); +} + + try { - $sClass = utils::ReadParam('class', ''); - $sSep = utils::ReadParam('separator', ';'); + ////////////////////////////////////////////////// + // + // Read parameters + // + $sClass = ReadMandatoryParam($oP, 'class'); + $sSep = ReadParam($oP, 'separator'); + $sQualifier = ReadParam($oP, 'qualifier'); $sCSVData = utils::ReadPostedParam('csvdata'); + $sCharSet = ReadParam($oP, 'charset'); + $sOutput = ReadParam($oP, 'output'); +// $sReportLevel = ReadParam($oP, 'reportlevel'); + $sReconcKeys = ReadParam($oP, 'reconciliationkeys'); + $sSimulate = ReadParam($oP, 'simulate'); + $sComment = ReadParam($oP, 'comment'); - $oCSVParser = new CSVParser($sCSVData, $sSep, $sDelimiter = '"'); + ////////////////////////////////////////////////// + // + // Check parameters format/consistency + // + if (!MetaModel::IsValidClass($sClass)) + { + throw new BulkLoadException("Unknown class: '$sClass'"); + } + + if (strlen($sSep) > 1) + { + throw new BulkLoadException("Separator is limited to one character, found '$sSep'"); + } + + if (strlen($sQualifier) > 1) + { + throw new BulkLoadException("Text qualifier is limited to one character, found '$sQualifier'"); + } + + if (!in_array($sOutput, array('retcode', 'summary', 'details'))) + { + throw new BulkLoadException("Unknown output format: '$sOutput'"); + } + +/* + $aReportLevels = explode('|', $sReportLevel); + foreach($aReportLevels as $sLevel) + { + if (!in_array($sLevel, explode('|', 'errors|warnings|created|changed|unchanged'))) + { + throw new BulkLoadException("Unknown level in reporting level: '$sLevel'"); + } + } +*/ + + if ($sSimulate == '1') + { + $bSimulate = true; + } + else + { + $bSimulate = false; + } + + ////////////////////////////////////////////////// + // + // Parse first line, check attributes, analyse the request + // + if ($sCharSet == 'UTF-8') + { + $sUTF8Data = $sCSVData; + } + else + { + $sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData); + } + $oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); // Limitation: as the attribute list is in the first line, we can not match external key by a third-party attribute - $sRawFieldList = $oCSVParser->ListFields(); + $aRawFieldList = $oCSVParser->ListFields(); + $iColCount = count($aRawFieldList); + $aAttList = array(); $aExtKeys = array(); - foreach($sRawFieldList as $iFieldId => $sFieldName) + foreach($aRawFieldList as $iFieldId => $sFieldName) { - if (!MetaModel::IsValidAttCode($sClass, $sFieldName)) + if (preg_match('/^(.*)->(.*)$/', trim($sFieldName), $aMatches)) { - throw new WebServiceException("Unknown attribute '$sFieldName' (class: '$sClass')"); - } - $oAtt = MetaModel::GetAttributeDef($sClass, $sFieldName); - if ($oAtt->IsExternalKey()) - { - $aExtKeys[$sFieldName]['id'] = $iFieldId; - $aAttList[$sFieldName] = $iFieldId; - } - elseif ($oAtt->IsExternalField()) - { - $sExtKeyAttCode = $oAtt->GetKeyAttCode(); - $sRemoteAttCode = $oAtt->GetExtAttCode(); + // The column has been specified as "extkey->attcode" + // + $sExtKeyAttCode = $aMatches[1]; + $sRemoteAttCode = $aMatches[2]; + if (!MetaModel::IsValidAttCode($sClass, $sExtKeyAttCode)) + { + throw new BulkLoadException("Unknown attribute '$sExtKeyAttCode' (class: '$sClass')"); + } + $oAtt = MetaModel::GetAttributeDef($sClass, $sExtKeyAttCode); + if (!$oAtt->IsExternalKey()) + { + throw new BulkLoadException("Not an external key '$sExtKeyAttCode' (class: '$sClass')"); + } + $sTargetClass = $oAtt->GetTargetClass(); + if (!MetaModel::IsValidAttCode($sTargetClass, $sRemoteAttCode)) + { + throw new BulkLoadException("Unknown attribute '$sRemoteAttCode' (key: '$sExtKeyAttCode', class: '$sTargetClass')"); + } $aExtKeys[$sExtKeyAttCode][$sRemoteAttCode] = $iFieldId; } else { - $aAttList[$sFieldName] = $iFieldId; + // The column has been specified as "attcode" + // + if (!MetaModel::IsValidAttCode($sClass, $sFieldName)) + { + throw new BulkLoadException("Unknown attribute '$sFieldName' (class: '$sClass')"); + } + $oAtt = MetaModel::GetAttributeDef($sClass, $sFieldName); + if ($oAtt->IsExternalKey()) + { + $aExtKeys[$sFieldName]['id'] = $iFieldId; + $aAttList[$sFieldName] = $iFieldId; + } + elseif ($oAtt->IsExternalField()) + { + $sExtKeyAttCode = $oAtt->GetKeyAttCode(); + $sRemoteAttCode = $oAtt->GetExtAttCode(); + $aExtKeys[$sExtKeyAttCode][$sRemoteAttCode] = $iFieldId; + } + else + { + $aAttList[$sFieldName] = $iFieldId; + } } } - // Limitation: the reconciliation key is the first attribute - $aReconcilKeys = array($sRawFieldList[0]); + // Make sure there are some reconciliation keys + // + if (empty($sReconcKeys)) + { + $aReconcSpec = array(); + // Base reconciliation scheme on the default one + // The reconciliation attributes not present in the data will be ignored + foreach(MetaModel::GetReconcKeys($sClass) as $sReconcKeyAttCode) + { + if (in_array($sReconcKeyAttCode, $aRawFieldList)) + { + $aReconcSpec[] = $sReconcKeyAttCode; + } + } + if (count($aReconcSpec) == 0) + { + throw new BulkLoadException("No reconciliation scheme could be defined, please add a column corresponding to one defined reconciliation key (class: '$sClass', reconciliation:".implode(',', MetaModel::GetReconcKeys($sClass)).")"); + } + $sReconcKeys = implode(',', $aReconcSpec); + } + +if (false) +{ +echo "Reconciliation keys
";
+print_r($sReconcKeys);
+throw new BulkLoadException("testing");
+}
+	// Interpret the list of reconciliation keys
+	//
+	$aFinalReconcilKeys = array();
+	$aReconcilKeysReport = array();
+	foreach (explode(',', $sReconcKeys) as $sReconcKey)
+	{
+		$sReconcKey = trim($sReconcKey);
+		if (empty($sReconcKey)) continue; // skip empty spec
+
+		if (!in_array($sReconcKey, $aRawFieldList))
+		{
+			throw new BulkLoadException("Reconciliation keys not found in the input columns '$sReconcKey' (class: '$sClass')");
+		}
+
+		if (preg_match('/^(.*)->(.*)$/', trim($sReconcKey), $aMatches))
+		{
+			// The column has been specified as "extkey->attcode"
+			//
+			$sExtKeyAttCode = $aMatches[1];
+			$sRemoteAttCode = $aMatches[2];
+
+			$aFinalReconcilKeys[] = $sExtKeyAttCode;
+			$aReconcilKeysReport[$sExtKeyAttCode][] = $sRemoteAttCode;
+		}
+		else
+		{
+			if (!MetaModel::IsValidAttCode($sClass, $sReconcKey))
+			{
+				// Safety net: should never happen, but...
+				throw new BulkLoadException("Unknown reconciliation attribute '$sReconcKey' (class: '$sClass')");
+			}
+			$oAtt = MetaModel::GetAttributeDef($sClass, $sReconcKey);
+			if ($oAtt->IsExternalKey())
+			{
+				$aFinalReconcilKeys[] = $sReconcKey;
+				$aReconcilKeysReport[$sReconcKey][] = 'id';
+			}
+			elseif ($oAtt->IsExternalField())
+			{
+				$sReconcAttCode = $oAtt->GetKeyAttCode();
+				$sReconcKeyReport = "$sReconcAttCode ($sReconcKey)";
+
+				$aFinalReconcilKeys[] = $sReconcAttCode;
+				$aReconcilKeysReport[$sReconcAttCode][] = $sReconcKeyReport;
+			}
+			else
+			{
+				$aFinalReconcilKeys[] = $sReconcKey;
+				$aReconcilKeysReport[$sReconcKey] = array();
+			}
+		}
+	}
+
+	//////////////////////////////////////////////////
+	//
+	// Go for parsing and interpretation
+	//
 
 	$aData = $oCSVParser->ToArray();
+	$iLineCount = count($aData);
+
 	$oBulk = new BulkChange(
 		$sClass,
 		$aData,
 		$aAttList,
 		$aExtKeys,
-		$aReconcilKeys
+		$aFinalReconcilKeys
 	);
 
-	$oMyChange = MetaModel::NewObject("CMDBChange");
-	$oMyChange->Set("date", time());
-	if (UserRights::IsImpersonated())
+	if ($bSimulate)
 	{
-		$sUserString = Dict::Format('UI:Archive_User_OnBehalfOf_User', UserRights::GetRealUser(), UserRights::GetUser());
+		$oMyChange = null;
 	}
 	else
 	{
-		$sUserString = UserRights::GetUser();
+		$oMyChange = MetaModel::NewObject("CMDBChange");
+		$oMyChange->Set("date", time());
+		if (UserRights::IsImpersonated())
+		{
+			$sUserString = Dict::Format('UI:Archive_User_OnBehalfOf_User', UserRights::GetRealUser(), UserRights::GetUser());
+		}
+		else
+		{
+			$sUserString = UserRights::GetUser();
+		}
+		if (strlen($sComment) > 0)
+		{
+			$sMoreInfo = 'Web Service (CSV) - '.$sComment;
+		}
+		else
+		{
+			$sMoreInfo = 'Web Service (CSV)';
+		}
+		$oMyChange->Set("userinfo", $sUserString.' '.$sMoreInfo);
+		$iChangeId = $oMyChange->DBInsert();
 	}
-	$oMyChange->Set("userinfo", $sUserString.' (bulk load by web service)');
-	$iChangeId = $oMyChange->DBInsert();
 
 	$aRes = $oBulk->Process($oMyChange);
 
-	// Setup result presentation
+	//////////////////////////////////////////////////
 	//
-	$aDisplayConfig = array();
-	$aDisplayConfig["__RECONCILIATION__"] = array("label"=>"Reconciliation", "description"=>"");
-	$aDisplayConfig["__STATUS__"] = array("label"=>"Status", "description"=>"");
-	if (isset($iPKeyId))
-	{
-		$aDisplayConfig["col$iPKeyId"] = array("label"=>"id", "description"=>"");
-	}
-	foreach($aReconcilKeys as $iCol => $sAttCode)
-	{
-		$sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel();
-		$aDisplayConfig["col$iCol"] = array("label"=>"$sLabel", "description"=>"");
-	}
-	foreach ($aAttList as $sAttCode => $iCol)
-	{
-		$sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel();
-		$aDisplayConfig["col$iCol"] = array("label"=>"$sLabel", "description"=>"");
-	}
-
-
-	$aResultDisp = array(); // to be displayed
+	// Compute statistics
+	//
+	$iCountErrors = 0;
+	$iCountWarnings = 0;
+	$iCountCreations = 0;
+	$iCountUpdates = 0;
+	$iCountUnchanged = 0;
 	foreach($aRes as $iRow => $aRowData)
 	{
-		$aRowDisp = array();
-		$aRowDisp["__RECONCILIATION__"] = $aRowData["__RECONCILIATION__"];
-		$aRowDisp["__STATUS__"] = $aRowData["__STATUS__"]->GetDescription();
-		foreach($aRowData as $sKey => $value)
+		$bWritten = false;
+
+		$oStatus = $aRowData["__STATUS__"];
+		switch(get_class($oStatus))
 		{
-			if ($sKey == '__RECONCILIATION__') continue;
-			if ($sKey == '__STATUS__') continue;
-			
-			switch (get_class($value))
+		case 'RowStatus_NoChange':
+			$iCountUnchanged++;
+			break;
+		case 'RowStatus_Modify':
+			$iCountUpdates++;
+			$bWritten = true;
+			break;
+		case 'RowStatus_NewObj':
+			$iCountCreations++;
+			$bWritten = true;
+			break;
+		case 'RowStatus_Issue':
+			$iCountErrors++;
+			break;		
+		}
+
+		if ($bWritten)
+		{
+			// Something has been done, still there may be some issues to report
+			foreach($aRowData as $key => $value)
 			{
-				case 'CellStatus_Void':
-					$sClass = '';
-					break;
-				case 'CellStatus_Modify':
-					$sClass = 'csvimport_ok';
-					break;
-				case 'CellStatus_Issue':
-					$sClass = 'csvimport_error';
-					break;
+				if (!is_object($value)) continue;
+
+				switch (get_class($value))
+				{
+					case 'CellStatus_Void':
+					case 'CellStatus_Modify':
+						break;
+					case 'CellStatus_Issue':
+					case 'CellStatus_Ambiguous':
+						$iCountWarnings++;
+						break;
+				}
 			}
-			if (empty($sClass))
+		}
+	}
+
+	//////////////////////////////////////////////////
+	//
+	// Summary of settings and results
+	//
+	if ($sOutput == 'retcode')
+	{
+		$oP->add($iCountErrors);
+	}
+
+	if (($sOutput == "summary") || ($sOutput == 'details'))
+	{
+		$aReconciliationReport = array();
+		foreach($aReconcilKeysReport as $sKey => $aKeyDetails)
+		{
+			if (count($aKeyDetails) > 0)
 			{
-				$aRowDisp[$sKey] = $value->GetDescription();
+				$aReconciliationReport[] = $sKey.' ('.implode(',', $aKeyDetails).')';
 			}
 			else
 			{
-				$aRowDisp[$sKey] = "
".$value->GetDescription()."
"; + $aReconciliationReport[] = $sKey; } } - $aResultDisp[$iRow] = $aRowDisp; + $oP->add_comment("Class: ".$sClass); + $oP->add_comment("Separator: ".$sSep); + $oP->add_comment("Qualifier: ".$sQualifier); + $oP->add_comment("Charset Encoding:".$sCharSet); + $oP->add_comment("Data Size: ".strlen($sCSVData)); + $oP->add_comment("Data Lines: ".$iLineCount); + $oP->add_comment("Columns: ".implode(', ', $aRawFieldList)); + $oP->add_comment("Reconciliation Keys: ".implode(', ', $aReconciliationReport)); + $oP->add_comment("Output format: ".$sOutput); +// $oP->add_comment("Report level: ".$sReportLevel); + $oP->add_comment("Simulate: ".($bSimulate ? '1' : '0')); + $oP->add_comment("Change tracking comment: ".$sComment); + $oP->add_comment("Issues: ".$iCountErrors); + $oP->add_comment("Warnings: ".$iCountWarnings); + $oP->add_comment("Created: ".$iCountCreations); + $oP->add_comment("Updated: ".$iCountUpdates); + $oP->add_comment("Unchanged: ".$iCountUnchanged); } - $oP->table($aDisplayConfig, $aResultDisp); + + if ($sOutput == 'details') + { + // Setup result presentation + // + $aDisplayConfig = array(); + $aDisplayConfig["__LINE__"] = array("label"=>"Line", "description"=>""); + $aDisplayConfig["__STATUS__"] = array("label"=>"Status", "description"=>""); + $aDisplayConfig["__OBJECT_CLASS__"] = array("label"=>"Object Class", "description"=>""); + $aDisplayConfig["__OBJECT_ID__"] = array("label"=>"Object Id", "description"=>""); + foreach($aExtKeys as $sExtKeyAttCode => $aRemoteAtt) + { + $sLabel = MetaModel::GetAttributeDef($sClass, $sExtKeyAttCode)->GetLabel(); + $aDisplayConfig["$sExtKeyAttCode"] = array("label"=>$sExtKeyAttCode, "description"=>$sLabel." - ext key"); + } + foreach($aFinalReconcilKeys as $iCol => $sAttCode) + { + // $sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel(); + // $aDisplayConfig["$iCol"] = array("label"=>"$sLabel", "description"=>""); + } + foreach ($aAttList as $sAttCode => $iCol) + { + $sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel(); + $aDisplayConfig["$iCol"] = array("label"=>$sAttCode, "description"=>$sLabel); + } + + $aResultDisp = array(); // to be displayed + foreach($aRes as $iRow => $aRowData) + { + $aRowDisp = array(); + $aRowDisp["__LINE__"] = $iRow; + if (is_object($aRowData["__STATUS__"])) + { + $aRowDisp["__STATUS__"] = $aRowData["__STATUS__"]->GetDescription(); + } + else + { + $aRowDisp["__STATUS__"] = "*No status available*"; + } + if (isset($aRowData["finalclass"]) && isset($aRowData["id"])) + { + $aRowDisp["__OBJECT_CLASS__"] = $aRowData["finalclass"]; + $aRowDisp["__OBJECT_ID__"] = $aRowData["id"]->GetValue(); + } + else + { + $aRowDisp["__OBJECT_CLASS__"] = "n/a"; + $aRowDisp["__OBJECT_ID__"] = "n/a"; + } + foreach($aRowData as $key => $value) + { + $sKey = (string) $key; + + if ($sKey == '__STATUS__') continue; + if ($sKey == 'finalclass') continue; + if ($sKey == 'id') continue; + + if (is_object($value)) + { + $aRowDisp["$sKey"] = $value->GetValue().$value->GetDescription(); + } + else + { + $aRowDisp["$sKey"] = $value; + } + } + $aResultDisp[$iRow] = $aRowDisp; + } + $oP->table($aDisplayConfig, $aResultDisp); + } +} +catch(BulkLoadException $e) +{ + $oP->add_comment($e->getMessage()); } catch(Exception $e) { - $oP->add(''.((string)$e).''); + $oP->add_comment((string)$e); } $oP->output();