diff --git a/synchro/synchro_import.php b/synchro/synchro_import.php new file mode 100644 index 000000000..2d9f7a615 --- /dev/null +++ b/synchro/synchro_import.php @@ -0,0 +1,489 @@ + + * @author Romain Quetiez + * @author Denis Flaven + * @license http://www.opensource.org/licenses/gpl-3.0.html LGPL + */ + +// +// Known limitations +// - reconciliation is made on the column primary_key +// + +require_once('../approot.inc.php'); +require_once(APPROOT.'/application/application.inc.php'); +require_once(APPROOT.'/application/webpage.class.inc.php'); +require_once(APPROOT.'/application/csvpage.class.inc.php'); +require_once(APPROOT.'/application/clipage.class.inc.php'); + +require_once(APPROOT.'/application/startup.inc.php'); + +class ExchangeException extends Exception +{ +} + +$aPageParams = array +( + 'auth_user' => array + ( + 'mandatory' => true, + 'modes' => 'cli', + 'default' => null, + 'description' => 'login (must have enough rights to create objects of the given class)', + ), + 'auth_pwd' => array + ( + 'mandatory' => true, + 'modes' => 'cli', + 'default' => null, + 'description' => 'password', + ), + 'data_source_id' => array + ( + 'mandatory' => true, + 'modes' => 'http,cli', + 'default' => null, + 'description' => 'Synchro data source id', + ), + 'csvdata' => array + ( + 'mandatory' => true, + 'modes' => 'http', + 'default' => null, + 'description' => 'data', + ), + 'csvfile' => array + ( + 'mandatory' => true, + 'modes' => 'cli', + 'default' => '', + 'description' => 'local data file, replaces csvdata if specified', + ), + 'charset' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => 'UTF-8', + 'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15', + ), + 'separator' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => ';', + 'description' => 'column separator in CSV data', + ), + 'qualifier' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => '"', + 'description' => 'test qualifier in CSV data', + ), + 'output' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => 'summary', + 'description' => '[retcode] to return the count of lines in error, [summary] to return a concise report, [details] to get a detailed report (each line listed)', + ), +/* + 'reportlevel' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => 'errors|warnings|created|changed|unchanged', + 'description' => 'combination of flags to limit the detailed output', + ), +*/ + 'simulate' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => '0', + 'description' => 'If set to 1, then the load will not be executed, but the expected report will be produced', + ), + 'comment' => array + ( + 'mandatory' => false, + 'modes' => 'http,cli', + 'default' => '', + 'description' => 'Comment to be added into the change log', + ), +); + +function UsageAndExit($oP) +{ + global $aPageParams; + $bModeCLI = utils::IsModeCLI(); + + $oP->p("USAGE:\n"); + foreach($aPageParams as $sParam => $aParamData) + { + $aModes = explode(',', $aParamData['modes']); + if ($bModeCLI) + { + if (in_array('cli', $aModes)) + { + $sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']'); + $oP->p("$sParam = $sDesc"); + } + } + else + { + if (in_array('http', $aModes)) + { + $sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']'); + $oP->p("$sParam = $sDesc"); + } + } + } + $oP->output(); + exit; +} + + +function ReadParam($oP, $sParam) +{ + global $aPageParams; + assert(isset($aPageParams[$sParam])); + assert(!$aPageParams[$sParam]['mandatory']); + $sValue = utils::ReadParam($sParam, $aPageParams[$sParam]['default'], true /* Allow CLI */); + return trim($sValue); +} + +function ReadMandatoryParam($oP, $sParam) +{ + global $aPageParams; + assert(isset($aPageParams[$sParam])); + assert($aPageParams[$sParam]['mandatory']); + + $sValue = utils::ReadParam($sParam, null, true /* Allow CLI */); + if (is_null($sValue)) + { + $oP->p("ERROR: Missing argument '$sParam'\n"); + UsageAndExit($oP); + } + return trim($sValue); +} + +///////////////////////////////// +// Main program + +if (utils::IsModeCLI()) +{ + $oP = new CLIPage("iTop - Data Exchange"); + + // Next steps: + // specific arguments: 'csvfile' + // + $sAuthUser = ReadMandatoryParam($oP, 'auth_user'); + $sAuthPwd = ReadMandatoryParam($oP, 'auth_pwd'); + $sCsvFile = ReadMandatoryParam($oP, 'csvfile'); + if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd)) + { + UserRights::Login($sAuthUser); // Login & set the user's language + } + else + { + $oP->p("Access restricted or wrong credentials ('$sAuthUser')"); + exit; + } + + if (!is_readable($sCsvFile)) + { + $oP->p("Input file could not be found or could not be read: '$sCsvFile'"); + exit; + } + $sCSVData = file_get_contents($sCsvFile); + +} +else +{ + $_SESSION['login_mode'] = 'basic'; + require_once(APPROOT.'/application/loginwebpage.class.inc.php'); + LoginWebPage::DoLogin(); // Check user rights and prompt if needed + + $oP = new CSVPage("iTop - Data Exchange"); + $sCSVData = utils::ReadPostedParam('csvdata'); + $sCSVData = "primary_key;brand;model +100;Hewlett;PC100".microtime()." +101;Hewlett;PC101 +"; +} + + +try +{ + ////////////////////////////////////////////////// + // + // Read parameters + // + $iDataSourceId = ReadMandatoryParam($oP, 'data_source_id'); + $sSep = ReadParam($oP, 'separator'); + $sQualifier = ReadParam($oP, 'qualifier'); + $sCharSet = ReadParam($oP, 'charset'); + $sOutput = ReadParam($oP, 'output'); +// $sReportLevel = ReadParam($oP, 'reportlevel'); + $sSimulate = ReadParam($oP, 'simulate'); + $sComment = ReadParam($oP, 'comment'); + + ////////////////////////////////////////////////// + // + // Statistics + // + $iCountErrors = 0; + $iCountCreations = 0; + $iCountUpdates = 0; + + ////////////////////////////////////////////////// + // + // Check parameters format/consistency + // + if (strlen($sCSVData) == 0) + { + throw new ExchangeException("Missing data - at least one line is expected"); + } + + $oDataSource = MetaModel::GetObject('SynchroDataSource', $iDataSourceId, false); + if (is_null($oDataSource)) + { + throw new ExchangeException("Unknown data source id: '$iDataSourceId'"); + } + $sClass = $oDataSource->GetTargetClass(); + + if (strlen($sSep) > 1) + { + throw new ExchangeException("Separator is limited to one character, found '$sSep'"); + } + + if (strlen($sQualifier) > 1) + { + throw new ExchangeException("Text qualifier is limited to one character, found '$sQualifier'"); + } + + if (!in_array($sOutput, array('retcode', 'summary', 'details'))) + { + throw new ExchangeException("Unknown output format: '$sOutput'"); + } + +/* + $aReportLevels = explode('|', $sReportLevel); + foreach($aReportLevels as $sLevel) + { + if (!in_array($sLevel, explode('|', 'errors|warnings|created|changed|unchanged'))) + { + throw new ExchangeException("Unknown level in reporting level: '$sLevel'"); + } + } +*/ + + if ($sSimulate == '1') + { + $bSimulate = true; + } + else + { + $bSimulate = false; + } + + ////////////////////////////////////////////////// + // + // Security + // + // #@# todo - implement + /* + if (!$oDataSource->IsUserAllowed(UserRights::GetUserId()) + { + throw new SecurityException(Dict::Format('UI:Error:DataExchangeNotAllowed', UserRights::GetUserFriendlyName())); + } + */ + + ////////////////////////////////////////////////// + // + // Parse first line, check attributes, analyse the request + // + if ($sCharSet == 'UTF-8') + { + $sUTF8Data = $sCSVData; + } + else + { + $sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData); + } + $oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); + + $aInputColumns = $oCSVParser->ListFields(); + $iColCount = count($aInputColumns); + + // Check columns + $aColumns = $oDataSource->GetSQLColumns(); + foreach($aInputColumns as $iFieldId => $sInputColumn) + { + if ($sInputColumn == 'primary_key') + { + $iPrimaryKeyCol = $iFieldId; + continue; + } + if (!array_key_exists($sInputColumn, $aColumns)) + { + throw new ExchangeException("Unknown column '$sInputColumn' (class: '$sClass')"); + } + } + if (!isset($iPrimaryKeyCol)) + { + throw new ExchangeException("Missing reconciliation column 'primary_key'"); + } + + ////////////////////////////////////////////////// + // + // Go for parsing and interpretation + // + + $aData = $oCSVParser->ToArray(); + $iLineCount = count($aData); + + $sTable = $oDataSource->GetDataTable(); + + // Prepare insert columns + $sInsertColumns = '`'.implode('`, `', $aInputColumns).'`'; + + foreach($aData as $iRow => $aRow) + { + $sReconciliationCondition = "`primary_key` = ".CMDBSource::Quote($aRow[$iPrimaryKeyCol]); + $sSelect = "SELECT COUNT(*) FROM `$sTable` WHERE $sReconciliationCondition"; + $aRes = CMDBSource::QueryToArray($sSelect); + $iCount = $aRes[0]['COUNT(*)']; + + if ($iCount == 0) + { + // No record... create it + // + $iCountCreations++; + if ($sOutput == 'details') + { + $oP->add("$iRow: New entry, reconciliation: '$sReconciliationCondition'\n"); + } + + $aValues = array(); // Used to build the insert query + foreach ($aRow as $iCol => $value) + { + $aValues[] = CMDBSource::Quote($value); + } + $sValues = implode(', ', $aValues); + $sInsert = "INSERT INTO `$sTable` ($sInsertColumns) VALUES ($sValues)"; + if ($bSimulate) + { + if ($sOutput == 'details') + { + $oP->add("$iRow: SIMULATE - Planned query: $sInsert'\n"); + } + } + else + { + CMDBSource::Query($sInsert); + } + } + elseif ($iCount == 1) + { + // Found a match... update it + // + $iCountUpdates++; + if ($sOutput == 'details') + { + $oP->add("$iRow: Update entry, reconciliation: '$sReconciliationCondition'\n"); + } + + $aValuePairs = array(); // Used to build the update query + foreach ($aRow as $iCol => $value) + { + // Skip reconciliation column + if ($iCol == $iPrimaryKeyCol) continue; + + $sCol = $aInputColumns[$iCol]; + $aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]); + } + $sValuePairs = implode(', ', $aValuePairs); + $sUpdateQuery = "UPDATE `$sTable` SET $sValuePairs WHERE $sReconciliationCondition"; + if ($bSimulate) + { + if ($sOutput == 'details') + { + $oP->add("$iRow: SIMULATE - Planned query: $sUpdateQuery'\n"); + } + } + else + { + CMDBSource::Query($sUpdateQuery); + } + } + else + { + // Too many records... ambiguity + // + $iCountErrors++; + $oP->add("$iRow: Error - Failed to reconcile, found $iCount rows having '$sReconciliationCondition'\n"); + } + } + + ////////////////////////////////////////////////// + // + // Summary of settings and results + // + if ($sOutput == 'retcode') + { + $oP->add($iCountErrors); + } + + if (($sOutput == "summary") || ($sOutput == 'details')) + { + $oP->add_comment("Class: ".$sClass); + $oP->add_comment("Separator: ".$sSep); + $oP->add_comment("Qualifier: ".$sQualifier); + $oP->add_comment("Charset Encoding:".$sCharSet); + $oP->add_comment("Data Size: ".strlen($sCSVData)); + $oP->add_comment("Data Lines: ".$iLineCount); + $oP->add_comment("Columns: ".implode(', ', $aInputColumns)); + $oP->add_comment("Output format: ".$sOutput); +// $oP->add_comment("Report level: ".$sReportLevel); + $oP->add_comment("Simulate: ".($bSimulate ? '1' : '0')); + $oP->add_comment("Change tracking comment: ".$sComment); + $oP->add_comment("Issues: ".$iCountErrors); +// $oP->add_comment("Warnings: ".$iCountWarnings); + $oP->add_comment("Created: ".$iCountCreations); + $oP->add_comment("Updated: ".$iCountUpdates); + } +} +catch(ExchangeException $e) +{ + $oP->add_comment($e->getMessage()); +} +catch(SecurityException $e) +{ + $oP->add_comment($e->getMessage()); +} +catch(Exception $e) +{ + $oP->add_comment((string)$e); +} + +$oP->output(); +?>