N°3452 - DB tools : better formatting

This commit is contained in:
Eric
2021-05-26 12:14:14 +02:00
parent 44952d1ea0
commit cf12578289
6 changed files with 134 additions and 114 deletions

View File

@@ -1004,10 +1004,8 @@ abstract class DBSearch
}
/**
* Generate a SQL query from the current search
*
* @internal
*
* Generate a SQL query from the current search
*
* @param array $aOrderBy Array of '[<classalias>.]attcode' => bAscending
* @param array $aArgs
* @param null $aAttToLoad
@@ -1015,12 +1013,16 @@ abstract class DBSearch
* @param int $iLimitCount
* @param int $iLimitStart
* @param bool $bGetCount
* @param bool $bBeautifulSQL
*
* @return string
* @throws CoreException
* @throws Exception
* @throws MissingQueryArgument
* @throws \ConfigException
* @throws \CoreException
* @throws \MissingQueryArgument
* @internal
*
*/
public function MakeSelectQuery($aOrderBy = array(), $aArgs = array(), $aAttToLoad = null, $aExtendedDataSpec = null, $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
public function MakeSelectQuery($aOrderBy = array(), $aArgs = array(), $aAttToLoad = null, $aExtendedDataSpec = null, $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false, $bBeautifulSQL = true)
{
// Check the order by specification, and prefix with the class alias
// and make sure that the ordering columns are going to be selected
@@ -1085,8 +1087,7 @@ abstract class DBSearch
}
try
{
// $bBeautifulSQL = self::$m_bTraceQueries || self::$m_bDebugQuery || self::$m_bIndentQueries;
$sRes = $oSQLQuery->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount, true);
$sRes = $oSQLQuery->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount, $bBeautifulSQL);
if ($sClassAlias == '_itop_')
{
IssueLog::Info('SQL Query (_itop_): '.$sRes);

View File

@@ -222,13 +222,11 @@ class DatabaseAnalyzer
{
$oFixSearch->AddCondition($sAttCode, $sValue, '=');
}
$aFixit[] = $oFixSearch->MakeSelectQuery().';';
$aFixit[] = $oFixSearch->MakeSelectQuery([], [], null, null, 0, 0, false, false).';';
$aFixit[] = "";
}
$aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'] = $aFixit;
}
return;
}
private function GetUniquenessRuleMessage($sUniquenessRuleId)
@@ -326,13 +324,8 @@ class DatabaseAnalyzer
{
$sField = MetaModel::DBGetClassField($sClass);
$sRootField = MetaModel::DBGetClassField($sRootClass);
$sSelWrongRecs = <<<SQL
SELECT `$sTable`.`$sKeyField` AS id
FROM `$sTable`
JOIN `$sRootTable` ON `$sRootTable`.`$sRootKey` = `$sTable`.`$sKeyField`
WHERE `$sTable`.`$sField` != `$sRootTable`.`$sRootField`
SQL;
// Copy the finalclass of the root table
$sSelWrongRecs = "SELECT `$sTable`.`$sKeyField` AS id FROM `$sTable` JOIN `$sRootTable` ON `$sRootTable`.`$sRootKey` = `$sTable`.`$sKeyField` WHERE `$sTable`.`$sField` != `$sRootTable`.`$sRootField`";
// Copy the final class of the root table
$sFixItRequest = "UPDATE `$sTable`,`$sRootTable` SET `$sTable`.`$sField` = `$sRootTable`.`$sRootField` WHERE `$sTable`.`$sKeyField` = `$sRootTable`.`$sRootKey`";
$this->ExecQuery($sSelWrongRecs, $sFixItRequest, Dict::Format('DBAnalyzer-Integrity-FinalClass', $sField, $sTable, $sRootTable), $sClass, $aErrorsAndFixes);
}
@@ -360,65 +353,79 @@ SQL;
// Note: a class/table may have an external key on itself
$sSelect = "SELECT DISTINCT `$sTable`.`$sKeyField` AS id, `$sTable`.`$sExtKeyField` AS value";
$sFilter = "FROM `$sTable` LEFT JOIN `$sRemoteTable` AS `{$sRemoteTable}_1` ON `$sTable`.`$sExtKeyField` = `{$sRemoteTable}_1`.`$sRemoteKey`";
$sFrom = "FROM `$sTable`";
$sJoin = "LEFT JOIN `$sRemoteTable` AS `{$sRemoteTable}_1` ON `$sTable`.`$sExtKeyField` = `{$sRemoteTable}_1`.`$sRemoteKey`";
$sFilter = $sFilter." WHERE `{$sRemoteTable}_1`.`$sRemoteKey` IS NULL";
$sFilter = "WHERE `{$sRemoteTable}_1`.`$sRemoteKey` IS NULL";
// Exclude the records pointing to 0/null from the errors (separate test below)
$sFilter .= " AND `$sTable`.`$sExtKeyField` IS NOT NULL";
$sFilter .= " AND `$sTable`.`$sExtKeyField` != 0";
$sSelWrongRecs = "$sSelect $sFilter";
$sSelWrongRecs = "$sSelect $sFrom $sJoin $sFilter";
$sErrorDesc = Dict::Format('DBAnalyzer-Integrity-InvalidExtKey', $sAttCode, $sTable, $sExtKeyField);
$this->ExecQuery($sSelWrongRecs, '', $sErrorDesc, $sClass, $aErrorsAndFixes);
$aFixIt = [];
// Fix it request needs the values of the enum to generate the requests
if (isset($aErrorsAndFixes[$sClass][$sErrorDesc]['values']))
{
$aFixIt = array();
$aFixIt[] = "-- Remove inconsistant entries:";
$sIds = implode(', ', array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']));
$aFixIt[] = "DELETE `$sTable` FROM `$sTable` WHERE `$sTable`.`$sExtKeyField` IN ($sIds)";
$aFixIt[] = "";
$aFixIt[] = "-- Or fix inconsistant values: Replace XXX with the appropriate value";
foreach (array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']) as $sKey)
{
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sExtKeyField` = XXX WHERE `$sTable`.`$sExtKeyField` = '$sKey'";
}
$aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'] = $aFixIt;
}
if (!$oAttDef->IsNullAllowed())
{
$sSelect = "SELECT DISTINCT `$sTable`.`$sKeyField` AS id";
$sDelete = "DELETE `$sTable`";
$sFilter = "FROM `$sTable` WHERE `$sTable`.`$sExtKeyField` IS NULL OR `$sTable`.`$sExtKeyField` = 0";
$sSelWrongRecs = "$sSelect $sFilter";
$sFixItRequest = "$sDelete $sFilter";
$sErrorDesc = Dict::Format('DBAnalyzer-Integrity-MissingExtKey', $sAttCode, $sTable, $sExtKeyField);
$this->ExecQuery($sSelWrongRecs, $sFixItRequest, $sErrorDesc, $sClass, $aErrorsAndFixes);
if (isset($aErrorsAndFixes[$sClass][$sErrorDesc]['count']) && ($aErrorsAndFixes[$sClass][$sErrorDesc]['count'] > 0))
{
$aFixIt = $aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'];
$aFixIt[] = "-- Alternate fix";
$aFixIt[] = "-- Replace XXX with the appropriate value";
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sExtKeyField` = XXX WHERE `$sTable`.`$sExtKeyField` IS NULL OR `$sTable`.`$sExtKeyField` = 0";
$aAdditionalFixIt = $this->GetSpecificExternalKeysFixItForNull($sTable, $sExtKeyField);
if ($oAttDef->IsNullAllowed()) {
$aFixIt[] = "-- Fix inconsistant values: remove the external key";
foreach (array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']) as $sKey) {
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sExtKeyField` = 0 WHERE `$sTable`.`$sExtKeyField` = '$sKey'";
}
} else {
$aAdditionalFixIt = $this->GetSpecificExternalKeysFixItForNull($sTable, $sExtKeyField, $sFilter, $sJoin);
foreach ($aAdditionalFixIt as $sFixIt)
{
$aFixIt[] = $sFixIt;
}
$aFixIt[] = "-- Alternate fix: remove inconsistant entries:";
$sIds = implode(', ', array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']));
$aFixIt[] = "DELETE `$sTable` FROM `$sTable` WHERE `$sTable`.`$sExtKeyField` IN ($sIds)";
$aFixIt[] = "-- Alternate fix: update inconsistant values: Replace XXX with the appropriate value";
foreach (array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']) as $sKey) {
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sExtKeyField` = XXX WHERE `$sTable`.`$sExtKeyField` = '$sKey'";
}
}
$aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'] = $aFixIt;
}
if (!$oAttDef->IsNullAllowed()) {
$sSelect = "SELECT DISTINCT `$sTable`.`$sKeyField` AS id";
$sDelete = "DELETE `$sTable`";
$sFrom = "FROM `$sTable`";
$sFilter = "WHERE `$sTable`.`$sExtKeyField` IS NULL OR `$sTable`.`$sExtKeyField` = 0";
$sSelWrongRecs = "$sSelect $sFrom $sFilter";
$sFixItRequest = "$sDelete $sFrom $sFilter";
$sErrorDesc = Dict::Format('DBAnalyzer-Integrity-MissingExtKey', $sAttCode, $sTable, $sExtKeyField);
$this->ExecQuery($sSelWrongRecs, '', $sErrorDesc, $sClass, $aErrorsAndFixes);
$aFixIt = [];
if (isset($aErrorsAndFixes[$sClass][$sErrorDesc]['count']) && ($aErrorsAndFixes[$sClass][$sErrorDesc]['count'] > 0))
{
$aAdditionalFixIt = $this->GetSpecificExternalKeysFixItForNull($sTable, $sExtKeyField, $sFilter);
foreach ($aAdditionalFixIt as $sFixIt)
{
$aFixIt[] = $sFixIt;
}
$aFixIt[] = "-- Alternate fix: remove inconsistant entries:";
$aFixIt[] = $sFixItRequest;
$aFixIt[] = "-- Alternate fix: replace XXX with the appropriate value";
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sExtKeyField` = XXX WHERE `$sTable`.`$sExtKeyField` IS NULL OR `$sTable`.`$sExtKeyField` = 0";
$aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'] = $aFixIt;
}
}
}
private function GetSpecificExternalKeysFixItForNull($sTable, $sExtKeyField)
private function GetSpecificExternalKeysFixItForNull($sTable, $sExtKeyField, $sFilter, $sJoin = '')
{
$aFixIt = array();
if ($sTable == 'ticket' && $sExtKeyField == 'org_id')
{
$aFixIt[] = "-- Alternate fix: set the ticket org to the caller org";
$aFixIt[] = "UPDATE ticket AS t JOIN contact AS c ON t.caller_id=c.id SET t.org_id=c.org_id WHERE t.org_id IS NULL OR t.org_id = 0";
$aFixIt[] = "UPDATE ticket JOIN contact AS c ON ticket.caller_id=c.id $sJoin SET ticket.org_id=c.org_id $sFilter";
}
return $aFixIt;
}
@@ -441,7 +448,8 @@ SQL;
$aAllowedValues = MetaModel::GetAllowedValues_att($sClass, $sAttCode);
if (!is_null($aAllowedValues) && count($aAllowedValues) > 0)
{
$sExpectedValues = implode(",", CMDBSource::Quote(array_keys($aAllowedValues), true));
$aAllowedValues = array_keys($aAllowedValues);
$sExpectedValues = implode(",", CMDBSource::Quote($aAllowedValues, true));
$aCols = $oAttDef->GetSQLExpressions(); // Workaround a PHP bug: sometimes issuing a Notice if invoking current(somefunc())
$sMyAttributeField = current($aCols); // get the first column for the moment
@@ -458,15 +466,18 @@ SQL;
if (isset($aErrorsAndFixes[$sClass][$sErrorDesc]['fixit']))
{
$aFixIt = $aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'];
$aFixIt[] = "-- Alternative: Replace 'XXX' with the appropriate value";
$aFixIt[] = "-- Alternative: Replace enums with the appropriate value";
}
else
{
$aFixIt = array("-- Replace 'XXX' with the appropriate value");
$aFixIt = ["-- Replace enums with the appropriate value"];
}
foreach (array_keys($aErrorsAndFixes[$sClass][$sErrorDesc]['values']) as $sKey)
{
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sMyAttributeField` = 'XXX' WHERE `$sTable`.`$sMyAttributeField` = '$sKey'";
foreach ($aAllowedValues as $sAllowedValue) {
$aFixIt[] = "-- Replace $sKey by $sAllowedValue";
$aFixIt[] = "UPDATE `$sTable` SET `$sTable`.`$sMyAttributeField` = '$sAllowedValue' WHERE `$sTable`.`$sMyAttributeField` = '$sKey'";
}
}
$aErrorsAndFixes[$sClass][$sErrorDesc]['fixit'] = $aFixIt;
}

View File

@@ -31,15 +31,16 @@ const MAX_RESULTS = 10;
* @param ApplicationContext $oAppContext
*
* @return \iTopWebPage
* @throws CoreException
* @throws DictExceptionMissingString
* @throws MySQLException
* @throws \CoreException
* @throws \DictExceptionMissingString
* @throws \MySQLException
*/
function DisplayDBInconsistencies(iTopWebPage &$oP, ApplicationContext &$oAppContext)
{
$iShowId = intval(utils::ReadParam('show_id', '0'));
$sErrorLabelSelection = utils::ReadParam('error_selection', '');
$sClassSelection = utils::ReadParam('class_selection', '');
$bVerbose = utils::ReadParam('verbose', 0);
if (!empty($sClassSelection))
{
$aClassSelection = explode(",", $sClassSelection);
@@ -109,11 +110,16 @@ function DisplayDBInconsistencies(iTopWebPage &$oP, ApplicationContext &$oAppCon
if ($iShowId == 3)
{
DisplayInconsistenciesReport($aResults);
DisplayInconsistenciesReport($aResults, $bVerbose);
}
$oP->p(Dict::S('DBTools:ErrorsFound'));
if ($iShowId > 0) {
$oP->p(Dict::S('DBTools:Disclaimer'));
$oP->p(Dict::S('DBTools:Indication'));
}
$oP->add('<table class="listResults"><tr><th>'.Dict::S('DBTools:Class').'</th><th>'.Dict::S('DBTools:Count').'</th><th>'.Dict::S('DBTools:Error').'</th></tr>');
$bTable = true;
foreach($aResults as $sClass => $aErrorList)
@@ -148,7 +154,7 @@ function DisplayDBInconsistencies(iTopWebPage &$oP, ApplicationContext &$oAppCon
$oP->p(Dict::S('DBTools:SQLquery'));
$sQuery = $aError['query'];
$oP->add('<div style="padding: 15px; background: #f1f1f1;">');
$oP->add('<code>'.$sQuery.'</code>');
$oP->add('<pre>'.$sQuery.'</pre>');
$oP->add('</div>');
if (isset($aError['fixit']))
@@ -163,27 +169,26 @@ function DisplayDBInconsistencies(iTopWebPage &$oP, ApplicationContext &$oAppCon
$oP->add('<br></div>');
}
$oP->p(Dict::S('DBTools:SQLresult'));
$sQueryResult = '';
$iCount = count($aError['res']);
$iMaxCount = MAX_RESULTS;
foreach($aError['res'] as $aRes)
{
$iMaxCount--;
if ($iMaxCount < 0)
{
$sQueryResult .= 'Displayed '.MAX_RESULTS."/$iCount results.<br>";
break;
if ($bVerbose) {
$oP->p(Dict::S('DBTools:SQLresult'));
$sQueryResult = '';
$iCount = count($aError['res']);
$iMaxCount = MAX_RESULTS;
foreach ($aError['res'] as $aRes) {
$iMaxCount--;
if ($iMaxCount < 0) {
$sQueryResult .= 'Displayed '.MAX_RESULTS."/$iCount results.<br>";
break;
}
foreach ($aRes as $sKey => $sValue) {
$sQueryResult .= "'$sKey'='$sValue'&nbsp;";
}
$sQueryResult .= '<br>';
}
foreach($aRes as $sKey => $sValue)
{
$sQueryResult .= "'$sKey'='$sValue'&nbsp;";
}
$sQueryResult .= '<br>';
$oP->add('<div style="padding: 15px; background: #f1f1f1;">');
$oP->add('<pre>'.$sQueryResult.'</pre>');
$oP->add('</div>');
}
$oP->add('<div style="padding: 15px; background: #f1f1f1;">');
$oP->add('<code>'.$sQueryResult.'</code>');
$oP->add('</div>');
}
}
}
@@ -194,14 +199,15 @@ function DisplayDBInconsistencies(iTopWebPage &$oP, ApplicationContext &$oAppCon
/**
* @param $aResults
* @param bool $bVerbose
*
* @return mixed
* @throws CoreException
* @throws DictExceptionMissingString
* @throws \CoreException
* @throws \DictExceptionMissingString
*/
function DisplayInconsistenciesReport($aResults)
function DisplayInconsistenciesReport($aResults, $bVerbose = false)
{
$sReportFile = DBAnalyzerUtils::GenerateReport($aResults);
$sReportFile = DBAnalyzerUtils::GenerateReport($aResults, $bVerbose);
$sZipReport = "{$sReportFile}.zip";
$oArchive = new ZipArchive();

View File

@@ -28,6 +28,8 @@ Dict::Add('EN US', 'English', 'English', array(
'DBTools:Class' => 'Class',
'DBTools:Title' => 'Database Maintenance Tools',
'DBTools:ErrorsFound' => 'Errors Found',
'DBTools:Indication' => 'Important: after fixing errors in the database you\'ll have to run the analysis again as new inconsistencies will be generated',
'DBTools:Disclaimer' => 'DISCLAIMER: BACKUP YOUR DATABASE BEFORE RUNNING THE FIXES',
'DBTools:Error' => 'Error',
'DBTools:Count' => 'Count',
'DBTools:SQLquery' => 'SQL query',

View File

@@ -23,6 +23,8 @@ Dict::Add('FR FR', 'French', 'Français', array(
'DBTools:Class' => 'Classe',
'DBTools:Title' => 'Outils maintenance base de données',
'DBTools:ErrorsFound' => 'Erreurs trouvées',
'DBTools:Indication' => 'Important : après correction il est nécessaire de relancer l\'analyse car d\'autres inconsistances peuvent être générées par les modifications',
'DBTools:Disclaimer' => 'ATTENTION : EFFECTUEZ UNE SAUVEGARDE DE LA BASE AVANT D\'APPLIQUER LES CORRECTIONS',
'DBTools:Error' => 'Erreur',
'DBTools:Count' => 'Nombre',
'DBTools:SQLquery' => 'Requête SQL',
@@ -41,7 +43,7 @@ Dict::Add('FR FR', 'French', 'Français', array(
'DBTools:Inconsistencies' => 'Incohérences de base de données',
'DBAnalyzer-Integrity-OrphanRecord' => 'Enregistrement orphelin dans `%1$s`, il devrait avoir son équivalent dans la tableit `%2$s`',
'DBAnalyzer-Integrity-OrphanRecord' => 'Enregistrement orphelin dans `%1$s`, il devrait avoir son équivalent dans la table `%2$s`',
'DBAnalyzer-Integrity-InvalidExtKey' => 'Clef externe invalide %1$s (colonne: `%2$s.%3$s`)',
'DBAnalyzer-Integrity-MissingExtKey' => 'Clef externe manquante %1$s (colonne: `%2$s.%3$s`)',
'DBAnalyzer-Integrity-InvalidValue' => 'Valeur invalide pour %1$s (colonne: `%2$s.%3$s`)',

View File

@@ -8,6 +8,7 @@
namespace Combodo\iTop\DBTools\Service;
use CoreException;
use Dict;
use DictExceptionMissingString;
use MetaModel;
@@ -20,29 +21,31 @@ class DBAnalyzerUtils
* @throws CoreException
* @throws DictExceptionMissingString
*/
public static function GenerateReport($aResults)
public static function GenerateReport($aResults, $bVerbose = false)
{
$sDBToolsFolder = str_replace("\\", '/', APPROOT.'log/');
$sReportFile = 'dbtools-report';
$fReport = fopen($sDBToolsFolder.$sReportFile.'.log', 'w');
fwrite($fReport, 'Database Maintenance tools: '.date('Y-m-d H:i:s')."\r\n");
fwrite($fReport, '-- Database Maintenance tools: '.date('Y-m-d H:i:s')."\r\n");
fwrite($fReport, "-- ".Dict::S('DBTools:Disclaimer')."\r\n");
fwrite($fReport, "-- ".Dict::S('DBTools:Indication')."\r\n");
foreach ($aResults as $sClass => $aErrorList)
{
fwrite($fReport, '');
foreach ($aErrorList as $sErrorLabel => $aError)
{
fwrite($fReport, "\r\n----------\r\n");
fwrite($fReport, 'Class: '.MetaModel::GetName($sClass).' ('.$sClass.")\r\n");
fwrite($fReport, "\r\n-- \r\n");
fwrite($fReport, '-- Class: '.MetaModel::GetName($sClass).' ('.$sClass.")\r\n");
$iCount = $aError['count'];
fwrite($fReport, 'Count: '.$iCount."\r\n");
fwrite($fReport, 'Error: '.$sErrorLabel."\r\n");
fwrite($fReport, '-- Count: '.$iCount."\r\n");
fwrite($fReport, '-- Error: '.$sErrorLabel."\r\n");
$sQuery = $aError['query'];
fwrite($fReport, 'Query: '.$sQuery."\r\n");
fwrite($fReport, '-- Query: '.$sQuery."\r\n");
if (isset($aError['fixit']))
{
fwrite($fReport, "\r\nFix it (indication):\r\n\r\n");
fwrite($fReport, "\r\n-- Fix it (indication):\r\n\r\n");
$aFixitQueries = $aError['fixit'];
foreach ($aFixitQueries as $sFixitQuery)
{
@@ -51,31 +54,26 @@ class DBAnalyzerUtils
fwrite($fReport, "\r\n");
}
$sQueryResult = '';
$aIdList = array();
foreach ($aError['res'] as $aRes)
{
foreach ($aRes as $sKey => $sValue)
{
$sQueryResult .= "'$sKey'='$sValue' ";
if ($sKey == 'id')
{
$aIdList[] = $sValue;
if ($bVerbose) {
$sQueryResult = '';
$aIdList = [];
foreach ($aError['res'] as $aRes) {
$sQueryResult .= " - ";
foreach ($aRes as $sKey => $sValue) {
$sQueryResult .= "'$sKey'='$sValue' ";
if ($sKey == 'id') {
$aIdList[] = $sValue;
}
}
}
$sQueryResult .= "\r\n";
fwrite($fReport, "-- Result: ".$sQueryResult);
$sIdList = '('.implode(',', $aIdList).')';
fwrite($fReport, "\r\n-- Ids: ".$sIdList."\r\n");
}
fwrite($fReport, "Result: \r\n".$sQueryResult);
$sIdList = '('.implode(',', $aIdList).')';
fwrite($fReport, 'Ids: '.$sIdList."\r\n");
}
}
fclose($fReport);
$sReportFile = $sDBToolsFolder.$sReportFile;
return $sReportFile;
return $sDBToolsFolder.$sReportFile;
}
}