composer/iTop
6
0
Fork 0
mirror of https://github.com/Combodo/iTop.git synced 2026-04-24 11:08:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

New developer tool: set log_queries=1 to enable query logging into data/

Browse Source 
The accumulative log data/queries.log can be replayed with test/replay_query_log.php which produces a result file (to check the stability of the results) and a benchmark file (to see the efficiency in CSV)

SVN:trunk[2469]
This commit is contained in:
Romain Quetiez
2012-11-27 16:53:24 +00:00
parent 51af2e9662
commit 9b982df4f0
4 changed files with 338 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/config.class.inc.php
View File

@@ -49,7 +49,7 @@ define ('DEFAULT_LOG_ISSUE', true);
define ('DEFAULT_LOG_WEB_SERVICE', true);
define ('DEFAULT_LOG_KPI_DURATION', false);
define ('DEFAULT_LOG_KPI_MEMORY', false);
define ('DEFAULT_DEBUG_QUERIES', false);
define ('DEFAULT_LOG_QUERIES', false);
define ('DEFAULT_QUERY_CACHE_ENABLED', true);
@@ -641,7 +641,7 @@ class Config
protected $m_bLogWebService;
protected $m_bLogKPIDuration; // private setting
protected $m_bLogKPIMemory; // private setting
protected $m_bDebugQueries; // private setting
protected $m_bLogQueries; // private setting
protected $m_bQueryCacheEnabled; // private setting
/**
@@ -888,7 +888,7 @@ class Config
$this->m_bLogWebService = isset($MySettings['log_web_service']) ? (bool) trim($MySettings['log_web_service']) : DEFAULT_LOG_WEB_SERVICE;
$this->m_bLogKPIDuration = isset($MySettings['log_kpi_duration']) ? (bool) trim($MySettings['log_kpi_duration']) : DEFAULT_LOG_KPI_DURATION;
$this->m_bLogKPIMemory = isset($MySettings['log_kpi_memory']) ? (bool) trim($MySettings['log_kpi_memory']) : DEFAULT_LOG_KPI_MEMORY;
$this->m_bDebugQueries = isset($MySettings['debug_queries']) ? (bool) trim($MySettings['debug_queries']) : DEFAULT_DEBUG_QUERIES;
$this->m_bLogQueries = isset($MySettings['log_queries']) ? (bool) trim($MySettings['log_queries']) : DEFAULT_LOG_QUERIES;
$this->m_bQueryCacheEnabled = isset($MySettings['query_cache_enabled']) ? (bool) trim($MySettings['query_cache_enabled']) : DEFAULT_QUERY_CACHE_ENABLED;
$this->m_iMinDisplayLimit = isset($MySettings['min_display_limit']) ? trim($MySettings['min_display_limit']) : DEFAULT_MIN_DISPLAY_LIMIT;
@@ -1036,9 +1036,9 @@ class Config
return $this->m_bLogKPIMemory;
}
public function GetDebugQueries()
public function GetLogQueries()
{
return $this->m_bDebugQueries;
return $this->m_bLogQueries;
}
public function GetQueryCacheEnabled()

116
core/metamodel.class.php
View File

@@ -2090,6 +2090,7 @@ abstract class MetaModel
$e->addInfo('OQL', $oFilter->ToOQL());
throw $e;
}
self::AddQueryTraceGroupBy($oFilter, $aArgs, $aGroupByExpr, $sRes);
return $sRes;
}
@@ -2138,7 +2139,8 @@ abstract class MetaModel
$aScalarArgs = array_merge(self::PrepareQueryArguments($aArgs), $oFilter->GetInternalParams());
try
{
$sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount);
$bBeautifulSQL = self::$m_bTraceQueries || self::$m_bDebugQuery;
$sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount, $bBeautifulSQL);
if ($sClassAlias == 'itop')
{
echo $sRes."<br/>\n";
@@ -2147,9 +2149,10 @@ abstract class MetaModel
catch (MissingQueryArgument $e)
{
// Add some information...
$e->addInfo('OQL', $sOqlQuery);
$e->addInfo('OQL', $oFilter->ToOQL());
throw $e;
}
self::AddQueryTraceSelect($oFilter, $aOrderBy, $aArgs, $aAttToLoad, $aExtendedDataSpec, $iLimitCount, $iLimitStart, $bGetCount, $sRes);
return $sRes;
}
@@ -2306,30 +2309,66 @@ abstract class MetaModel
$oSelect->AddInnerJoin($oSelectExt, 'id', $aExtendedDataSpec['join_key'] /*, $sTableAlias*/);
}
return $oSelect;
}
protected static function AddQueryTraceSelect($oFilter, $aOrderBy, $aArgs, $aAttToLoad, $aExtendedDataSpec, $iLimitCount, $iLimitStart, $bGetCount, $sSql)
{
$aQueryData = array(
'type' => 'select',
'filter' => $oFilter,
'order_by' => $aOrderBy,
'args' => $aArgs,
'att_to_load' => $aAttToLoad,
'extended_data_spec' => $aExtendedDataSpec,
'limit_count' => $iLimitCount,
'limit_start' => $iLimitStart,
'is_count' => $bGetCount
);
$sOql = $oFilter->ToOQL(true, $aArgs);
self::AddQueryTrace($aQueryData, $sOql, $sSql);
}
protected static function AddQueryTraceGroupBy($oFilter, $aArgs, $aGroupByExpr, $sSql)
{
$aQueryData = array(
'type' => 'group_by',
'filter' => $oFilter,
'args' => $aArgs,
'group_by_expr' => $aGroupByExpr
);
$sOql = $oFilter->ToOQL(true, $aArgs);
self::AddQueryTrace($aQueryData, $sOql, $sSql);
}
protected static function AddQueryTrace($aQueryData, $sOql, $sSql)
{
if (self::$m_bTraceQueries)
{
$sQueryId = md5($sRes);
if(!isset(self::$m_aQueriesLog[$sOqlId]))
$sQueryId = md5(serialize($aQueryData));
$sMySQLQueryId = md5($sSql);
if(!isset(self::$m_aQueriesLog[$sQueryId]))
{
self::$m_aQueriesLog[$sOqlId]['oql'] = $sOqlQuery;
self::$m_aQueriesLog[$sOqlId]['hits'] = 1;
self::$m_aQueriesLog[$sQueryId]['data'] = serialize($aQueryData);
self::$m_aQueriesLog[$sQueryId]['oql'] = $sOql;
self::$m_aQueriesLog[$sQueryId]['hits'] = 1;
}
else
{
self::$m_aQueriesLog[$sOqlId]['hits']++;
self::$m_aQueriesLog[$sQueryId]['hits']++;
}
if(!isset(self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]))
if(!isset(self::$m_aQueriesLog[$sQueryId]['queries'][$sMySQLQueryId]))
{
self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['sql'] = $sRes;
self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count'] = 1;
self::$m_aQueriesLog[$sQueryId]['queries'][$sMySQLQueryId]['sql'] = $sSql;
self::$m_aQueriesLog[$sQueryId]['queries'][$sMySQLQueryId]['count'] = 1;
$iTableCount = count(CMDBSource::ExplainQuery($sSql));
self::$m_aQueriesLog[$sQueryId]['queries'][$sMySQLQueryId]['table_count'] = $iTableCount;
}
else
{
self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count']++;
self::$m_aQueriesLog[$sQueryId]['queries'][$sMySQLQueryId]['count']++;
}
}
return $oSelect;
}
public static function ShowQueryTrace()
@@ -2337,34 +2376,55 @@ abstract class MetaModel
if (!self::$m_bTraceQueries) return;
$iOqlCount = count(self::$m_aQueriesLog);
if ($iOqlCount == 0)
{
echo "<p>Trace activated, but no query found</p>\n";
return;
}
$iSqlCount = 0;
foreach (self::$m_aQueriesLog as $aOqlData)
foreach (self::$m_aQueriesLog as $sQueryId => $aOqlData)
{
$iSqlCount += $aOqlData['hits'];
}
echo "<h2>Stats on SELECT queries: OQL=$iOqlCount, SQL=$iSqlCount</h2>\n";
foreach (self::$m_aQueriesLog as $aOqlData)
$sHtml = "<h2>Stats on SELECT queries: OQL=$iOqlCount, SQL=$iSqlCount</h2>\n";
foreach (self::$m_aQueriesLog as $sQueryId => $aOqlData)
{
$sOql = $aOqlData['oql'];
$sHits = $aOqlData['hits'];
echo "<p><b>$sHits</b> hits for OQL query: $sOql</p>\n";
echo "<ul id=\"ClassesRelationships\" class=\"treeview\">\n";
$sHtml .= "<p><b>$sHits</b> hits for OQL query: $sOql</p>\n";
$sHtml .= "<ul id=\"ClassesRelationships\" class=\"treeview\">\n";
foreach($aOqlData['queries'] as $aSqlData)
{
$sQuery = $aSqlData['sql'];
$sSqlHits = $aSqlData['count'];
echo "<li><b>$sSqlHits</b> hits for SQL: <span style=\"font-size:60%\">$sQuery</span><li>\n";
$iTableCount = $aSqlData['table_count'];
$sHtml .= "<li><b>$sSqlHits</b> hits for SQL ($iTableCount tables): <pre style=\"font-size:60%\">$sQuery</pre></li>\n";
}
echo "</ul>\n";
$sHtml .= "</ul>\n";
}
$sLogFile = 'queries.latest';
file_put_contents(APPROOT.'data/'.$sLogFile.'.html', $sHtml);
$sLog = "<?php\n\$aQueriesLog = ".var_export(self::$m_aQueriesLog, true).";";
file_put_contents(APPROOT.'data/'.$sLogFile.'.log', $sLog);
// Cumulate the queries
$sAllQueries = APPROOT.'data/queries.log';
if (file_exists($sAllQueries))
{
// Merge the new queries into the existing log
include($sAllQueries);
foreach (self::$m_aQueriesLog as $sQueryId => $aOqlData)
{
if (!array_key_exists($sQueryId, $aQueriesLog))
{
$aQueriesLog[$sQueryId] = $aOqlData;
}
}
}
else
{
$aQueriesLog = self::$m_aQueriesLog;
}
$sLog = "<?php\n\$aQueriesLog = ".var_export($aQueriesLog, true).";";
file_put_contents($sAllQueries, $sLog);
}
protected static function MakeModifierProperties($oFilter)
@@ -4430,7 +4490,7 @@ abstract class MetaModel
ExecutionKPI::EnableMemory();
}
self::$m_bTraceQueries = self::$m_oConfig->GetDebugQueries();
self::$m_bTraceQueries = self::$m_oConfig->GetLogQueries();
self::$m_bQueryCacheEnabled = self::$m_oConfig->GetQueryCacheEnabled();
self::$m_bSkipCheckToWrite = self::$m_oConfig->Get('skip_check_to_write');

37
core/sqlquery.class.inc.php
View File

@@ -48,6 +48,7 @@ class SQLQuery
private $m_aValues = array(); // Values to set in case of an update query
private $m_oSelectedIdField = null;
private $m_aJoinSelects = array();
private $m_bBeautifulQuery = false;
public function __construct($sTable, $sTableAlias, $aFields, $bToDelete = true, $aValues = array(), $oSelectedIdField = null)
{
@@ -293,8 +294,11 @@ class SQLQuery
}
// Interface, build the SQL query
public function RenderSelect($aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
public function RenderSelect($aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false, $bBeautifulQuery = false)
{
$this->m_bBeautifulQuery = $bBeautifulQuery;
$sLineSep = $this->m_bBeautifulQuery ? "\n" : '';
// The goal will be to complete the lists as we build the Joins
$aFrom = array();
$aFields = array();
@@ -305,7 +309,9 @@ class SQLQuery
$aSelectedIdFields = array();
$this->privRender($aFrom, $aFields, $aGroupBy, $oCondition, $aDelTables, $aSetValues, $aSelectedIdFields);
$sFrom = self::ClauseFrom($aFrom);
$sIndent = $this->m_bBeautifulQuery ? " " : null;
$sFrom = self::ClauseFrom($aFrom, $sIndent);
$sWhere = self::ClauseWhere($oCondition, $aArgs);
if ($bGetCount)
{
@@ -317,11 +323,11 @@ class SQLQuery
$aCountFields[] = "COALESCE($sFieldExpr, 0)"; // Null values are excluded from the count
}
$sCountFields = implode(', ', $aCountFields);
$sSQL = "SELECT COUNT(DISTINCT $sCountFields) AS COUNT FROM $sFrom WHERE $sWhere";
$sSQL = "SELECT$sLineSep COUNT(DISTINCT $sCountFields) AS COUNT$sLineSep FROM $sFrom$sLineSep WHERE $sWhere";
}
else
{
$sSQL = "SELECT COUNT(*) AS COUNT FROM $sFrom WHERE $sWhere";
$sSQL = "SELECT$sLineSep COUNT(*) AS COUNT$sLineSep FROM $sFrom$sLineSep WHERE $sWhere";
}
}
else
@@ -340,7 +346,7 @@ class SQLQuery
{
$sLimit = '';
}
$sSQL = "SELECT DISTINCT $sSelect FROM $sFrom WHERE $sWhere $sOrderBy $sLimit";
$sSQL = "SELECT$sLineSep DISTINCT $sSelect$sLineSep FROM $sFrom$sLineSep WHERE $sWhere$sLineSep $sOrderBy$sLineSep $sLimit";
}
return $sSQL;
}
@@ -394,27 +400,30 @@ class SQLQuery
return $sDelTables;
}
private static function ClauseFrom($aFrom)
private static function ClauseFrom($aFrom, $sIndent = null, $iIndentLevel = 0)
{
$sLineBreakLong = $sIndent ? "\n".str_repeat($sIndent, $iIndentLevel + 1) : '';
$sLineBreak = $sIndent ? "\n".str_repeat($sIndent, $iIndentLevel) : '';
$sFrom = "";
foreach ($aFrom as $sTableAlias => $aJoinInfo)
{
switch ($aJoinInfo["jointype"])
{
case "first":
$sFrom .= "`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= " ".self::ClauseFrom($aJoinInfo["subfrom"]);
$sFrom .= $sLineBreakLong."`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= self::ClauseFrom($aJoinInfo["subfrom"], $sIndent, $iIndentLevel + 1);
break;
case "inner":
case "inner_tree":
$sFrom .= " INNER JOIN (`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= " ".self::ClauseFrom($aJoinInfo["subfrom"]);
$sFrom .= ") ON ".$aJoinInfo["joincondition"];
$sFrom .= $sLineBreak."INNER JOIN ($sLineBreakLong`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= " ".self::ClauseFrom($aJoinInfo["subfrom"], $sIndent, $iIndentLevel + 1);
$sFrom .= $sLineBreak.") ON ".$aJoinInfo["joincondition"];
break;
case "left":
$sFrom .= " LEFT JOIN (`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= " ".self::ClauseFrom($aJoinInfo["subfrom"]);
$sFrom .= ") ON ".$aJoinInfo["joincondition"];
$sFrom .= $sLineBreak."LEFT JOIN ($sLineBreakLong`".$aJoinInfo["tablename"]."` AS `$sTableAlias`";
$sFrom .= " ".self::ClauseFrom($aJoinInfo["subfrom"], $sIndent, $iIndentLevel + 1);
$sFrom .= $sLineBreak.") ON ".$aJoinInfo["joincondition"];
break;
default:
throw new CoreException("Unknown jointype: '".$aJoinInfo["jointype"]."'");