From a9c0ba63df068ce65e162aa046834cf5a0363b2e Mon Sep 17 00:00:00 2001 From: Romain Quetiez Date: Fri, 3 Sep 2010 21:23:37 +0000 Subject: [PATCH] Optimizations: DBObjectSet::Count does it without loading all the data + possibility to specify LIMIT + restored the query cache (was inoperant) + improved the debug reports + added two settings (query_cache_enabled and debug_queries) SVN:trunk[756] --- application/cmdbabstract.class.inc.php | 3 +- core/config.class.inc.php | 19 +++++ core/dbobjectset.class.php | 39 ++++++++-- core/kpi.class.inc.php | 38 +++++++--- core/metamodel.class.php | 100 +++++++++++++++++-------- core/sqlquery.class.inc.php | 28 +++++-- 6 files changed, 174 insertions(+), 53 deletions(-) diff --git a/application/cmdbabstract.class.inc.php b/application/cmdbabstract.class.inc.php index b9453ebf8..183ab29e7 100644 --- a/application/cmdbabstract.class.inc.php +++ b/application/cmdbabstract.class.inc.php @@ -438,7 +438,6 @@ abstract class cmdbAbstractObject extends CMDBObject $aAttribs[$sAttCode] = array('label' => MetaModel::GetLabel($sClassName, $sAttCode), 'description' => MetaModel::GetDescription($sClassName, $sAttCode)); } $aValues = array(); - $oSet->Seek(0); $bDisplayLimit = isset($aExtraParams['display_limit']) ? $aExtraParams['display_limit'] : true; $iMaxObjects = -1; if ($bDisplayLimit) @@ -446,8 +445,10 @@ abstract class cmdbAbstractObject extends CMDBObject if ($oSet->Count() > utils::GetConfig()->GetMaxDisplayLimit()) { $iMaxObjects = utils::GetConfig()->GetMinDisplayLimit(); + $oSet->SetLimit($iMaxObjects); } } + $oSet->Seek(0); while (($oObj = $oSet->Fetch()) && ($iMaxObjects != 0)) { $aRow = array(); diff --git a/core/config.class.inc.php b/core/config.class.inc.php index 09597a4dc..46733fcd2 100644 --- a/core/config.class.inc.php +++ b/core/config.class.inc.php @@ -42,6 +42,10 @@ define ('DEFAULT_LOG_ISSUE', true); define ('DEFAULT_LOG_WEB_SERVICE', true); define ('DEFAULT_LOG_KPI_DURATION', false); define ('DEFAULT_LOG_KPI_MEMORY', false); +define ('DEFAULT_DEBUG_QUERIES', false); + +define ('DEFAULT_QUERY_CACHE_ENABLED', true); + define ('DEFAULT_MIN_DISPLAY_LIMIT', 10); define ('DEFAULT_MAX_DISPLAY_LIMIT', 15); @@ -88,6 +92,8 @@ class Config protected $m_bLogWebService; protected $m_bLogKpiDuration; // private setting protected $m_bLogKpiMemory; // private setting + protected $m_bDebugQueries; // private setting + protected $m_bQueryCacheEnabled; // private setting /** * @var integer Number of elements to be displayed when there are more than m_iMaxDisplayLimit elements @@ -300,6 +306,9 @@ class Config $this->m_bLogWebService = isset($MySettings['log_web_service']) ? (bool) trim($MySettings['log_web_service']) : DEFAULT_LOG_WEB_SERVICE; $this->m_bLogKPIDuration = isset($MySettings['log_kpi_duration']) ? (bool) trim($MySettings['log_kpi_duration']) : DEFAULT_LOG_KPI_DURATION; $this->m_bLogKPIMemory = isset($MySettings['log_kpi_memory']) ? (bool) trim($MySettings['log_kpi_memory']) : DEFAULT_LOG_KPI_MEMORY; + $this->m_bDebugQueries = isset($MySettings['debug_queries']) ? (bool) trim($MySettings['debug_queries']) : DEFAULT_DEBUG_QUERIES; + $this->m_bQueryCacheEnabled = isset($MySettings['query_cache_enabled']) ? (bool) trim($MySettings['query_cache_enabled']) : DEFAULT_QUERY_CACHE_ENABLED; + $this->m_iMinDisplayLimit = isset($MySettings['min_display_limit']) ? trim($MySettings['min_display_limit']) : DEFAULT_MIN_DISPLAY_LIMIT; $this->m_iMaxDisplayLimit = isset($MySettings['max_display_limit']) ? trim($MySettings['max_display_limit']) : DEFAULT_MAX_DISPLAY_LIMIT; $this->m_iStandardReloadInterval = isset($MySettings['standard_reload_interval']) ? trim($MySettings['standard_reload_interval']) : DEFAULT_STANDARD_RELOAD_INTERVAL; @@ -451,6 +460,16 @@ class Config return $this->m_bLogKPIMemory; } + public function GetDebugQueries() + { + return $this->m_bDebugQueries; + } + + public function GetQueryCacheEnabled() + { + return $this->m_bQueryCacheEnabled; + } + public function GetMinDisplayLimit() { return $this->m_iMinDisplayLimit; diff --git a/core/dbobjectset.class.php b/core/dbobjectset.class.php index 077cc4be3..979826c5b 100644 --- a/core/dbobjectset.class.php +++ b/core/dbobjectset.class.php @@ -38,11 +38,13 @@ class DBObjectSet private $m_aId2Row; private $m_iCurrRow; - public function __construct(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array()) + public function __construct(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0) { $this->m_oFilter = $oFilter; $this->m_aOrderBy = $aOrderBy; $this->m_aArgs = $aArgs; + $this->m_iLimitCount = $iLimitCount; + $this->m_iLimitStart = $iLimitStart; $this->m_bLoaded = false; $this->m_aData = array(); // array of (row => array of (classalias) => object) @@ -194,11 +196,33 @@ class DBObjectSet return MetaModel::GetRootClass($this->GetClass()); } + public function SetLimit($iLimitCount, $iLimitStart = 0) + { + $this->m_iLimitCount = $iLimitCount; + $this->m_iLimitStart = $iLimitStart; + } + + public function GetLimitCount() + { + return $this->m_iLimitCount; + } + + public function GetLimitStart() + { + return $this->m_iLimitStart; + } + public function Load() { if ($this->m_bLoaded) return; - - $sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs); + if ($this->m_iLimitCount > 0) + { + $sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_iLimitCount, $this->m_iLimitStart); + } + else + { + $sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs); + } $resQuery = CMDBSource::Query($sSQL); if (!$resQuery) return; @@ -220,8 +244,13 @@ class DBObjectSet public function Count() { - if (!$this->m_bLoaded) $this->Load(); - return count($this->m_aData); + $sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, 0, 0, true); + $resQuery = CMDBSource::Query($sSQL); + if (!$resQuery) return 0; + + $aRow = CMDBSource::FetchArray($resQuery); + CMDBSource::FreeResult($resQuery); + return $aRow['COUNT']; } public function Fetch($sClassAlias = '') diff --git a/core/kpi.class.inc.php b/core/kpi.class.inc.php index ade2c6464..6822c94e9 100644 --- a/core/kpi.class.inc.php +++ b/core/kpi.class.inc.php @@ -47,34 +47,50 @@ class ExecutionKPI { foreach (self::$m_aStats as $sOperation => $aOpStats) { - echo "====================
\n"; - echo "KPIs for $sOperation
\n"; - echo "====================
\n"; + echo "

KPIs for $sOperation

\n"; $fTotalOp = 0; $iTotalOp = 0; $fMinOp = null; $fMaxOp = 0; + echo "\n"; + echo "\n"; } } diff --git a/core/metamodel.class.php b/core/metamodel.class.php index ab6b47854..be10e3331 100644 --- a/core/metamodel.class.php +++ b/core/metamodel.class.php @@ -202,7 +202,8 @@ abstract class MetaModel private static $m_oConfig = null; - private static $m_bTraceQueries = true; + private static $m_bQueryCacheEnabled = false; + private static $m_bTraceQueries = false; private static $m_aQueriesLog = array(); private static $m_bLogIssue = false; @@ -1539,7 +1540,7 @@ abstract class MetaModel return $aScalarArgs; } - public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array()) + public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false) { // Hide objects that are not visible to the current user // @@ -1562,12 +1563,23 @@ abstract class MetaModel } } + if (self::$m_bQueryCacheEnabled || self::$m_bTraceQueries) + { + // Need to identify the query + $sOqlQuery = $oFilter->ToOql(); + $sOqlId = md5($sOqlQuery); + } + else + { + $sOqlQuery = "SELECTING... ".$oFilter->GetClass(); + $sOqlId = "query id ? n/a"; + } + + // Query caching // - $bQueryCacheEnabled = true; - if ($bQueryCacheEnabled) + if (self::$m_bQueryCacheEnabled) { - $sOqlQuery = $oFilter->ToOql(); // Warning: using directly the query string as the key to the hash array can FAIL if the string // is long and the differences are only near the end... so it's safer (but not bullet proof?) // to use a hash (like md5) of the string as the key ! @@ -1578,16 +1590,12 @@ abstract class MetaModel // SELECT SLT JOIN lnkSLTToSLA AS L1 ON L1.slt_id=SLT.id JOIN SLA ON L1.sla_id = SLA.id JOIN lnkContractToSLA AS L2 ON L2.sla_id = SLA.id JOIN CustomerContract ON L2.contract_id = CustomerContract.id WHERE SLT.ticket_priority = 1 AND SLA.service_id = 3 AND SLT.metric = 'TTR' AND CustomerContract.customer_id = 2 // the only difference is R instead or O at position 285 (TTR instead of TTO)... // - if (array_key_exists(md5($sOqlQuery), self::$m_aQueryStructCache)) + if (array_key_exists($sOqlId, self::$m_aQueryStructCache)) { // hit! - $oSelect = clone self::$m_aQueryStructCache[$sOqlQuery]; + $oSelect = clone self::$m_aQueryStructCache[$sOqlId]; } } - else - { - $sOqlQuery = "dummy"; - } if (!isset($oSelect)) { @@ -1600,7 +1608,7 @@ abstract class MetaModel $oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oConditionTree, $aClassAliases, $aTableAliases, $aTranslation, $oFilter); $oKPI->ComputeStats('MakeQuery (select)', $sOqlQuery); - self::$m_aQueryStructCache[$sOqlQuery] = clone $oSelect; + self::$m_aQueryStructCache[$sOqlId] = clone $oSelect; } // Check the order by specification, and prefix with the class alias @@ -1636,7 +1644,7 @@ abstract class MetaModel try { - $sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs); + $sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount); } catch (MissingQueryArgument $e) { @@ -1647,16 +1655,25 @@ abstract class MetaModel if (self::$m_bTraceQueries) { - $aParams = array(); - if (!array_key_exists($sOqlQuery, self::$m_aQueriesLog)) + $sQueryId = md5($sRes); + if(!isset(self::$m_aQueriesLog[$sOqlId])) { - self::$m_aQueriesLog[$sOqlQuery] = array( - 'sql' => array(), - 'count' => 0, - ); + self::$m_aQueriesLog[$sOqlId]['oql'] = $sOqlQuery; + self::$m_aQueriesLog[$sOqlId]['hits'] = 1; + } + else + { + self::$m_aQueriesLog[$sOqlId]['hits']++; + } + if(!isset(self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId])) + { + self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['sql'] = $sRes; + self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count'] = 1; + } + else + { + self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count']++; } - self::$m_aQueriesLog[$sOqlQuery]['count']++; - self::$m_aQueriesLog[$sOqlQuery]['sql'][] = $sRes; } return $sRes; @@ -1664,17 +1681,37 @@ abstract class MetaModel public static function ShowQueryTrace() { - $iTotal = 0; - foreach (self::$m_aQueriesLog as $sOql => $aQueryData) + if (!self::$m_bTraceQueries) return; + + $iOqlCount = count(self::$m_aQueriesLog); + if ($iOqlCount == 0) { - echo "

$sOql

\n"; - $iTotal += $aQueryData['count']; - echo '

'.$aQueryData['count'].'

'; - echo '

Example: '.$aQueryData['sql'][0].'

'; + echo "

Trace activated, but no query found

\n"; + return; + } + + $iSqlCount = 0; + foreach (self::$m_aQueriesLog as $aOqlData) + { + $iSqlCount += $aOqlData['hits']; + } + echo "

Stats on SELECT queries: OQL=$iOqlCount, SQL=$iSqlCount

\n"; + + foreach (self::$m_aQueriesLog as $aOqlData) + { + $sOql = $aOqlData['oql']; + $sHits = $aOqlData['hits']; + + echo "

$sHits hits for OQL query: $sOql

\n"; + echo "\n"; } - echo "

Total

\n"; - echo "

Count of executed queries: $iTotal

"; - echo "

Count of built queries: ".count(self::$m_aQueriesLog)."

"; } public static function MakeDeleteQuery(DBObjectSearch $oFilter, $aArgs = array()) @@ -3211,6 +3248,9 @@ abstract class MetaModel ExecutionKPI::EnableMemory(); } + self::$m_bTraceQueries = self::$m_oConfig->GetDebugQueries(); + self::$m_bQueryCacheEnabled = self::$m_oConfig->GetQueryCacheEnabled(); + // Note: load the dictionary as soon as possible, because it might be // needed when some error occur foreach (self::$m_oConfig->GetDictionaries() as $sModule => $sToInclude) diff --git a/core/sqlquery.class.inc.php b/core/sqlquery.class.inc.php index 36daf2c50..e7b9ae5a9 100644 --- a/core/sqlquery.class.inc.php +++ b/core/sqlquery.class.inc.php @@ -238,7 +238,7 @@ class SQLQuery } // Interface, build the SQL query - public function RenderSelect($aOrderBy = array(), $aArgs = array()) + public function RenderSelect($aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false) { // The goal will be to complete the lists as we build the Joins $aFrom = array(); @@ -248,15 +248,31 @@ class SQLQuery $aSetValues = array(); $this->privRender($aFrom, $aFields, $oCondition, $aDelTables, $aSetValues); - $sSelect = self::ClauseSelect($aFields); $sFrom = self::ClauseFrom($aFrom); $sWhere = self::ClauseWhere($oCondition, $aArgs); - $sOrderBy = self::ClauseOrderBy($aOrderBy); - if (!empty($sOrderBy)) + if ($bGetCount) { - $sOrderBy = "ORDER BY $sOrderBy"; + $sSQL = "SELECT COUNT(*) AS COUNT FROM $sFrom WHERE $sWhere"; } - return "SELECT DISTINCT $sSelect FROM $sFrom WHERE $sWhere $sOrderBy"; + else + { + $sSelect = self::ClauseSelect($aFields); + $sOrderBy = self::ClauseOrderBy($aOrderBy); + if (!empty($sOrderBy)) + { + $sOrderBy = "ORDER BY $sOrderBy"; + } + if ($iLimitCount > 0) + { + $sLimit = 'LIMIT '.$iLimitStart.', '.$iLimitCount; + } + else + { + $sLimit = ''; + } + $sSQL = "SELECT DISTINCT $sSelect FROM $sFrom WHERE $sWhere $sOrderBy $sLimit"; + } + return $sSQL; } private static function ClauseSelect($aFields)