N°2272 - OQL performance (OQL class tree wip)

This commit is contained in:
Eric
2019-09-13 17:20:03 +02:00
parent 128afc8a56
commit 93a736e42a
7 changed files with 557 additions and 4 deletions

View File

@@ -0,0 +1,164 @@
<?php
/**
* @copyright Copyright (C) 2010-2019 Combodo SARL
* @license http://opensource.org/licenses/AGPL-3.0
*/
class OQLClassNode
{
private $sClass;
private $sClassAlias;
/** @var OQLJoin[] */
private $aJoins;
private $aExtKeys;
/**
* OQLClassNode constructor.
*
* @param string $sClass
* @param string $sClassAlias
*/
public function __construct($sClass, $sClassAlias)
{
$this->sClass = $sClass;
$this->sClassAlias = $sClassAlias;
$this->aJoins = array();
$this->aExtKeys = array();
}
public function AddExternalKey($sKeyAttCode)
{
if (!isset($this->aExtKeys[$sKeyAttCode]))
{
$this->aExtKeys[$sKeyAttCode] = array();
}
}
public function AddExternalField($sKeyAttCode, $sFieldAttCode, $oAttDef)
{
$this->AddExternalKey($sKeyAttCode);
$this->aExtKeys[$sKeyAttCode][$sFieldAttCode] = $oAttDef;
}
public function AddInnerJoin($oOQLClassNode, $sLeftField, $sRightField)
{
$this->AddJoin(OQLJoin::JOIN_INNER, $oOQLClassNode, $sLeftField, $sRightField);
}
public function AddLeftJoin($oOQLClassNode, $sLeftField, $sRightField)
{
$this->AddJoin(OQLJoin::JOIN_LEFT, $oOQLClassNode, $sLeftField, $sRightField);
}
public function AddInnerJoinTree($oOQLClassNode, $sLeftField, $sRightField, $iOperatorCode = TREE_OPERATOR_BELOW, $bInvertOnClause = false)
{
$this->AddJoin(OQLJoin::JOIN_INNER_TREE, $oOQLClassNode, $sLeftField, $sRightField, $iOperatorCode, $bInvertOnClause);
}
private function AddJoin($sJoinType, $oOQLClassNode, $sLeftField, $sRightField, $sTreeOperator = null, $bInvertOnClause = false)
{
$oOQLJoin = new OQLJoin($sJoinType, $oOQLClassNode, $sLeftField, $sRightField, $sTreeOperator, $bInvertOnClause);
$this->aJoins[] = $oOQLJoin;
return $oOQLJoin;
}
public function DisplayHtml()
{
}
public function RenderDebug()
{
$sOQL = "SELECT `{$this->sClassAlias}` FROM `{$this->sClass}` AS `{$this->sClassAlias}`";
foreach ($this->aJoins as $oJoin)
{
$sOQL .= "{$oJoin->RenderDebug($this->sClassAlias)}";
}
return $sOQL;
}
public function GetExternalKeys()
{
return $this->aExtKeys;
}
public function HasExternalKey($sAttCode)
{
return array_key_exists($sAttCode, $this->aExtKeys);
}
public function GetExternalKey($sAttCode)
{
return $this->aExtKeys[$sAttCode];
}
public function GetClass()
{
return $this->sClass;
}
public function GetClassAlias()
{
return $this->sClassAlias;
}
public function GetJoins()
{
return $this->aJoins;
}
}
class OQLJoin
{
const JOIN_INNER = 'inner';
const JOIN_LEFT = 'left';
const JOIN_INNER_TREE = 'inner_tree';
private $sJoinType;
/** @var \OQLClassNode */
private $oOQLClassNode;
private $sLeftField;
private $sRightField;
private $sTreeOperator;
private $bInvertOnClause;
/**
* OQLJoin constructor.
*
* @param $sJoinType
* @param $oOQLClassNode
* @param $sLeftField
* @param $sRightField
* @param string $sTreeOperator
* @param bool $bInvertOnClause
*/
public function __construct($sJoinType, $oOQLClassNode, $sLeftField, $sRightField, $sTreeOperator = null, $bInvertOnClause = false)
{
$this->sJoinType = $sJoinType;
$this->oOQLClassNode = $oOQLClassNode;
$this->sLeftField = $sLeftField;
$this->sRightField = $sRightField;
$this->sTreeOperator = $sTreeOperator;
$this->bInvertOnClause = $bInvertOnClause;
}
public function RenderDebug($sClassAlias, $sPrefix = " ")
{
$sType = strtoupper($this->sJoinType);
$sOQL = "\n{$sPrefix}{$sType} JOIN `{$this->oOQLClassNode->GetClass()}` AS `{$this->oOQLClassNode->GetClassAlias()}`";
//$sOQL = str_pad($sOQL, 100);
$sOQL .= "\n{$sPrefix} ON `{$sClassAlias}`.`{$this->sLeftField}` = `{$this->oOQLClassNode->GetClassAlias()}`.`{$this->sRightField}`";
$sPrefix .= " ";
foreach ($this->oOQLClassNode->GetJoins() as $oJoin)
{
$sOQL .= " {$oJoin->RenderDebug($this->oOQLClassNode->GetClassAlias(), $sPrefix)}";
}
return $sOQL;
}
}

View File

@@ -0,0 +1,377 @@
<?php
/**
* @copyright Copyright (C) 2010-2019 Combodo SARL
* @license http://opensource.org/licenses/AGPL-3.0
*/
class OQLClassTreeBuilder
{
/** @var \DBObjectSearch */
private $oDBObjetSearch;
/** @var OQLClassNode */
private $oOQLClassNode;
/** @var \QueryBuilderContext */
private $oBuild;
private $sClass;
private $sClassAlias;
/**
* OQLClassTreeBuilder constructor.
*
* @param \DBObjectSearch $oDBObjetSearch
* @param \QueryBuilderContext $oBuild
*/
public function __construct($oDBObjetSearch, $oBuild)
{
$this->oBuild = $oBuild;
$this->oDBObjetSearch = $oDBObjetSearch;
$this->sClass = $oDBObjetSearch->GetFirstJoinedClass();
$this->sClassAlias = $oDBObjetSearch->GetFirstJoinedClassAlias();
$this->oOQLClassNode = new OQLClassNode($this->sClass, $this->sClassAlias);
}
/**
*
* @return \OQLClassNode
* @throws \CoreException
* @throws \Exception
*/
public function DevelopOQLClassNode()
{
// array of (attcode => fieldexpression)
$aExpectedAttributes = $this->oBuild->m_oQBExpressions->GetUnresolvedFields($this->sClassAlias);
$this->AddExternalKeysFromSearch();
$aPolymorphicJoinAlias = $this->TranslatePolymorphicExpressions($aExpectedAttributes);
$this->AddExpectedExternalFields($aExpectedAttributes);
$this->JoinClassesForExternalKeys();
$this->JoinClassesReferencedBy();
$this->JoinClassesForPolymorphicExpressions($aPolymorphicJoinAlias);
// That's all... cross fingers and we'll get some working query
return $this->oOQLClassNode;
}
/**
* Get all Ext keys used by the filter
*
*/
private function AddExternalKeysFromSearch()
{
foreach ($this->oDBObjetSearch->GetCriteria_PointingTo() as $sKeyAttCode => $aPointingTo)
{
if (array_key_exists(TREE_OPERATOR_EQUALS, $aPointingTo))
{
$this->oOQLClassNode->AddExternalKey($sKeyAttCode);
}
}
}
/**
* @param array $aExpectedAttributes
*
* @return array of classes to join for polymorphic expressions
*
* @throws \CoreException
*/
private function TranslatePolymorphicExpressions($aExpectedAttributes)
{
$aPolymorphicJoinAlias = array(); // array of (subclass => alias)
foreach ($aExpectedAttributes as $sExpectedAttCode => $oExpression)
{
if (!MetaModel::IsValidAttCode($this->sClass, $sExpectedAttCode))
{
continue;
}
$oAttDef = MetaModel::GetAttributeDef($this->sClass, $sExpectedAttCode);
if ($oAttDef->IsBasedOnOQLExpression())
{
// To optimize: detect a restriction on child classes in the condition expression
// e.g. SELECT FunctionalCI WHERE finalclass IN ('Server', 'VirtualMachine')
$oExpression = DBObjectSearch::GetPolymorphicExpression($this->sClass, $sExpectedAttCode);
$aRequiredFields = array();
$oExpression->GetUnresolvedFields('', $aRequiredFields);
$aTranslateFields = array();
foreach ($aRequiredFields as $sSubClass => $aFields)
{
foreach ($aFields as $sAttCode => $oField)
{
$oAttDef = MetaModel::GetAttributeDef($sSubClass, $sAttCode);
if ($oAttDef->IsExternalKey())
{
$sClassOfAttribute = MetaModel::GetAttributeOrigin($sSubClass, $sAttCode);
$this->oOQLClassNode->AddExternalKey($sAttCode);
}
elseif ($oAttDef->IsExternalField())
{
$sKeyAttCode = $oAttDef->GetKeyAttCode();
$sClassOfAttribute = MetaModel::GetAttributeOrigin($sSubClass, $sKeyAttCode);
$this->oOQLClassNode->AddExternalField($sKeyAttCode, $sAttCode, $oAttDef);
}
else
{
$sClassOfAttribute = MetaModel::GetAttributeOrigin($sSubClass, $sAttCode);
}
if (MetaModel::IsParentClass($sClassOfAttribute, $this->sClass))
{
// The attribute is part of the standard query
//
$sAliasForAttribute = $this->sClassAlias;
}
else
{
// The attribute will be available from an additional outer join
// For each subclass (table) one single join is enough
//
if (!array_key_exists($sClassOfAttribute, $aPolymorphicJoinAlias))
{
$sAliasForAttribute = $this->oBuild->GenerateClassAlias($this->sClassAlias.'_poly_'.$sClassOfAttribute,
$sClassOfAttribute);
$aPolymorphicJoinAlias[$sClassOfAttribute] = $sAliasForAttribute;
}
else
{
$sAliasForAttribute = $aPolymorphicJoinAlias[$sClassOfAttribute];
}
}
$aTranslateFields[$sSubClass][$sAttCode] = new FieldExpression($sAttCode, $sAliasForAttribute);
}
}
$oExpression = $oExpression->Translate($aTranslateFields, false);
$aTranslateNow = array();
$aTranslateNow[$this->sClassAlias][$sExpectedAttCode] = $oExpression;
$this->oBuild->m_oQBExpressions->Translate($aTranslateNow, false);
}
}
return $aPolymorphicJoinAlias;
}
/**
* Add the ext fields used in the select (external keys may be created for that)
*
* @param array $aExpectedAttributes
*
* @throws \CoreException
*/
private function AddExpectedExternalFields($aExpectedAttributes)
{
foreach (MetaModel::ListAttributeDefs($this->sClass) as $sAttCode => $oAttDef)
{
if ($oAttDef->IsExternalField())
{
if (array_key_exists($sAttCode, $aExpectedAttributes))
{
// Add the external attribute
$sKeyAttCode = $oAttDef->GetKeyAttCode();
$this->oOQLClassNode->AddExternalField($sKeyAttCode, $sAttCode, $oAttDef);
}
}
}
}
/**
* Add joins for external keys/fields
*
* @throws \Exception
*/
private function JoinClassesForExternalKeys()
{
// Get filters from the search outgoing joins
$aAllPointingTo = $this->oDBObjetSearch->GetCriteria_PointingTo();
// Add filters from external keys
foreach (array_keys($this->oOQLClassNode->GetExternalKeys()) as $sKeyAttCode)
{
if (!MetaModel::IsValidAttCode($this->sClass, $sKeyAttCode))
{
continue;
} // Not defined in the class, skip it
$oKeyAttDef = MetaModel::GetAttributeDef($this->sClass, $sKeyAttCode);
$aPointingTo = isset($aAllPointingTo[$sKeyAttCode]) ? $aAllPointingTo[$sKeyAttCode] : array();
if (!array_key_exists(TREE_OPERATOR_EQUALS, $aPointingTo))
{
// The join was not explicitly defined in the filter,
// we need to do it now
$sKeyClass = $oKeyAttDef->GetTargetClass();
$sKeyClassAlias = $this->oBuild->GenerateClassAlias($sKeyClass.'_'.$sKeyAttCode, $sKeyClass);
$oExtFilter = new DBObjectSearch($sKeyClass, $sKeyClassAlias);
$aAllPointingTo[$sKeyAttCode][TREE_OPERATOR_EQUALS][$sKeyClassAlias] = $oExtFilter;
}
}
$oQBContextExpressions = $this->oBuild->m_oQBExpressions;
foreach ($aAllPointingTo as $sKeyAttCode => $aPointingTo)
{
foreach ($aPointingTo as $iOperatorCode => $aFilter)
{
foreach ($aFilter as $oExtFilter)
{
if (!MetaModel::IsValidAttCode($this->sClass, $sKeyAttCode))
{
continue;
} // Not defined in the class, skip it
// The aliases should not conflict because normalization occurred while building the filter
$oKeyAttDef = MetaModel::GetAttributeDef($this->sClass, $sKeyAttCode);
$sKeyClass = $oExtFilter->GetFirstJoinedClass();
$sKeyClassAlias = $oExtFilter->GetFirstJoinedClassAlias();
// Note: there is no search condition in $oExtFilter, because normalization did merge the condition onto the top of the filter tree
if ($iOperatorCode == TREE_OPERATOR_EQUALS)
{
if ($this->oOQLClassNode->HasExternalKey($sKeyAttCode))
{
// Specify expected attributes for the target class query
// ... and use the current alias !
$aTranslateNow = array(); // Translation for external fields - must be performed before the join is done (recursion...)
foreach ($this->oOQLClassNode->GetExternalKey($sKeyAttCode) as $sAttCode => $oAtt)
{
$oExtAttDef = $oAtt->GetExtAttDef();
if ($oExtAttDef->IsBasedOnOQLExpression())
{
$sExtAttCode = $oExtAttDef->GetCode();
}
else
{
$sExtAttCode = $oAtt->GetExtAttCode();
}
// Translate mainclass.extfield => remoteclassalias.remotefieldcode
$aTranslateNow[$this->sClassAlias][$sAttCode] = new FieldExpression($sExtAttCode, $sKeyClassAlias);
}
if ($oKeyAttDef instanceof AttributeObjectKey)
{
// Add the condition: `$sTargetAlias`.$sClassAttCode IN (subclasses of $sKeyClass')
$sClassAttCode = $oKeyAttDef->Get('class_attcode');
$oClassListExpr = ListExpression::FromScalars(MetaModel::EnumChildClasses($sKeyClass,
ENUM_CHILD_CLASSES_ALL));
$oClassExpr = new FieldExpression($sClassAttCode, $this->sClassAlias);
$oClassRestriction = new BinaryExpression($oClassExpr, 'IN', $oClassListExpr);
$oQBContextExpressions->AddCondition($oClassRestriction);
}
// Translate prior to recursing
//
$oQBContextExpressions->Translate($aTranslateNow, false);
$oQBContextExpressions->PushJoinField(new FieldExpression('id', $sKeyClassAlias));
$oOQLClassTreeBuilder = new OQLClassTreeBuilder($oExtFilter, $this->oBuild);
$oSelectExtKey = $oOQLClassTreeBuilder->DevelopOQLClassNode();
$oJoinExpr = $oQBContextExpressions->PopJoinField();
//$sExternalKeyTable = $oJoinExpr->GetParent();
$sExternalKeyField = $oJoinExpr->GetName();
if ($oKeyAttDef->IsNullAllowed())
{
$this->oOQLClassNode->AddLeftJoin($oSelectExtKey, $sKeyAttCode, $sExternalKeyField);
}
else
{
$this->oOQLClassNode->AddInnerJoin($oSelectExtKey, $sKeyAttCode, $sExternalKeyField);
}
}
}
elseif (MetaModel::GetAttributeOrigin($sKeyClass, $sKeyAttCode) == $this->sClass)
{
$oQBContextExpressions->PushJoinField(new FieldExpression($sKeyAttCode, $sKeyClassAlias));
$oOQLClassTreeBuilder = new OQLClassTreeBuilder($oExtFilter, $this->oBuild);
$oSelectExtKey = $oOQLClassTreeBuilder->DevelopOQLClassNode();
$oJoinExpr = $oQBContextExpressions->PopJoinField();
//$sExternalKeyTable = $oJoinExpr->GetParent();
$sExternalKeyField = $oJoinExpr->GetName();
$this->oOQLClassNode->AddInnerJoinTree($oSelectExtKey, $sKeyAttCode, $sExternalKeyField, $iOperatorCode);
}
}
}
}
}
/**
* Filter on objects referencing me
*
* @throws \CoreException
*/
private function JoinClassesReferencedBy()
{
$sKeyField = MetaModel::DBGetKey($this->sClass);
foreach ($this->oDBObjetSearch->GetCriteria_ReferencedBy() as $sForeignClass => $aReferences)
{
foreach ($aReferences as $sForeignExtKeyAttCode => $aFiltersByOperator)
{
foreach ($aFiltersByOperator as $iOperatorCode => $aFilters)
{
foreach ($aFilters as $oForeignFilter)
{
$oForeignKeyAttDef = MetaModel::GetAttributeDef($sForeignClass, $sForeignExtKeyAttCode);
$sForeignClassAlias = $oForeignFilter->GetFirstJoinedClassAlias();
$this->oBuild->m_oQBExpressions->PushJoinField(new FieldExpression($sForeignExtKeyAttCode, $sForeignClassAlias));
if ($oForeignKeyAttDef instanceof AttributeObjectKey)
{
$sClassAttCode = $oForeignKeyAttDef->Get('class_attcode');
// Add the condition: `$sForeignClassAlias`.$sClassAttCode IN (subclasses of $sClass')
$oClassListExpr = ListExpression::FromScalars(MetaModel::EnumChildClasses($this->sClass,
ENUM_CHILD_CLASSES_ALL));
$oClassExpr = new FieldExpression($sClassAttCode, $sForeignClassAlias);
$oClassRestriction = new BinaryExpression($oClassExpr, 'IN', $oClassListExpr);
$this->oBuild->m_oQBExpressions->AddCondition($oClassRestriction);
}
$oOQLClassTreeBuilder = new OQLClassTreeBuilder($oForeignFilter, $this->oBuild);
$oSelectForeign = $oOQLClassTreeBuilder->DevelopOQLClassNode();
$oJoinExpr = $this->oBuild->m_oQBExpressions->PopJoinField();
$sForeignKeyColumn = $oJoinExpr->GetName();
if ($iOperatorCode == TREE_OPERATOR_EQUALS)
{
$this->oOQLClassNode->AddInnerJoin($oSelectForeign, $sKeyField, $sForeignKeyColumn);
}
else
{
// Hierarchical key
$this->oOQLClassNode->AddInnerJoinTree($oSelectForeign, $sKeyField, $sForeignKeyColumn, $iOperatorCode, true);
}
}
}
}
}
}
/**
* Additional JOINS for polymorphic expressions (friendlyname, obsolescenceflag...)
*
* @param array $aPolymorphicJoinAlias
*
* @throws \CoreException
*/
private function JoinClassesForPolymorphicExpressions($aPolymorphicJoinAlias)
{
$sKeyField = MetaModel::DBGetKey($this->sClass);
foreach ($aPolymorphicJoinAlias as $sSubClass => $sSubClassAlias)
{
$oSubClassFilter = new DBObjectSearch($sSubClass, $sSubClassAlias);
$oOQLClassTreeBuilder = new OQLClassTreeBuilder($oSubClassFilter, $this->oBuild);
$oSelectFN = $oOQLClassTreeBuilder->DevelopOQLClassNode();
$this->oOQLClassNode->AddLeftJoin($oSelectFN, $sKeyField, MetaModel::DBGetKey($sSubClass));
}
}
}

View File

@@ -122,7 +122,7 @@ class QueryBuilderExpressions
/**
* @param $sAlias
*
* @return array of unresolved fields
* @return \FieldExpression[] of unresolved fields
*/
public function GetUnresolvedFields($sAlias)
{

View File

@@ -54,7 +54,11 @@ class SQLObjectQueryBuilder
}
$oBuild = new QueryBuilderContext($this->oDBObjetSearch, $aModifierProperties, $aGroupByExpr, $aSelectedClasses, $aSelectExpr, $aAttToLoad);
$oOQLClassTreeBuilder = new OQLClassTreeBuilder($this->oDBObjetSearch, $oBuild);
$oOQLQuery = $oOQLClassTreeBuilder->DevelopOQLClassNode();
$oBuild = new QueryBuilderContext($this->oDBObjetSearch, $aModifierProperties, $aGroupByExpr, $aSelectedClasses, $aSelectExpr, $aAttToLoad);
$oSQLQuery = $this->MakeSQLObjectQueryRoot($oBuild, $aAttToLoad, array(), $aGroupByExpr, $aSelectExpr);
return $oSQLQuery;
@@ -158,11 +162,13 @@ class SQLObjectQueryBuilder
return $oSQLQuery;
}
/**
* @param \QueryBuilderContext $oBuild
* @param null $aAttToLoad
* @param array $aValues
* @return null|SQLObjectQuery
*
* @return \SQLObjectQuery|null
* @throws \CoreException
*/
private function MakeSQLObjectQuery($oBuild, $aAttToLoad = null, $aValues = array())

View File

@@ -279,7 +279,7 @@ class ClassLoader
*/
public function setApcuPrefix($apcuPrefix)
{
$this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null;
$this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null;
}
/**
@@ -377,7 +377,7 @@ class ClassLoader
$subPath = $class;
while (false !== $lastPos = strrpos($subPath, '\\')) {
$subPath = substr($subPath, 0, $lastPos);
$search = $subPath.'\\';
$search = $subPath . '\\';
if (isset($this->prefixDirsPsr4[$search])) {
$pathEnd = DIRECTORY_SEPARATOR . substr($logicalPathPsr4, $lastPos + 1);
foreach ($this->prefixDirsPsr4[$search] as $dir) {

View File

@@ -282,7 +282,10 @@ return array(
'NewObjectMenuNode' => $baseDir . '/application/menunode.class.inc.php',
'NewsroomProviderBase' => $baseDir . '/application/newsroomprovider.class.inc.php',
'NiceWebPage' => $baseDir . '/application/nicewebpage.class.inc.php',
'OQLClassNode' => $baseDir . '/core/oqlclassnode.class.inc.php',
'OQLClassTreeBuilder' => $baseDir . '/core/oqlclasstreebuilder.class.inc.php',
'OQLException' => $baseDir . '/core/oql/oqlexception.class.inc.php',
'OQLJoin' => $baseDir . '/core/oqlclassnode.class.inc.php',
'OQLLexer' => $baseDir . '/core/oql/oql-lexer.php',
'OQLLexerException' => $baseDir . '/core/oql/oql-lexer.php',
'OQLLexerRaw' => $baseDir . '/core/oql/oql-lexer.php',

View File

@@ -503,7 +503,10 @@ class ComposerStaticInit0018331147de7601e7552f7da8e3bb8b
'NewObjectMenuNode' => __DIR__ . '/../..' . '/application/menunode.class.inc.php',
'NewsroomProviderBase' => __DIR__ . '/../..' . '/application/newsroomprovider.class.inc.php',
'NiceWebPage' => __DIR__ . '/../..' . '/application/nicewebpage.class.inc.php',
'OQLClassNode' => __DIR__ . '/../..' . '/core/oqlclassnode.class.inc.php',
'OQLClassTreeBuilder' => __DIR__ . '/../..' . '/core/oqlclasstreebuilder.class.inc.php',
'OQLException' => __DIR__ . '/../..' . '/core/oql/oqlexception.class.inc.php',
'OQLJoin' => __DIR__ . '/../..' . '/core/oqlclassnode.class.inc.php',
'OQLLexer' => __DIR__ . '/../..' . '/core/oql/oql-lexer.php',
'OQLLexerException' => __DIR__ . '/../..' . '/core/oql/oql-lexer.php',
'OQLLexerRaw' => __DIR__ . '/../..' . '/core/oql/oql-lexer.php',