From 5d532a78794ddbddb30d36c09c3ee4847103c543 Mon Sep 17 00:00:00 2001 From: odain Date: Mon, 27 Apr 2026 17:00:12 +0200 Subject: [PATCH] =?UTF-8?q?N=C2=B09165=20-=20Allow=20direct=20suppression?= =?UTF-8?q?=20or=20delayed=20suppression=20depending=20on=20the=20amount?= =?UTF-8?q?=20of=20data=20to=20remove?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- application/utils.inc.php | 44 +++++ .../src/Service/DeletionPlanService.php | 183 +++++++++++------- 2 files changed, 157 insertions(+), 70 deletions(-) diff --git a/application/utils.inc.php b/application/utils.inc.php index e19f7f18a..713666c10 100644 --- a/application/utils.inc.php +++ b/application/utils.inc.php @@ -3190,4 +3190,48 @@ TXT } } } + + public static function GetMemoryLimit(): int + { + $sLimit = ini_get('memory_limit'); + if ($sLimit == '-1') { + return 128 * 1048576; + } + switch (substr($sLimit, -1)) { + case 'M': + case 'm': + return (int)$sLimit * 1048576; + case 'K': + case 'k': + return (int)$sLimit * 1024; + case 'G': + case 'g': + return (int)$sLimit * 1073741824; + default: + return (int)$sLimit; + } + } + + /** + * @param int $iMaxTime + * @param int $iMaxMemory + * + * @return bool + */ + public static function ShouldStopExecution(int $iMaxTime = 0, int $iMaxMemoryPercent = 100): bool + { + if (($iMaxTime != 0) && (time() > $iMaxTime)) { + \IssueLog::Debug(__METHOD__.' timeout '.time()." (current) > $iMaxTime (max)"); + return true; + } + + $iMemory = memory_get_usage(true); + $iMaxMemory = self::GetMemoryLimit() * $iMaxMemoryPercent / 100; + if ($iMemory > $iMaxMemory) { + \IssueLog::Debug(__METHOD__." Memory limit $iMemory (current) > $iMaxMemory (max)"); + return true; + } + + return false; + } } diff --git a/datamodels/2.x/combodo-data-feature-removal/src/Service/DeletionPlanService.php b/datamodels/2.x/combodo-data-feature-removal/src/Service/DeletionPlanService.php index c0cce332b..a51d13ba2 100644 --- a/datamodels/2.x/combodo-data-feature-removal/src/Service/DeletionPlanService.php +++ b/datamodels/2.x/combodo-data-feature-removal/src/Service/DeletionPlanService.php @@ -2,34 +2,19 @@ namespace Combodo\iTop\DataFeatureRemoval\Service; +use CMDBObjectSet; use CMDBSource; use Combodo\iTop\DataFeatureRemoval\Entity\DeletionPlanSummaryEntity; use Combodo\iTop\DataFeatureRemoval\Helper\DataFeatureRemovalException; +use DBObject; use DBObjectSearch; use DeletionPlan; use MetaModel; +use utils; class DeletionPlanService { - private static DeletionPlanService $oInstance; - - protected function __construct() - { - } - - final public static function GetInstance(): DeletionPlanService - { - if (!isset(self::$oInstance)) { - self::$oInstance = new DeletionPlanService(); - } - - return self::$oInstance; - } - - final public static function SetInstance(?DeletionPlanService $oInstance): void - { - self::$oInstance = $oInstance; - } + private array $aVisited = []; /** * Get a summary of the deletion plan computed for the classes. @@ -88,73 +73,131 @@ class DeletionPlanService return $oSet->ToArray(); } - /** - * @param array $aClasses - * - * @return array<\Combodo\iTop\DataFeatureRemoval\Entity\DeletionPlanSummaryEntity> - * @throws \Combodo\iTop\DataFeatureRemoval\Helper\DataFeatureRemovalException - * @throws \CoreException - * @throws \CoreUnexpectedValue - * @throws \MySQLException - */ - public function ExecuteDeletionPlan(array $aClasses): array + private function GetNextObjectToDelete(array $aClasses): ?DBObject { - $oDeletionPlan = $this->GetDeletionPlan($aClasses); - - if (count($oDeletionPlan->GetIssues()) > 0) { - throw new DataFeatureRemovalException("Deletion Plan cannot be executed due to issues"); + foreach ($aClasses as $sClass) { + $oFilter = new DBObjectSearch($sClass); + $oFilter->AllowAllData(); + $oSet = new \DBObjectSet($oFilter); + $oObject = $oSet->Fetch(); + if (! is_null($oObject)) { + return $oObject; + } } - $aSummary = []; - foreach ($oDeletionPlan->ListUpdates() as $sClass => $aToUpdate) { - $oDeletionPlanSummaryEntity = $aSummary[$sClass] ?? new DeletionPlanSummaryEntity($sClass); + return null; + } - foreach ($aToUpdate as $aData) { - $oToUpdate = $aData['to_reset']; - /** @var \DBObject $oToUpdate */ - foreach ($aData['attributes'] as $sRemoteExtKey => $aRemoteAttDef) { - $oToUpdate->Set($sRemoteExtKey, $aData['values'][$sRemoteExtKey]); - } - $oToUpdate->DBUpdate(); - $oDeletionPlanSummaryEntity->iUpdateCount++; + private function Update(DBObject $oToUpdate, string $sAttCode, $value) + { + $oToUpdate->Set($sAttCode, $value); + $oToUpdate->DBUpdate(); + } + + private function Delete(string $sClass, string $sId) + { + try { + CMDBSource::Query('START TRANSACTION'); + // Delete any existing change tracking about the current object + $oFilter = new DBObjectSearch('CMDBChangeOp'); + $oFilter->AddCondition('objclass', $sClass, '='); + $oFilter->AddCondition('objkey', $sId, '='); + MetaModel::PurgeData($oFilter); + + // Delete the entry + $aClassesToRemove = array_merge(MetaModel::EnumChildClasses($sClass, ENUM_CHILD_CLASSES_ALL), MetaModel::EnumParentClasses($sClass, ENUM_PARENT_CLASSES_EXCLUDELEAF, false)); + foreach ($aClassesToRemove as $sParentClass) { + $oFilter = DBObjectSearch::FromOQL_AllData("SELECT $sParentClass WHERE id=:id"); + $sQuery = $oFilter->MakeDeleteQuery(['id' => $sId]); + CMDBSource::DeleteFrom($sQuery); } - $aSummary[$sClass] = $oDeletionPlanSummaryEntity; + CMDBSource::Query('COMMIT'); + } catch (\Exception $e) { + \IssueLog::Exception(__METHOD__.': Cleanup failed', $e); + CMDBSource::Query('ROLLBACK'); + throw $e; + } + } + + /** + * @param array $aClasses + * @param int $iMaxExecutionTime + * @param int $iMaxMemoryPercent + * @return void + * @throws \Combodo\iTop\DataFeatureRemoval\Helper\DataFeatureRemovalException + */ + public function ExecuteDeletionPlan(array $aClasses, int $iMaxExecutionTime = 30, int $iMaxMemoryPercent = 80): void + { + $oObject = $this->GetNextObjectToDelete($aClasses); + if (is_null($oObject)) { + return; } - foreach ($oDeletionPlan->ListDeletes() as $sClass => $aDeletes) { - $oDeletionPlanSummaryEntity = $aSummary[$sClass] ?? new DeletionPlanSummaryEntity($sClass); + $iMaxTime = time() + $iMaxExecutionTime; + $this->RecursiveDeletion($oObject, $iMaxTime, $iMaxMemoryPercent); + } - foreach ($aDeletes as $sId => $aDelete) { - try { - CMDBSource::Query('START TRANSACTION'); - // Delete any existing change tracking about the current object - $oFilter = new DBObjectSearch('CMDBChangeOp'); - $oFilter->AddCondition('objclass', $sClass, '='); - $oFilter->AddCondition('objkey', $sId, '='); - MetaModel::PurgeData($oFilter); + public function IsVisited(DBObject $oObject): bool + { + $sClass = get_class($oObject); + $sId = $oObject->GetKey(); + $sKey = "{$sClass}_{$sId}"; - // Delete the entry - $aClassesToRemove = array_merge(MetaModel::EnumChildClasses($sClass, ENUM_CHILD_CLASSES_ALL), MetaModel::EnumParentClasses($sClass, ENUM_PARENT_CLASSES_EXCLUDELEAF, false)); - foreach ($aClassesToRemove as $sParentClass) { - $oFilter = DBObjectSearch::FromOQL_AllData("SELECT $sParentClass WHERE id=:id"); - $sQuery = $oFilter->MakeDeleteQuery(['id' => $sId]); - CMDBSource::DeleteFrom($sQuery); + $bRes = $this->aVisited[$sKey] ?? false; + $this->aVisited[$sKey] = true; + return $bRes; + } + + private function RecursiveDeletion(DBObject $oObjectToClean, int $iMaxTime, int $iMaxMemoryPercent): void + { + if (utils::ShouldStopExecution($iMaxTime, $iMaxMemoryPercent)) { + return; + } + + $sClass = get_class($oObjectToClean); + + $aReferencingMe = MetaModel::EnumReferencingClasses($sClass); + foreach ($aReferencingMe as $sRemoteClass => $aExtKeys) { + /** @var \AttributeExternalKey $oExtKeyAttDef */ + foreach ($aExtKeys as $sExtKeyAttCode => $oExtKeyAttDef) { + // skip if this external key is behind an external field + if (!$oExtKeyAttDef->IsExternalKey(EXTKEY_ABSOLUTE)) { + continue; + } + + $oSearch = new DBObjectSearch($sRemoteClass); + $oSearch->AddCondition($sExtKeyAttCode, $oObjectToClean->GetKey(), '='); + $oSearch->AllowAllData(); + $oSet = new CMDBObjectSet($oSearch); + $oSet->OptimizeColumnLoad([$sRemoteClass => ['id', $oExtKeyAttDef->GetCode()]]); + /** @var DBObject $oDependentObj */ + while ($oDependentObj = $oSet->Fetch()) { + $iDeletePropagationOption = $oExtKeyAttDef->GetDeletionPropagationOption(); + if ($iDeletePropagationOption == DEL_MANUAL) { + throw new DataFeatureRemovalException("DEL_MANUAL object"); } - CMDBSource::Query('COMMIT'); - } catch (\Exception $e) { - \IssueLog::Exception(__METHOD__.': Cleanup failed', $e); - CMDBSource::Query('ROLLBACK'); - throw $e; + if ($oExtKeyAttDef->IsNullAllowed()) { + // Optional external key, list to reset + if (($iDeletePropagationOption == DEL_MOVEUP) && ($oExtKeyAttDef->IsHierarchicalKey())) { + // Move the child up one level i.e. set the same parent as the current object + $iParentId = $oObjectToClean->Get($oExtKeyAttDef->GetCode()); + $this->Update($oDependentObj, $oExtKeyAttDef->GetCode(), $iParentId); + } else { + $this->Update($oDependentObj, $oExtKeyAttDef->GetCode(), 0); + } + } else { + if ($this->IsVisited($oDependentObj)) { + continue; + } + $this->RecursiveDeletion($oDependentObj, $iMaxTime, $iMaxMemoryPercent); + } } - $oDeletionPlanSummaryEntity->iDeleteCount++; } - - $aSummary[$sClass] = $oDeletionPlanSummaryEntity; } - return $aSummary; + $this->Delete($sClass, $oObjectToClean->GetKey()); } /**