composer/iTop
6
0
Fork 0
mirror of https://github.com/Combodo/iTop.git synced 2026-02-13 07:24:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

N°4342 - Improve generic bulk deletion function with memory limit handling (#321)

Browse Source
This commit is contained in:
Anne-Catherine
2024-02-28 09:55:04 +01:00
committed by GitHub
parent e7b493dafa
commit baa05ba8d4
3 changed files with 82 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
core/config.class.inc.php
View File

@@ -447,14 +447,14 @@ class Config
'show_in_conf_sample' => true, 'show_in_conf_sample' => true,
], ],
'export_pdf_font' => [ // @since 2.7.0 PR #49 / N°1947 'export_pdf_font' => [ // @since 2.7.0 PR #49 / N°1947
'type' => 'string', 'type' => 'string',
'description' => 'Font used when generating a PDF file', 'description' => 'Font used when generating a PDF file',
'default' => 'DejaVuSans', // DejaVuSans is a UTF-8 Unicode font, embedded in the TCPPDF lib we're using 'default' => 'DejaVuSans', // DejaVuSans is a UTF-8 Unicode font, embedded in the TCPPDF lib we're using
// Standard PDF fonts like helvetica or times newroman are NOT Unicode // Standard PDF fonts like helvetica or times newroman are NOT Unicode
// A new DroidSansFallback can be used to improve CJK support (se PR #49) // A new DroidSansFallback can be used to improve CJK support (se PR #49)
'value' => '', 'value' => '',
'source_of_value' => '', 'source_of_value' => '',
'show_in_conf_sample' => false, 'show_in_conf_sample' => false,
], ],
'access_mode' => [ 'access_mode' => [
'type' => 'integer', 'type' => 'integer',
@@ -1119,6 +1119,14 @@ class Config
'source_of_value' => '', 'source_of_value' => '',
'show_in_conf_sample' => false, 'show_in_conf_sample' => false,
], ],
'purge_data.max_chunk_size' => [
'type' => 'integer',
'description' => 'Maximum number of items deleted per loop. Used in function MetaModel::PurgeData',
'default' => 1000,
'value' => 1000,
'source_of_value' => '',
'show_in_conf_sample' => false,
],
'max_history_length' => [ 'max_history_length' => [
'type' => 'integer', 'type' => 'integer',
'description' => 'Maximum length of the history table (in the "History" tab on each object) before it gets truncated. Latest modifications are displayed first.', 'description' => 'Maximum length of the history table (in the "History" tab on each object) before it gets truncated. Latest modifications are displayed first.',
@@ -1324,9 +1332,9 @@ class Config
'draft_attachments_lifetime' => [ 'draft_attachments_lifetime' => [
'type' => 'integer', 'type' => 'integer',
'description' => 'Lifetime (in seconds) of drafts\' attachments and inline images: after this duration, the garbage collector will delete them.', 'description' => 'Lifetime (in seconds) of drafts\' attachments and inline images: after this duration, the garbage collector will delete them.',
'default' => 86400, 'default' => 86400,
'value' => '', 'value' => '',
'source_of_value' => '', 'source_of_value' => '',
'show_in_conf_sample' => false, 'show_in_conf_sample' => false,
], ],
'date_and_time_format' => [ 'date_and_time_format' => [
@@ -1882,6 +1890,7 @@ class Config
* @var integer Number of seconds between two reloads of the display (standard) * @var integer Number of seconds between two reloads of the display (standard)
*/ */
protected $m_iStandardReloadInterval; protected $m_iStandardReloadInterval;
/** /**
* @var integer Number of seconds between two reloads of the display (fast) * @var integer Number of seconds between two reloads of the display (fast)
*/ */
@@ -2553,9 +2562,9 @@ class Config
// Old fashioned integer settings // Old fashioned integer settings
$aIntValues = array( $aIntValues = array(
'fast_reload_interval' => $this->m_iFastReloadInterval, 'fast_reload_interval' => $this->m_iFastReloadInterval,
'max_display_limit' => $this->m_iMaxDisplayLimit, 'max_display_limit' => $this->m_iMaxDisplayLimit,
'min_display_limit' => $this->m_iMinDisplayLimit, 'min_display_limit' => $this->m_iMinDisplayLimit,
'standard_reload_interval' => $this->m_iStandardReloadInterval, 'standard_reload_interval' => $this->m_iStandardReloadInterval,
); );
foreach ($aIntValues as $sKey => $iValue) foreach ($aIntValues as $sKey => $iValue)

53
core/metamodel.class.php
View File

@@ -7168,32 +7168,45 @@ abstract class MetaModel
*/ */
public static function PurgeData($oFilter) public static function PurgeData($oFilter)
{ {
$iMaxChunkSize = MetaModel::GetConfig()->Get('purge_data.max_chunk_size');
$sTargetClass = $oFilter->GetClass(); $sTargetClass = $oFilter->GetClass();
$oSet = new DBObjectSet($oFilter); $iNbIdsDeleted = 0;
$oSet->OptimizeColumnLoad(array($sTargetClass => array('finalclass'))); $bExecuteQuery = true;
$aIdToClass = $oSet->GetColumnAsArray('finalclass', true);
$aIds = array_keys($aIdToClass); // This loop allows you to delete objects in batches of $iMaxChunkSize elements
if (count($aIds) > 0) while ($bExecuteQuery) {
{ $oSet = new DBObjectSet($oFilter);
$aQuotedIds = CMDBSource::Quote($aIds); $oSet->SetLimit($iMaxChunkSize);
$sIdList = implode(',', $aQuotedIds); $oSet->OptimizeColumnLoad(array($sTargetClass => array('finalclass')));
$aTargetClasses = array_merge( $aIdToClass = $oSet->GetColumnAsArray('finalclass', true);
self::EnumChildClasses($sTargetClass, ENUM_CHILD_CLASSES_ALL),
self::EnumParentClasses($sTargetClass, ENUM_PARENT_CLASSES_EXCLUDELEAF)
);
foreach($aTargetClasses as $sSomeClass)
{
$sTable = MetaModel::DBGetTable($sSomeClass);
$sPKField = MetaModel::DBGetKey($sSomeClass);
$sDeleteSQL = "DELETE FROM `$sTable` WHERE `$sPKField` IN ($sIdList)"; $aIds = array_keys($aIdToClass);
CMDBSource::DeleteFrom($sDeleteSQL); $iNbIds = count($aIds);
if ($iNbIds > 0) {
$aQuotedIds = CMDBSource::Quote($aIds);
$sIdList = implode(',', $aQuotedIds);
$aTargetClasses = array_merge(
self::EnumChildClasses($sTargetClass, ENUM_CHILD_CLASSES_ALL),
self::EnumParentClasses($sTargetClass, ENUM_PARENT_CLASSES_EXCLUDELEAF)
);
foreach ($aTargetClasses as $sSomeClass) {
$sTable = MetaModel::DBGetTable($sSomeClass);
$sPKField = MetaModel::DBGetKey($sSomeClass);
$sDeleteSQL = "DELETE FROM `$sTable` WHERE `$sPKField` IN ($sIdList)";
CMDBSource::DeleteFrom($sDeleteSQL);
}
$iNbIdsDeleted += $iNbIds;
}
// stop loop if query returned fewer objects than $iMaxChunkSize. In this case, all objects have been deleted.
if ($iNbIds < $iMaxChunkSize) {
$bExecuteQuery = false;
} }
} }
return count($aIds);
}
return $iNbIdsDeleted;
}
// Links // Links
// //
// //

26
tests/php-unit-tests/unitary-tests/core/MetaModelTest.php
View File

@@ -5,6 +5,7 @@ namespace Combodo\iTop\Test\UnitTest\Core;
use Combodo\iTop\Test\UnitTest\ItopDataTestCase; use Combodo\iTop\Test\UnitTest\ItopDataTestCase;
use CoreException; use CoreException;
use DBObjectSearch;
use MetaModel; use MetaModel;
/** /**
@@ -456,6 +457,31 @@ class MetaModelTest extends ItopDataTestCase
'Non existing person' => [10, false], 'Non existing person' => [10, false],
]; ];
} }
/**
* @return void
* @throws CoreException
* @throws \OQLException
*/
public function testPurgeData(){
// Set max_chunk_size to 2 (default 1000) to test chunk deletion with only 10 items
$oConfig = MetaModel::GetConfig();
$oConfig->Set('purge_data.max_chunk_size', 2);
MetaModel::SetConfig($oConfig);
$aPkPerson = [];
for ($i=0; $i < 10; $i++) {
$oPerson = $this->CreatePerson($i, 1);
$sClass = get_class($oPerson);
$aPkPerson[] = $oPerson->GetKey();
}
$sDeleteOQL = 'SELECT '.$sClass.' WHERE id IN ('.implode(',', $aPkPerson).')';
$oFilter = DBObjectSearch::FromOQL($sDeleteOQL);
$iNbDelete = MetaModel::PurgeData($oFilter);
$this->assertEquals($iNbDelete, 10, 'MetaModel::PurgeData must delete 10 objects per batch of 2 items');
}
} }
abstract class Wizzard abstract class Wizzard