From 66383ddaac8983d3ffd189918a52db0e8c6f7d78 Mon Sep 17 00:00:00 2001 From: Eric Espie Date: Fri, 1 Jul 2022 18:00:38 +0200 Subject: [PATCH] Cron parallelization # Conflicts: # webservices/cron.php --- core/config.class.inc.php | 16 +++--- webservices/cron.php | 116 ++++++++++++++++++++++---------------- 2 files changed, 75 insertions(+), 57 deletions(-) diff --git a/core/config.class.inc.php b/core/config.class.inc.php index 8dbc72ab4b..60543e404f 100644 --- a/core/config.class.inc.php +++ b/core/config.class.inc.php @@ -585,14 +585,6 @@ class Config 'source_of_value' => '', 'show_in_conf_sample' => true, ], - 'cron_task_max_execution_time' => [ - 'type' => 'integer', - 'description' => 'Background tasks will use this value (integer) multiplicated by its periodicity (in seconds) as max duration per cron execution. 0 is unlimited time', - 'default' => 0, - 'value' => 0, - 'source_of_value' => '', - 'show_in_conf_sample' => false, - ], 'cron_sleep' => [ 'type' => 'integer', 'description' => 'Duration (seconds) before cron.php checks again if something must be done', @@ -601,6 +593,14 @@ class Config 'source_of_value' => '', 'show_in_conf_sample' => false, ], + 'cron.max_process' => [ + 'type' => 'integer', + 'description' => 'Maximum number of cron process to run', + 'default' => 10, + 'value' => 10, + 'source_of_value' => '', + 'show_in_conf_sample' => true, + ], 'async_task_retries' => [ 'type' => 'array', 'description' => 'Automatic retries of asynchronous tasks in case of failure (per class)', diff --git a/webservices/cron.php b/webservices/cron.php index 2f1a576f78..5705ff6834 100644 --- a/webservices/cron.php +++ b/webservices/cron.php @@ -91,7 +91,6 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) $oProcess = new $TaskClass(); $oRefClass = new ReflectionClass(get_class($oProcess)); $oDateStarted = new DateTime(); - $oDatePlanned = new DateTime($oTask->Get('next_run_date')); $fStart = microtime(true); $oCtx = new ContextTag('CRON:Task:'.$TaskClass); @@ -99,25 +98,34 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) $oExceptionToThrow = null; try { // Record (when starting) that this task was started, just in case it crashes during the execution + if ($oTask->Get('total_exec_count') == 0) { + // First execution + $oTask->Set('first_run_date', $oDateStarted->format('Y-m-d H:i:s')); + } $oTask->Set('latest_run_date', $oDateStarted->format('Y-m-d H:i:s')); // Record the current user running the cron $oTask->Set('system_user', utils::GetCurrentUserName()); $oTask->Set('running', 1); - $oTask->DBUpdate(); - // Time in seconds allowed to the task - $iCurrTimeLimit = $iTimeLimit; - // Compute allowed time - if ($oRefClass->implementsInterface('iScheduledProcess') === false) { - // Periodic task, allow only X times ($iMaxTaskExecutionTime) its periodicity (GetPeriodicity()) - $iMaxTaskExecutionTime = MetaModel::GetConfig()->Get('cron_task_max_execution_time'); - $iTaskLimit = time() + $oProcess->GetPeriodicity() * $iMaxTaskExecutionTime; - // If our proposed time limit is less than cron limit, and cron_task_max_execution_time is > 0 - if ($iTaskLimit < $iTimeLimit && $iMaxTaskExecutionTime > 0) { - $iCurrTimeLimit = $iTaskLimit; + // Compute the next run date + if ($oRefClass->implementsInterface('iScheduledProcess')) { + // Schedules process do repeat at specific moments + $oPlannedStart = $oProcess->GetNextOccurrence(); + } else { + // Background processes do repeat periodically + $oDatePlanned = new DateTime($oTask->Get('next_run_date')); + $oPlannedStart = clone $oDatePlanned; + // Let's schedule from the previous planned date of execution to avoid shift + $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); + $oNow = new DateTime(); + while ($oPlannedStart->format('U') <= $oNow->format('U')) { + // Next planned start is already in the past, increase it again by a period + $oPlannedStart = $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); } } - $sMessage = $oProcess->Process($iCurrTimeLimit); - $oTask->Set('running', 0); + $oTask->Set('next_run_date', $oPlannedStart->format('Y-m-d H:i:s')); + $oTask->DBUpdate(); + + $sMessage = $oProcess->Process($iTimeLimit); } catch (MySQLHasGoneAwayException $e) { throw $e; } catch (ProcessFatalException $e) { @@ -128,35 +136,12 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) } else { $sMessage = 'Processing failed with message: '.$e->getMessage(); } + } finally { + $oTask->Set('running', 0); + $fDuration = microtime(true) - $fStart; + $oTask->ComputeDurations($fDuration); // does increment the counter and compute statistics + $oTask->DBUpdate(); } - $fDuration = microtime(true) - $fStart; - if ($oTask->Get('total_exec_count') == 0) { - // First execution - $oTask->Set('first_run_date', $oDateStarted->format('Y-m-d H:i:s')); - } - $oTask->ComputeDurations($fDuration); // does increment the counter and compute statistics - - // Update the timestamp since we want to be able to re-order the tasks based on the time they finished - $oDateEnded = new DateTime(); - $oTask->Set('latest_run_date', $oDateEnded->format('Y-m-d H:i:s')); - - if ($oRefClass->implementsInterface('iScheduledProcess')) { - // Schedules process do repeat at specific moments - $oPlannedStart = $oProcess->GetNextOccurrence(); - } else { - // Background processes do repeat periodically - $oPlannedStart = clone $oDatePlanned; - // Let's schedule from the previous planned date of execution to avoid shift - $oPlannedStart->modify($oProcess->GetPeriodicity().' seconds'); - $oEnd = new DateTime(); - while ($oPlannedStart->format('U') < $oEnd->format('U')) { - // Next planned start is already in the past, increase it again by a period - $oPlannedStart = $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); - } - } - - $oTask->Set('next_run_date', $oPlannedStart->format('Y-m-d H:i:s')); - $oTask->DBUpdate(); if ($oExceptionToThrow) { throw $oExceptionToThrow; @@ -190,6 +175,7 @@ function CronExec($oP, $bVerbose, $bDebug = false) $iMaxDuration = MetaModel::GetConfig()->Get('cron_max_execution_time'); $iTimeLimit = $iStarted + $iMaxDuration; $iCronSleep = MetaModel::GetConfig()->Get('cron_sleep'); + $iMaxCronProcess = MetaModel::GetConfig()->Get('cron.max_process'); if ($bVerbose) { $oP->p("Planned duration = $iMaxDuration seconds"); @@ -235,6 +221,19 @@ function CronExec($oP, $bVerbose, $bDebug = false) $aRunTasks = []; foreach ($aTasks as $oTask) { $sTaskClass = $oTask->Get('class_name'); + + // Check if the current task is running + $oTaskMutex = new iTopMutex("cron_$sTaskClass"); + if (!$oTaskMutex->TryLock()) { + // Task is already running, try next one + continue; + } + + // Just to indicate to Itop that the cron is running for setup + // The mutex will be released when the process dies + $oCronMutex = new iTopMutex('cron'); + $oCronMutex->TryLock(); + $aRunTasks[] = $sTaskClass; // N°3219 for each process will use a specific CMDBChange object with a specific track info @@ -255,6 +254,9 @@ function CronExec($oP, $bVerbose, $bDebug = false) $oP->p("ERROR : an exception was thrown when processing '$sTaskClass' (".$e->getInfoLog().")"); IssueLog::Error("Cron.php error : an exception was thrown when processing '$sTaskClass' (".$e->getInfoLog().')'); } + finally { + $oTaskMutex->Unlock(); + } if ($bVerbose) { if (!empty($sMessage)) { $oP->p("$sTaskClass: $sMessage"); @@ -267,6 +269,10 @@ function CronExec($oP, $bVerbose, $bDebug = false) break 2; } CheckMaintenanceMode($oP); + if ($iMaxCronProcess > 1) { + // Reindex tasks every time + break; + } } // Tasks to run later @@ -498,16 +504,28 @@ try { exit(EXIT_CODE_FATAL); } -try { - $oMutex = new iTopMutex('cron'); - if (!MetaModel::DBHasAccess(ACCESS_ADMIN_WRITE)) { +try +{ + if (!MetaModel::DBHasAccess(ACCESS_ADMIN_WRITE)) + { $oP->p("A maintenance is ongoing"); - } else { - if ($oMutex->TryLock()) { + } + else + { + // Limit the number of cron process to run in parallel + $iMaxCronProcess = MetaModel::GetConfig()->Get('cron.max_process'); + $bCanRun = false; + for ($i = 0; $i < $iMaxCronProcess; $i++) { + $oMutex = new iTopMutex("cron#$i"); + if ($oMutex->TryLock()) { + $bCanRun = true; + break; + } + } + if ($bCanRun) { CronExec($oP, $bVerbose, $bDebug); } else { - // Exit silently - $oP->p("Already running..."); + $oP->p("Already $iMaxCronProcess are running..."); } } } catch (Exception $e) {