diff --git a/application/utils.inc.php b/application/utils.inc.php index 4f0ded9679..68dd4ba4e9 100644 --- a/application/utils.inc.php +++ b/application/utils.inc.php @@ -3249,4 +3249,24 @@ TXT return (int)$sLimit; } } + + /** + * Count the number of cron.php processes currently running + * + * @return int process count + * @throws \Exception + */ + public static function CountCronRunning(): int + { + $iMaxCronProcess = max(MetaModel::GetConfig()->Get('cron.max_processes'), 1); + $iCount = 0; + for ($i = 0; $i < $iMaxCronProcess; $i++) { + $oMutex = new iTopMutex("cron#$i"); + if ($oMutex->IsLocked()) { + $iCount++; + } + } + + return $iCount; + } } diff --git a/core/config.class.inc.php b/core/config.class.inc.php index fae11cc153..fdad731f6d 100644 --- a/core/config.class.inc.php +++ b/core/config.class.inc.php @@ -578,19 +578,11 @@ class Config 'cron_max_execution_time' => [ 'type' => 'integer', 'description' => 'Duration (seconds) of the cron.php script : if exceeded the script will exit even if there are remaining tasks to process. Must be shorter than php max_execution_time setting (note than when using CLI, this is set to 0 by default which means unlimited). If cron.php is ran via web, it must be shorter than the web server response timeout.', - 'default' => 600, - 'value' => 600, + 'default' => 590, + 'value' => 590, 'source_of_value' => '', 'show_in_conf_sample' => true, ], - 'cron_task_max_execution_time' => [ - 'type' => 'integer', - 'description' => 'Background tasks will use this value (integer) multiplicated by its periodicity (in seconds) as max duration per cron execution. 0 is unlimited time', - 'default' => 0, - 'value' => 0, - 'source_of_value' => '', - 'show_in_conf_sample' => false, - ], 'cron_sleep' => [ 'type' => 'integer', 'description' => 'Duration (seconds) before cron.php checks again if something must be done', @@ -599,6 +591,14 @@ class Config 'source_of_value' => '', 'show_in_conf_sample' => false, ], + 'cron.max_processes' => [ + 'type' => 'integer', + 'description' => 'Maximum number of cron processes to run', + 'default' => 10, + 'value' => 10, + 'source_of_value' => '', + 'show_in_conf_sample' => true, + ], 'async_task_retries' => [ 'type' => 'array', 'description' => 'Automatic retries of asynchronous tasks in case of failure (per class)', diff --git a/core/inlineimage.class.inc.php b/core/inlineimage.class.inc.php index f919f517be..27864afd36 100644 --- a/core/inlineimage.class.inc.php +++ b/core/inlineimage.class.inc.php @@ -563,7 +563,7 @@ class InlineImageGC implements iBackgroundProcess */ public function GetPeriodicity() { - return 1; + return 600; } /** diff --git a/lib/composer/autoload_classmap.php b/lib/composer/autoload_classmap.php index 7b0a6ad759..6b8d6b15f3 100644 --- a/lib/composer/autoload_classmap.php +++ b/lib/composer/autoload_classmap.php @@ -614,6 +614,7 @@ return array( 'Combodo\\iTop\\Service\\Base\\ObjectRepository' => $baseDir . '/sources/Service/Base/ObjectRepository.php', 'Combodo\\iTop\\Service\\Base\\iDataPostProcessor' => $baseDir . '/sources/Service/Base/iDataPostProcessor.php', 'Combodo\\iTop\\Service\\Cache\\DataModelDependantCache' => $baseDir . '/sources/Service/Cache/DataModelDependantCache.php', + 'Combodo\\iTop\\Service\\Cron\\CronLog' => $baseDir . '/sources/Service/Cron/CronLog.php', 'Combodo\\iTop\\Service\\DependencyInjection\\DIException' => $baseDir . '/sources/Service/DependencyInjection/DIException.php', 'Combodo\\iTop\\Service\\DependencyInjection\\ServiceLocator' => $baseDir . '/sources/Service/DependencyInjection/ServiceLocator.php', 'Combodo\\iTop\\Service\\Events\\Description\\EventDataDescription' => $baseDir . '/sources/Service/Events/Description/EventDataDescription.php', diff --git a/lib/composer/autoload_static.php b/lib/composer/autoload_static.php index 292ff47043..13f45db790 100644 --- a/lib/composer/autoload_static.php +++ b/lib/composer/autoload_static.php @@ -28,12 +28,12 @@ class ComposerStaticInitfc0e9e9dea11dcbb6272414776c30685 ); public static $prefixLengthsPsr4 = array ( - 'T' => + 'T' => array ( 'Twig\\' => 5, 'TheNetworg\\OAuth2\\Client\\' => 25, ), - 'S' => + 'S' => array ( 'Symfony\\Runtime\\Symfony\\Component\\' => 34, 'Symfony\\Polyfill\\Php83\\' => 23, @@ -85,7 +85,7 @@ class ComposerStaticInitfc0e9e9dea11dcbb6272414776c30685 'ScssPhp\\ScssPhp\\' => 16, 'Sabberworm\\CSS\\' => 15, ), - 'P' => + 'P' => array ( 'Psr\\Log\\' => 8, 'Psr\\Http\\Message\\' => 17, @@ -96,310 +96,310 @@ class ComposerStaticInitfc0e9e9dea11dcbb6272414776c30685 'PhpParser\\' => 10, 'Pelago\\Emogrifier\\' => 18, ), - 'L' => + 'L' => array ( 'League\\OAuth2\\Client\\' => 21, ), - 'G' => + 'G' => array ( 'GuzzleHttp\\Psr7\\' => 16, 'GuzzleHttp\\Promise\\' => 19, 'GuzzleHttp\\' => 11, ), - 'F' => + 'F' => array ( 'Firebase\\JWT\\' => 13, ), - 'E' => + 'E' => array ( 'Egulias\\EmailValidator\\' => 23, ), - 'D' => + 'D' => array ( 'Doctrine\\Common\\Lexer\\' => 22, ), ); public static $prefixDirsPsr4 = array ( - 'Twig\\' => + 'Twig\\' => array ( 0 => __DIR__ . '/..' . '/twig/twig/src', ), - 'TheNetworg\\OAuth2\\Client\\' => + 'TheNetworg\\OAuth2\\Client\\' => array ( 0 => __DIR__ . '/..' . '/thenetworg/oauth2-azure/src', ), - 'Symfony\\Runtime\\Symfony\\Component\\' => + 'Symfony\\Runtime\\Symfony\\Component\\' => array ( 0 => __DIR__ . '/..' . '/symfony/runtime/Internal', ), - 'Symfony\\Polyfill\\Php83\\' => + 'Symfony\\Polyfill\\Php83\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-php83', ), - 'Symfony\\Polyfill\\Mbstring\\' => + 'Symfony\\Polyfill\\Mbstring\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-mbstring', ), - 'Symfony\\Polyfill\\Intl\\Normalizer\\' => + 'Symfony\\Polyfill\\Intl\\Normalizer\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-intl-normalizer', ), - 'Symfony\\Polyfill\\Intl\\Idn\\' => + 'Symfony\\Polyfill\\Intl\\Idn\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-intl-idn', ), - 'Symfony\\Polyfill\\Intl\\Icu\\' => + 'Symfony\\Polyfill\\Intl\\Icu\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-intl-icu', ), - 'Symfony\\Polyfill\\Intl\\Grapheme\\' => + 'Symfony\\Polyfill\\Intl\\Grapheme\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-intl-grapheme', ), - 'Symfony\\Polyfill\\Ctype\\' => + 'Symfony\\Polyfill\\Ctype\\' => array ( 0 => __DIR__ . '/..' . '/symfony/polyfill-ctype', ), - 'Symfony\\Contracts\\Translation\\' => + 'Symfony\\Contracts\\Translation\\' => array ( 0 => __DIR__ . '/..' . '/symfony/translation-contracts', ), - 'Symfony\\Contracts\\Service\\' => + 'Symfony\\Contracts\\Service\\' => array ( 0 => __DIR__ . '/..' . '/symfony/service-contracts', ), - 'Symfony\\Contracts\\EventDispatcher\\' => + 'Symfony\\Contracts\\EventDispatcher\\' => array ( 0 => __DIR__ . '/..' . '/symfony/event-dispatcher-contracts', ), - 'Symfony\\Contracts\\Cache\\' => + 'Symfony\\Contracts\\Cache\\' => array ( 0 => __DIR__ . '/..' . '/symfony/cache-contracts', ), - 'Symfony\\Component\\Yaml\\' => + 'Symfony\\Component\\Yaml\\' => array ( 0 => __DIR__ . '/..' . '/symfony/yaml', ), - 'Symfony\\Component\\VarExporter\\' => + 'Symfony\\Component\\VarExporter\\' => array ( 0 => __DIR__ . '/..' . '/symfony/var-exporter', ), - 'Symfony\\Component\\VarDumper\\' => + 'Symfony\\Component\\VarDumper\\' => array ( 0 => __DIR__ . '/..' . '/symfony/var-dumper', ), - 'Symfony\\Component\\Validator\\' => + 'Symfony\\Component\\Validator\\' => array ( 0 => __DIR__ . '/..' . '/symfony/validator', ), - 'Symfony\\Component\\String\\' => + 'Symfony\\Component\\String\\' => array ( 0 => __DIR__ . '/..' . '/symfony/string', ), - 'Symfony\\Component\\Stopwatch\\' => + 'Symfony\\Component\\Stopwatch\\' => array ( 0 => __DIR__ . '/..' . '/symfony/stopwatch', ), - 'Symfony\\Component\\Security\\Csrf\\' => + 'Symfony\\Component\\Security\\Csrf\\' => array ( 0 => __DIR__ . '/..' . '/symfony/security-csrf', ), - 'Symfony\\Component\\Security\\Core\\' => + 'Symfony\\Component\\Security\\Core\\' => array ( 0 => __DIR__ . '/..' . '/symfony/security-core', ), - 'Symfony\\Component\\Runtime\\' => + 'Symfony\\Component\\Runtime\\' => array ( 0 => __DIR__ . '/..' . '/symfony/runtime', ), - 'Symfony\\Component\\Routing\\' => + 'Symfony\\Component\\Routing\\' => array ( 0 => __DIR__ . '/..' . '/symfony/routing', ), - 'Symfony\\Component\\PropertyInfo\\' => + 'Symfony\\Component\\PropertyInfo\\' => array ( 0 => __DIR__ . '/..' . '/symfony/property-info', ), - 'Symfony\\Component\\PropertyAccess\\' => + 'Symfony\\Component\\PropertyAccess\\' => array ( 0 => __DIR__ . '/..' . '/symfony/property-access', ), - 'Symfony\\Component\\PasswordHasher\\' => + 'Symfony\\Component\\PasswordHasher\\' => array ( 0 => __DIR__ . '/..' . '/symfony/password-hasher', ), - 'Symfony\\Component\\OptionsResolver\\' => + 'Symfony\\Component\\OptionsResolver\\' => array ( 0 => __DIR__ . '/..' . '/symfony/options-resolver', ), - 'Symfony\\Component\\Mime\\' => + 'Symfony\\Component\\Mime\\' => array ( 0 => __DIR__ . '/..' . '/symfony/mime', ), - 'Symfony\\Component\\Mailer\\' => + 'Symfony\\Component\\Mailer\\' => array ( 0 => __DIR__ . '/..' . '/symfony/mailer', ), - 'Symfony\\Component\\HttpKernel\\' => + 'Symfony\\Component\\HttpKernel\\' => array ( 0 => __DIR__ . '/..' . '/symfony/http-kernel', ), - 'Symfony\\Component\\HttpFoundation\\' => + 'Symfony\\Component\\HttpFoundation\\' => array ( 0 => __DIR__ . '/..' . '/symfony/http-foundation', ), - 'Symfony\\Component\\Form\\' => + 'Symfony\\Component\\Form\\' => array ( 0 => __DIR__ . '/..' . '/symfony/form', ), - 'Symfony\\Component\\Finder\\' => + 'Symfony\\Component\\Finder\\' => array ( 0 => __DIR__ . '/..' . '/symfony/finder', ), - 'Symfony\\Component\\Filesystem\\' => + 'Symfony\\Component\\Filesystem\\' => array ( 0 => __DIR__ . '/..' . '/symfony/filesystem', ), - 'Symfony\\Component\\EventDispatcher\\' => + 'Symfony\\Component\\EventDispatcher\\' => array ( 0 => __DIR__ . '/..' . '/symfony/event-dispatcher', ), - 'Symfony\\Component\\ErrorHandler\\' => + 'Symfony\\Component\\ErrorHandler\\' => array ( 0 => __DIR__ . '/..' . '/symfony/error-handler', ), - 'Symfony\\Component\\Dotenv\\' => + 'Symfony\\Component\\Dotenv\\' => array ( 0 => __DIR__ . '/..' . '/symfony/dotenv', ), - 'Symfony\\Component\\DependencyInjection\\' => + 'Symfony\\Component\\DependencyInjection\\' => array ( 0 => __DIR__ . '/..' . '/symfony/dependency-injection', ), - 'Symfony\\Component\\CssSelector\\' => + 'Symfony\\Component\\CssSelector\\' => array ( 0 => __DIR__ . '/..' . '/symfony/css-selector', ), - 'Symfony\\Component\\Console\\' => + 'Symfony\\Component\\Console\\' => array ( 0 => __DIR__ . '/..' . '/symfony/console', ), - 'Symfony\\Component\\Config\\' => + 'Symfony\\Component\\Config\\' => array ( 0 => __DIR__ . '/..' . '/symfony/config', ), - 'Symfony\\Component\\Cache\\' => + 'Symfony\\Component\\Cache\\' => array ( 0 => __DIR__ . '/..' . '/symfony/cache', ), - 'Symfony\\Bundle\\WebProfilerBundle\\' => + 'Symfony\\Bundle\\WebProfilerBundle\\' => array ( 0 => __DIR__ . '/..' . '/symfony/web-profiler-bundle', ), - 'Symfony\\Bundle\\TwigBundle\\' => + 'Symfony\\Bundle\\TwigBundle\\' => array ( 0 => __DIR__ . '/..' . '/symfony/twig-bundle', ), - 'Symfony\\Bundle\\FrameworkBundle\\' => + 'Symfony\\Bundle\\FrameworkBundle\\' => array ( 0 => __DIR__ . '/..' . '/symfony/framework-bundle', ), - 'Symfony\\Bundle\\DebugBundle\\' => + 'Symfony\\Bundle\\DebugBundle\\' => array ( 0 => __DIR__ . '/..' . '/symfony/debug-bundle', ), - 'Symfony\\Bridge\\Twig\\' => + 'Symfony\\Bridge\\Twig\\' => array ( 0 => __DIR__ . '/..' . '/symfony/twig-bridge', ), - 'Soundasleep\\' => + 'Soundasleep\\' => array ( 0 => __DIR__ . '/..' . '/soundasleep/html2text/src', ), - 'ScssPhp\\ScssPhp\\' => + 'ScssPhp\\ScssPhp\\' => array ( 0 => __DIR__ . '/..' . '/scssphp/scssphp/src', ), - 'Sabberworm\\CSS\\' => + 'Sabberworm\\CSS\\' => array ( 0 => __DIR__ . '/..' . '/sabberworm/php-css-parser/src', ), - 'Psr\\Log\\' => + 'Psr\\Log\\' => array ( 0 => __DIR__ . '/..' . '/psr/log/src', ), - 'Psr\\Http\\Message\\' => + 'Psr\\Http\\Message\\' => array ( 0 => __DIR__ . '/..' . '/psr/http-factory/src', 1 => __DIR__ . '/..' . '/psr/http-message/src', ), - 'Psr\\Http\\Client\\' => + 'Psr\\Http\\Client\\' => array ( 0 => __DIR__ . '/..' . '/psr/http-client/src', ), - 'Psr\\EventDispatcher\\' => + 'Psr\\EventDispatcher\\' => array ( 0 => __DIR__ . '/..' . '/psr/event-dispatcher/src', ), - 'Psr\\Container\\' => + 'Psr\\Container\\' => array ( 0 => __DIR__ . '/..' . '/psr/container/src', ), - 'Psr\\Cache\\' => + 'Psr\\Cache\\' => array ( 0 => __DIR__ . '/..' . '/psr/cache/src', ), - 'PhpParser\\' => + 'PhpParser\\' => array ( 0 => __DIR__ . '/..' . '/nikic/php-parser/lib/PhpParser', ), - 'Pelago\\Emogrifier\\' => + 'Pelago\\Emogrifier\\' => array ( 0 => __DIR__ . '/..' . '/pelago/emogrifier/src', ), - 'League\\OAuth2\\Client\\' => + 'League\\OAuth2\\Client\\' => array ( 0 => __DIR__ . '/..' . '/league/oauth2-google/src', 1 => __DIR__ . '/..' . '/league/oauth2-client/src', ), - 'GuzzleHttp\\Psr7\\' => + 'GuzzleHttp\\Psr7\\' => array ( 0 => __DIR__ . '/..' . '/guzzlehttp/psr7/src', ), - 'GuzzleHttp\\Promise\\' => + 'GuzzleHttp\\Promise\\' => array ( 0 => __DIR__ . '/..' . '/guzzlehttp/promises/src', ), - 'GuzzleHttp\\' => + 'GuzzleHttp\\' => array ( 0 => __DIR__ . '/..' . '/guzzlehttp/guzzle/src', ), - 'Firebase\\JWT\\' => + 'Firebase\\JWT\\' => array ( 0 => __DIR__ . '/..' . '/firebase/php-jwt/src', ), - 'Egulias\\EmailValidator\\' => + 'Egulias\\EmailValidator\\' => array ( 0 => __DIR__ . '/..' . '/egulias/email-validator/src', ), - 'Doctrine\\Common\\Lexer\\' => + 'Doctrine\\Common\\Lexer\\' => array ( 0 => __DIR__ . '/..' . '/doctrine/lexer/src', ), ); public static $prefixesPsr0 = array ( - 'C' => + 'C' => array ( - 'Console' => + 'Console' => array ( 0 => __DIR__ . '/..' . '/pear/console_getopt', ), ), - 'A' => + 'A' => array ( - 'Archive_Tar' => + 'Archive_Tar' => array ( 0 => __DIR__ . '/..' . '/pear/archive_tar', ), @@ -1015,6 +1015,7 @@ class ComposerStaticInitfc0e9e9dea11dcbb6272414776c30685 'Combodo\\iTop\\Service\\Base\\ObjectRepository' => __DIR__ . '/../..' . '/sources/Service/Base/ObjectRepository.php', 'Combodo\\iTop\\Service\\Base\\iDataPostProcessor' => __DIR__ . '/../..' . '/sources/Service/Base/iDataPostProcessor.php', 'Combodo\\iTop\\Service\\Cache\\DataModelDependantCache' => __DIR__ . '/../..' . '/sources/Service/Cache/DataModelDependantCache.php', + 'Combodo\\iTop\\Service\\Cron\\CronLog' => __DIR__ . '/../..' . '/sources/Service/Cron/CronLog.php', 'Combodo\\iTop\\Service\\DependencyInjection\\DIException' => __DIR__ . '/../..' . '/sources/Service/DependencyInjection/DIException.php', 'Combodo\\iTop\\Service\\DependencyInjection\\ServiceLocator' => __DIR__ . '/../..' . '/sources/Service/DependencyInjection/ServiceLocator.php', 'Combodo\\iTop\\Service\\Events\\Description\\EventDataDescription' => __DIR__ . '/../..' . '/sources/Service/Events/Description/EventDataDescription.php', diff --git a/setup/setuputils.class.inc.php b/setup/setuputils.class.inc.php index ed17756afb..ac853550be 100644 --- a/setup/setuputils.class.inc.php +++ b/setup/setuputils.class.inc.php @@ -2069,25 +2069,45 @@ JS */ private static function WaitCronTermination($oConfig, $sMode) { + $iMaxDuration = $oConfig->Get('cron_max_execution_time'); + // Avoid PHP stopping while waiting the cron + set_time_limit($iMaxDuration); try { // Wait for cron to stop if (is_null($oConfig) || ContextTag::Check(ContextTag::TAG_CRON)) { return; } - // Use mutex to check if cron is running - $oMutex = self::GetCronMutex($oConfig); + // Limit the number of cron process to run in parallel + $iMaxCronProcess = $oConfig->Get('cron.max_processes'); $iCount = 1; - $iStarted = time(); - $iMaxDuration = $oConfig->Get('cron_max_execution_time'); - $iTimeLimit = $iStarted + $iMaxDuration; - while ($oMutex->IsLocked()) { - SetupLog::Info("Waiting for cron to stop ($iCount)"); - $iCount++; - sleep(1); - if (time() > $iTimeLimit) { - throw new Exception("Cannot enter $sMode mode, consider stopping the cron temporarily"); + $iTimeLimit = time() + $iMaxDuration; + do { + $bIsRunning = false; + // Use all mutexes to check if cron is running + for ($i = 0; $i < $iMaxCronProcess; $i++) { + $sName = "cron#$i"; + + $oMutex = new iTopMutex( + $sName.$oConfig->Get('db_name').$oConfig->Get('db_subname'), + $oConfig->Get('db_host'), + $oConfig->Get('db_user'), + $oConfig->Get('db_pwd'), + $oConfig->Get('db_tls.enabled'), + $oConfig->Get('db_tls.ca') + ); + if ($oMutex->IsLocked()) { + $bIsRunning = true; + SetupLog::Info("Waiting for cron to stop ($iCount)"); + $iCount++; + sleep(1); + if (time() > $iTimeLimit) { + SetupLog::Error("Cannot enter $sMode mode, consider stopping the cron temporarily"); + throw new Exception("Cannot enter $sMode mode, consider stopping the cron temporarily"); + } + break; + } } - } + } while ($bIsRunning); } catch (Exception $e) { // Ignore errors } diff --git a/sources/Service/Cron/CronLog.php b/sources/Service/Cron/CronLog.php new file mode 100644 index 0000000000..25159e21ef --- /dev/null +++ b/sources/Service/Cron/CronLog.php @@ -0,0 +1,57 @@ +p("ERROR: Missing argument '$sParam'\n"); + CronLog::Error("ERROR: Missing argument '$sParam'\n"); UsageAndExit($oP); } @@ -63,7 +64,7 @@ function UsageAndExit($oP) if ($bModeCLI) { $oP->p("USAGE:\n"); - $oP->p("php cron.php --auth_user= --auth_pwd= [--param_file=] [--verbose=1] [--debug=1] [--status_only=1]\n"); + $oP->p("php cron.php --auth_user= --auth_pwd= [--param_file=] [--verbose=0] [--status_only=1]\n"); } else { $oP->p("Optional parameters: verbose, param_file, status_only\n"); } @@ -91,7 +92,6 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) $oProcess = new $TaskClass(); $oRefClass = new ReflectionClass(get_class($oProcess)); $oDateStarted = new DateTime(); - $oDatePlanned = new DateTime($oTask->Get('next_run_date')); $fStart = microtime(true); $oCtx = new ContextTag('CRON:Task:'.$TaskClass); @@ -99,25 +99,34 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) $oExceptionToThrow = null; try { // Record (when starting) that this task was started, just in case it crashes during the execution + if ($oTask->Get('total_exec_count') == 0) { + // First execution + $oTask->Set('first_run_date', $oDateStarted->format('Y-m-d H:i:s')); + } $oTask->Set('latest_run_date', $oDateStarted->format('Y-m-d H:i:s')); // Record the current user running the cron $oTask->Set('system_user', utils::GetCurrentUserName()); $oTask->Set('running', 1); - $oTask->DBUpdate(); - // Time in seconds allowed to the task - $iCurrTimeLimit = $iTimeLimit; - // Compute allowed time - if ($oRefClass->implementsInterface('iScheduledProcess') === false) { - // Periodic task, allow only X times ($iMaxTaskExecutionTime) its periodicity (GetPeriodicity()) - $iMaxTaskExecutionTime = MetaModel::GetConfig()->Get('cron_task_max_execution_time'); - $iTaskLimit = time() + $oProcess->GetPeriodicity() * $iMaxTaskExecutionTime; - // If our proposed time limit is less than cron limit, and cron_task_max_execution_time is > 0 - if ($iTaskLimit < $iTimeLimit && $iMaxTaskExecutionTime > 0) { - $iCurrTimeLimit = $iTaskLimit; + // Compute the next run date + if ($oRefClass->implementsInterface('iScheduledProcess')) { + // Schedules process do repeat at specific moments + $oPlannedStart = $oProcess->GetNextOccurrence(); + } else { + // Background processes do repeat periodically + $oDatePlanned = new DateTime($oTask->Get('next_run_date')); + $oPlannedStart = clone $oDatePlanned; + // Let's schedule from the previous planned date of execution to avoid shift + $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); + $oNow = new DateTime(); + while ($oPlannedStart->format('U') <= $oNow->format('U')) { + // Next planned start is already in the past, increase it again by a period + $oPlannedStart = $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); } } - $sMessage = $oProcess->Process($iCurrTimeLimit); - $oTask->Set('running', 0); + $oTask->Set('next_run_date', $oPlannedStart->format('Y-m-d H:i:s')); + $oTask->DBUpdate(); + + $sMessage = $oProcess->Process($iTimeLimit); } catch (MySQLHasGoneAwayException $e) { throw $e; } catch (ProcessFatalException $e) { @@ -128,35 +137,12 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) } else { $sMessage = 'Processing failed with message: '.$e->getMessage(); } + } finally { + $oTask->Set('running', 0); + $fDuration = microtime(true) - $fStart; + $oTask->ComputeDurations($fDuration); // does increment the counter and compute statistics + $oTask->DBUpdate(); } - $fDuration = microtime(true) - $fStart; - if ($oTask->Get('total_exec_count') == 0) { - // First execution - $oTask->Set('first_run_date', $oDateStarted->format('Y-m-d H:i:s')); - } - $oTask->ComputeDurations($fDuration); // does increment the counter and compute statistics - - // Update the timestamp since we want to be able to re-order the tasks based on the time they finished - $oDateEnded = new DateTime(); - $oTask->Set('latest_run_date', $oDateEnded->format('Y-m-d H:i:s')); - - if ($oRefClass->implementsInterface('iScheduledProcess')) { - // Schedules process do repeat at specific moments - $oPlannedStart = $oProcess->GetNextOccurrence(); - } else { - // Background processes do repeat periodically - $oPlannedStart = clone $oDatePlanned; - // Let's schedule from the previous planned date of execution to avoid shift - $oPlannedStart->modify($oProcess->GetPeriodicity().' seconds'); - $oEnd = new DateTime(); - while ($oPlannedStart->format('U') < $oEnd->format('U')) { - // Next planned start is already in the past, increase it again by a period - $oPlannedStart = $oPlannedStart->modify('+'.$oProcess->GetPeriodicity().' seconds'); - } - } - - $oTask->Set('next_run_date', $oPlannedStart->format('Y-m-d H:i:s')); - $oTask->DBUpdate(); if ($oExceptionToThrow) { throw $oExceptionToThrow; @@ -168,8 +154,6 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) } /** - * @param CLIPage|WebPage $oP - * @param boolean $bVerbose * * @param bool $bDebug * @@ -184,22 +168,29 @@ function RunTask(BackgroundTask $oTask, $iTimeLimit) * @throws \OQLException * @throws \ReflectionException */ -function CronExec($oP, $bVerbose, $bDebug = false) +function CronExec($bDebug) { $iStarted = time(); $iMaxDuration = MetaModel::GetConfig()->Get('cron_max_execution_time'); $iTimeLimit = $iStarted + $iMaxDuration; $iCronSleep = MetaModel::GetConfig()->Get('cron_sleep'); + $iMaxCronProcess = max(MetaModel::GetConfig()->Get('cron.max_processes'), 1); - if ($bVerbose) { - $oP->p("Planned duration = $iMaxDuration seconds"); - $oP->p("Loop pause = $iCronSleep seconds"); + CronLog::Trace("Planned duration = $iMaxDuration seconds"); + CronLog::Trace("Loop pause = $iCronSleep seconds"); + + $oResyncMutex = new iTopMutex("cron_resync_processes"); + try { + if ($oResyncMutex->TryLock()) { + ReSyncProcesses($bDebug); + $oResyncMutex->Unlock(); + } + } catch (Exception $e) { + CronLog::Error('Error: '.$e->GetMessage(), CronLog::CHANNEL_DEFAULT, ['stack' => $e->getTraceAsString()]); } - ReSyncProcesses($oP, $bVerbose, $bDebug); - while (time() < $iTimeLimit) { - CheckMaintenanceMode($oP); + CheckMaintenanceMode(); $oNow = new DateTime(); $sNow = $oNow->format('Y-m-d H:i:s'); @@ -207,103 +198,104 @@ function CronExec($oP, $bVerbose, $bDebug = false) $oSearch->AddCondition('next_run_date', $sNow, '<='); $oSearch->AddCondition('status', 'active'); $oTasks = new DBObjectSet($oSearch, ['next_run_date' => true]); - $bWorkDone = false; + $aTasks = []; if ($oTasks->CountExceeds(0)) { - $bWorkDone = true; - $aTasks = []; - if ($bVerbose) { - $sCount = $oTasks->Count(); - $oP->p("$sCount Tasks planned to run now ($sNow):"); - $oP->p('+---------------------------+---------+---------------------+---------------------+'); - $oP->p('| Task Class | Status | Last Run | Next Run |'); - $oP->p('+---------------------------+---------+---------------------+---------------------+'); - } + $aDebugMessages = []; while ($oTask = $oTasks->Fetch()) { - $aTasks[$oTask->Get('class_name')] = $oTask; - if ($bVerbose) { - $sTaskName = $oTask->Get('class_name'); - $sStatus = $oTask->Get('status'); - $sLastRunDate = $oTask->Get('latest_run_date'); - $sNextRunDate = $oTask->Get('next_run_date'); - $oP->p(sprintf('| %1$-25.25s | %2$-7s | %3$-19s | %4$-19s |', $sTaskName, $sStatus, $sLastRunDate, $sNextRunDate)); + $sTaskName = $oTask->Get('class_name'); + $oTaskMutex = new iTopMutex("cron_$sTaskName"); + if ($oTaskMutex->IsLocked()) { + // Already running, ignore + continue; } + $aTasks[] = $oTask; + $sStatus = $oTask->Get('status'); + $sLastRunDate = $oTask->Get('latest_run_date'); + $sNextRunDate = $oTask->Get('next_run_date'); + $aDebugMessages[] = sprintf('Task Class: %1$-25.25s Status: %2$-7s Last Run: %3$-19s Next Run: %4$-19s', $sTaskName, $sStatus, $sLastRunDate, $sNextRunDate); } - if ($bVerbose) { - $oP->p('+---------------------------+---------+---------------------+---------------------+'); + $sCount = count($aDebugMessages); + CronLog::Trace("$sCount Tasks planned to run now ($sNow):"); + foreach ($aDebugMessages as $sDebugMessage) { + CronLog::Trace($sDebugMessage); } $aRunTasks = []; - foreach ($aTasks as $oTask) { + while (count($aTasks) > 0) { + $oTask = array_shift($aTasks); $sTaskClass = $oTask->Get('class_name'); + + // Check if the current task is running + $oTaskMutex = new iTopMutex("cron_$sTaskClass"); + if (!$oTaskMutex->TryLock()) { + // Task is already running, try next one + continue; + } + $aRunTasks[] = $sTaskClass; // N°3219 for each process will use a specific CMDBChange object with a specific track info - // Any BackgroundProcess can overrides this as needed + // Any BackgroundProcess can override this as needed CMDBObject::SetCurrentChangeFromParams("Background task ($sTaskClass)"); // Run the task and record its next run time - if ($bVerbose) { - $oNow = new DateTime(); - $oP->p(">> === ".$oNow->format('Y-m-d H:i:s').sprintf(" Starting:%-'=49s", ' '.$sTaskClass.' ')); - } + $sDebugTaskClass = CronLog::GetDebugClassName($sTaskClass); + $oNow = new DateTime(); + CronLog::Debug(sprintf("> Starting >>> %-'>49s", $sDebugTaskClass.' ')); try { - $sMessage = RunTask($aTasks[$sTaskClass], $iTimeLimit); + // The limit of time for this task corresponds to the time slot allowed for every task + // but limited to the cron job time limit + $sMessage = RunTask($oTask, $iTimeLimit); } catch (MySQLHasGoneAwayException $e) { - $oP->p("ERROR : 'MySQL has gone away' thrown when processing $sTaskClass (error_code=".$e->getCode().")"); + CronLog::Error("ERROR : 'MySQL has gone away' thrown when processing $sDebugTaskClass (error_code=".$e->getCode().")", CronLog::CHANNEL_DEFAULT, ['stack' => $e->getTraceAsString()]); exit(EXIT_CODE_FATAL); } catch (ProcessFatalException $e) { - $oP->p("ERROR : an exception was thrown when processing '$sTaskClass' (".$e->getInfoLog().")"); - IssueLog::Error("Cron.php error : an exception was thrown when processing '$sTaskClass' (".$e->getInfoLog().')'); + CronLog::Error("ERROR : an exception was thrown when processing '$sDebugTaskClass' (".$e->getInfoLog().")", CronLog::CHANNEL_DEFAULT, ['stack' => $e->getTraceAsString()]); + } finally { + $oTaskMutex->Unlock(); } - if ($bVerbose) { - if (!empty($sMessage)) { - $oP->p("$sTaskClass: $sMessage"); - } - $oEnd = new DateTime(); - $sNextRunDate = $oTask->Get('next_run_date'); - $oP->p("<< === ".$oEnd->format('Y-m-d H:i:s').sprintf(" End of: %-'=42s", ' '.$sTaskClass.' ')." Next: $sNextRunDate"); + if (!empty($sMessage)) { + CronLog::Debug("$sDebugTaskClass: $sMessage"); } + $sNextRunDate = $oTask->Get('next_run_date'); + CronLog::Debug(sprintf("< Ending <<<<< %-'<49s", $sDebugTaskClass.' ')." Next: $sNextRunDate"); if (time() > $iTimeLimit) { break 2; } - CheckMaintenanceMode($oP); + CheckMaintenanceMode(); + if ($iMaxCronProcess > 1) { + // Reindex tasks every time + break; + } } // Tasks to run later - if ($bVerbose) { - $oP->p('--'); + if (count($aTasks) == 0) { $oSearch = new DBObjectSearch('BackgroundTask'); $oSearch->AddCondition('next_run_date', $sNow, '>'); $oSearch->AddCondition('status', 'active'); $oTasks = new DBObjectSet($oSearch, ['next_run_date' => true]); while ($oTask = $oTasks->Fetch()) { if (!in_array($oTask->Get('class_name'), $aRunTasks)) { - $oP->p(sprintf("-- Skipping task: %-'-40s", $oTask->Get('class_name').' ')." until: ".$oTask->Get('next_run_date')); + $sDebugTaskClass = CronLog::GetDebugClassName($oTask->Get('class_name')); + CronLog::Trace(sprintf("-- Skipping task: %-'-40s", $sDebugTaskClass.' ')." until: ".$oTask->Get('next_run_date')); } } } } - - if ($bVerbose && $bWorkDone) { - $oP->p("Sleeping...\n"); + if (count($aTasks) == 0) { + CronLog::Trace("sleeping..."); + sleep($iCronSleep); } - sleep($iCronSleep); - } - if ($bVerbose) { - $oP->p(''); - DisplayStatus($oP, ['next_run_date' => true]); - $oP->p("Reached normal execution time limit (exceeded by ".(time() - $iTimeLimit)."s)"); } + CronLog::Trace("Reached normal execution time limit (exceeded by ".(time() - $iTimeLimit)."s)"); } -/** - * @param WebPage $oP - */ -function CheckMaintenanceMode(Page $oP) +function CheckMaintenanceMode() { // Verify files instead of reloading the full config each time if (file_exists(MAINTENANCE_MODE_FILE) || file_exists(READONLY_MODE_FILE)) { - $oP->p("Maintenance detected, exiting"); + CronLog::Info("Maintenance detected, exiting"); exit(EXIT_CODE_ERROR); } } @@ -318,7 +310,7 @@ function CheckMaintenanceMode(Page $oP) * @throws \MySQLException * @throws \OQLException */ -function DisplayStatus($oP, $aTaskOrderBy = []) +function DisplayStatus($oP = null, $aTaskOrderBy = []) { $oSearch = new DBObjectSearch('BackgroundTask'); $oTasks = new DBObjectSet($oSearch, $aTaskOrderBy); @@ -346,8 +338,6 @@ function DisplayStatus($oP, $aTaskOrderBy = []) } /** - * @param $oP - * @param $bVerbose * @param $bDebug * * @throws \ArchivedObjectException @@ -359,7 +349,7 @@ function DisplayStatus($oP, $aTaskOrderBy = []) * @throws \OQLException * @throws \ReflectionException */ -function ReSyncProcesses($oP, $bVerbose, $bDebug) +function ReSyncProcesses($bDebug) { // Enumerate classes implementing BackgroundProcess // @@ -394,10 +384,9 @@ function ReSyncProcesses($oP, $bVerbose, $bDebug) // Background processes do start asap, i.e. "now" $oTask->Set('next_run_date', $oNow->format('Y-m-d H:i:s')); } - if ($bVerbose) { - $oP->p('Creating record for: '.$sTaskClass); - $oP->p('First execution planned at: '.$oTask->Get('next_run_date')); - } + $sDebugTaskClass = CronLog::GetDebugClassName($sTaskClass); + CronLog::Trace('Creating record for: '.$sDebugTaskClass); + CronLog::Trace('First execution planned at: '.$oTask->Get('next_run_date')); $oTask->DBInsert(); } else { /** @var \BackgroundTask $oTask */ @@ -430,14 +419,12 @@ function ReSyncProcesses($oP, $bVerbose, $bDebug) } } - if ($bVerbose) { - $aDisplayProcesses = []; - foreach ($aProcesses as $oExecInstance) { - $aDisplayProcesses[] = get_class($oExecInstance); - } - $sDisplayProcesses = implode(', ', $aDisplayProcesses); - $oP->p("Background processes: ".$sDisplayProcesses); + $aDisplayProcesses = []; + foreach ($aProcesses as $oExecInstance) { + $aDisplayProcesses[] = get_class($oExecInstance); } + $sDisplayProcesses = implode(', ', $aDisplayProcesses); + CronLog::Trace("Background processes: ".$sDisplayProcesses); } //////////////////////////////////////////////////////////////////////////////// @@ -445,20 +432,25 @@ function ReSyncProcesses($oP, $bVerbose, $bDebug) // Main // +CronLog::Enable(APPROOT.'/log/itop-cron.log'); set_time_limit(0); // Some background actions may really take long to finish (like backup) +$bIsModeCLI = utils::IsModeCLI(); +if ($bIsModeCLI) { + $oP = new CLIPage("iTop - cron"); + + SetupUtils::CheckPhpAndExtensionsForCli($oP, EXIT_CODE_FATAL); + utils::UseParamFile(); +} else { + $oP = new WebPage("iTop - cron"); +} + try { - $bIsModeCLI = utils::IsModeCLI(); - if ($bIsModeCLI) { - $oP = new CLIPage("iTop - cron"); + utils::UseParamFile(); - SetupUtils::CheckPhpAndExtensionsForCli($oP, EXIT_CODE_FATAL); - utils::UseParamFile(); - } else { - $oP = new WebPage("iTop - cron"); - } - - $bVerbose = utils::ReadParam('verbose', false, true /* Allow CLI */); - $bDebug = utils::ReadParam('debug', false, true /* Allow CLI */); + // Allow verbosity on output from 0 => none, 1 => debug, 2 => trace + // (writing debug messages to the cron.log file is configured with log_level_min config parameter) + $iVerbose = utils::ReadParam('verbose', 0, true /* Allow CLI */); + CronLog::SetDebug($oP, $iVerbose); if ($bIsModeCLI) { // Next steps: @@ -469,7 +461,7 @@ try { if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd)) { UserRights::Login($sAuthUser); // Login & set the user's language } else { - $oP->p("Access wrong credentials ('$sAuthUser')"); + CronLog::Error("Access wrong credentials ('$sAuthUser')"); $oP->output(); exit(EXIT_CODE_ERROR); } @@ -479,7 +471,7 @@ try { } if (!UserRights::IsAdministrator()) { - $oP->p("Access restricted to administrators"); + CronLog::Error("Access restricted to administrators"); $oP->Output(); exit(EXIT_CODE_ERROR); } @@ -491,42 +483,49 @@ try { } require_once(APPROOT.'core/mutex.class.inc.php'); - $oP->p("Starting: ".time().' ('.date('Y-m-d H:i:s').')'); } catch (Exception $e) { - $oP->p("Error: ".$e->GetMessage()); + CronLog::Error("Error: ".$e->GetMessage(), CronLog::CHANNEL_DEFAULT, ['stack' => $e->getTraceAsString()]); $oP->output(); exit(EXIT_CODE_FATAL); } try { - $oMutex = new iTopMutex('cron'); if (!MetaModel::DBHasAccess(ACCESS_ADMIN_WRITE)) { - $oP->p("A maintenance is ongoing"); + CronLog::Debug("A maintenance is ongoing"); } else { - if ($oMutex->TryLock()) { - CronExec($oP, $bVerbose, $bDebug); + // Limit the number of cron process to run in parallel + $iMaxCronProcess = max(MetaModel::GetConfig()->Get('cron.max_processes'), 1); + $bCanRun = false; + $iProcessNumber = 0; + for ($i = 0; $i < $iMaxCronProcess; $i++) { + $oMutex = new iTopMutex("cron#$i"); + if ($oMutex->TryLock()) { + $iProcessNumber = $i + 1; + $bCanRun = true; + break; + } + } + if ($bCanRun) { + CronLog::$iProcessNumber = $iProcessNumber; + $iCronRunningCount = utils::CountCronRunning(); + CronLog::Info("Starting ($iCronRunningCount total cron running)"); + CronExec($iVerbose > 0); } else { - // Exit silently - $oP->p("Already running..."); + CronLog::$iProcessNumber = 0; + CronLog::Debug("The limit of $iMaxCronProcess cron process running in parallel is already reached"); + exit(0); } } } catch (Exception $e) { - $oP->p("ERROR: '".$e->getMessage()."'"); - if ($bDebug) { - // Might contain verb parameters such a password... - $oP->p($e->getTraceAsString()); - } + CronLog::Error("ERROR: '".$e->getMessage()."'", CronLog::CHANNEL_DEFAULT, ['stack' => $e->getTraceAsString()]); } finally { try { $oMutex->Unlock(); } catch (Exception $e) { - $oP->p("ERROR: '".$e->getMessage()."'"); - if ($bDebug) { - // Might contain verb parameters such a password... - $oP->p($e->getTraceAsString()); - } + CronLog::Error("ERROR: '".$e->getMessage()."'"); } } -$oP->p("Exiting: ".time().' ('.date('Y-m-d H:i:s').')'); +$iCronRunningCount = utils::CountCronRunning(); +CronLog::Info("Exiting ($iCronRunningCount total cron running)"); $oP->Output(); diff --git a/webservices/cron_multi_launcher_linux.php b/webservices/cron_multi_launcher_linux.php new file mode 100644 index 0000000000..2b1a7e586b --- /dev/null +++ b/webservices/cron_multi_launcher_linux.php @@ -0,0 +1,85 @@ + 590, +// 'cron_sleep' => 2 +// 'cron.max_processes' => 10 + +require_once(__DIR__.'/../approot.inc.php'); + +$sConfigFile = utils::GetConfigFilePath(); +if (!file_exists($sConfigFile)) { + echo "iTop is not yet installed. Exiting...\n"; + exit(EXIT_CODE_ERROR); +} + +require_once(APPROOT.'/application/startup.inc.php'); + +function UsageAndExit() +{ + echo "USAGE:\n"; + echo "php cron_multi_launcher_linux.php --param_file=\n"; + exit(EXIT_CODE_FATAL); +} + +function ReadMandatoryParam($sParam, $sSanitizationFilter = 'parameter') +{ + $sValue = utils::ReadParam($sParam, null, true, $sSanitizationFilter); + if (is_null($sValue)) { + echo "ERROR: Missing argument '$sParam'\n"; + UsageAndExit(); + } + + return trim($sValue); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// Main +// + +try { + utils::UseParamFile(); + $sAuthUser = ReadMandatoryParam('auth_user', 'raw_data'); + $sAuthPwd = ReadMandatoryParam('auth_pwd', 'raw_data'); + if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd)) { + UserRights::Login($sAuthUser); // Login & set the user's language + } else { + echo "Access wrong credentials ('$sAuthUser')\n"; + exit(EXIT_CODE_ERROR); + } + + if (!UserRights::IsAdministrator()) { + echo "Access restricted to administrators\n"; + exit(EXIT_CODE_ERROR); + } + + if (!MetaModel::DBHasAccess(ACCESS_ADMIN_WRITE) || file_exists(MAINTENANCE_MODE_FILE) || file_exists(READONLY_MODE_FILE)) { + echo "A maintenance is ongoing\n"; + exit(EXIT_CODE_ERROR); + } + + $oConfig = MetaModel::GetConfig(); + $iMaxCronProcess = max($oConfig->Get('cron.max_processes'), 1); + $iRespawnTime = 60 / $iMaxCronProcess; + $sPhpPath = $oConfig->Get('php_path'); + for ($i = 0; $i < $iMaxCronProcess; $i++) { + $sParamsFile = utils::ReadParam('param_file', '', true, 'raw_data'); + $sCronCmd = APPROOT."webservices/cron.php --param_file=$sParamsFile"; + $sOutputFile = APPROOT.'log/itop-cron.log'; + + // Execute command, redirect stdout and stderr LINUX ONLY + exec(sprintf($sPhpPath.' %s >> %s 2>&1 &', escapeshellcmd($sCronCmd), escapeshellarg($sOutputFile))); + sleep((int)(floor($iRespawnTime))); + } + +} catch (Exception $e) { + echo "ERROR: {$e->getMessage()}\n"; +}