N°8764 - skip audit when itop reinstalled on top of a backup without env-production to read previous datamodel + phpdoc fix

setup: phpstan level 2

core/config.class.inc.php

@@ -2685,14 +2685,13 @@ class Config
 	 *
 	 * @param array $aParamValues
 	 * @param ?string $sModulesDir
-	 * @param bool $bPreserveModuleSettings
 	 *
 	 * @return void The current object is modified directly
 	 *
 	 * @throws \Exception
 	 * @throws \CoreException
 	 */
-	public function UpdateFromParams($aParamValues, $sModulesDir = null, $bPreserveModuleSettings = false)
+	public function UpdateFromParams($aParamValues, $sModulesDir = null)
 	{
 		if (isset($aParamValues['application_path'])) {
 			$this->Set('app_root_url', $aParamValues['application_path']);
@@ -2740,7 +2739,10 @@ class Config
 		} else {
 			$aSelectedModules = null;
 		}
-		$this->UpdateIncludes($sModulesDir, $aSelectedModules);
+
+		if (! is_null($sModulesDir)) {
+			$this->UpdateIncludes($sModulesDir, $aSelectedModules);
+		}
 
 		if (isset($aParamValues['source_dir'])) {
 			$this->Set('source_dir', $aParamValues['source_dir']);
@@ -2758,12 +2760,8 @@ class Config
 	 *
 	 * @throws Exception
 	 */
-	public function UpdateIncludes($sModulesDir, $aSelectedModules = null)
+	public function UpdateIncludes(string $sModulesDir, $aSelectedModules = null)
 	{
-		if ($sModulesDir === null) {
-			return;
-		}
-
 		// Initialize the arrays below with default values for the application...
 		$oEmptyConfig = new Config('dummy_file', false); // Do NOT load any config file, just set the default values
 		$aAddOns = $oEmptyConfig->GetAddOns();

datamodels/2.x/combodo-db-tools/src/Service/DBToolsUtils.php

@@ -10,6 +10,7 @@ namespace Combodo\iTop\DBTools\Service;
 use CMDBSource;
 use DBObjectSearch;
 use DBObjectSet;
+use IssueLog;
 
 class DBToolsUtils
 {

datamodels/2.x/itop-core-update/src/Service/CoreUpdater.php

@@ -92,7 +92,7 @@ final class CoreUpdater
 			$sFinalEnv = 'production';
 			$oRuntimeEnv = new RunTimeEnvironmentCoreUpdater($sFinalEnv, false);
 			$oRuntimeEnv->CheckDirectories($sFinalEnv);
-			$oRuntimeEnv->CompileFrom('production');
+			$oRuntimeEnv->CompileFrom($sFinalEnv);
 
 			$oRuntimeEnv->Rollback();
 
@@ -155,21 +155,13 @@ final class CoreUpdater
 				APPROOT.'extensions',
 			];
 			$aAvailableModules = $oRuntimeEnv->AnalyzeInstallation($oConfig, $aDirsToScanForModules);
-			$aSelectedModules = [];
-			foreach ($aAvailableModules as $sModuleId => $aModule) {
-				if (($sModuleId == ROOT_MODULE) || ($sModuleId == DATAMODEL_MODULE)) {
-					continue;
-				} else {
-					$aSelectedModules[] = $sModuleId;
-				}
-			}
-			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'BeforeDatabaseCreation');
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'BeforeDatabaseCreation');
 			$oRuntimeEnv->CreateDatabaseStructure($oConfig, 'upgrade');
-			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseCreation');
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseCreation');
 			$oRuntimeEnv->UpdatePredefinedObjects();
-			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseSetup');
-			$oRuntimeEnv->LoadData($aAvailableModules, $aSelectedModules, false /* no sample data*/);
-			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDataLoad');
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseSetup');
+			$oRuntimeEnv->LoadData($aAvailableModules, false /* no sample data*/);
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDataLoad');
 			$sDataModelVersion = $oRuntimeEnv->GetCurrentDataModelVersion();
 			$oExtensionsMap = new iTopExtensionsMap();
 			// Default choices = as before
@@ -187,7 +179,7 @@ final class CoreUpdater
 			$oRuntimeEnv->RecordInstallation(
 				$oConfig,
 				$sDataModelVersion,
-				$aSelectedModules,
+				array_keys($aAvailableModules),
 				$aSelectedExtensionCodes,
 				'Done by the iTop Core Updater'
 			);

datamodels/2.x/itop-hub-connector/ajax.php

@@ -24,129 +24,13 @@
  * @license http://opensource.org/licenses/AGPL-3.0
  */
 
-use Combodo\iTop\Application\WebPage\JsonPage;
+use Combodo\iTop\HubConnector\Controller\HubController;
 
 require_once(APPROOT.'application/utils.inc.php');
 require_once(APPROOT.'core/log.class.inc.php');
 IssueLog::Enable(APPROOT.'log/error.log');
 
-require_once(APPROOT.'setup/runtimeenv.class.inc.php');
-require_once(APPROOT.'setup/backup.class.inc.php');
-require_once(APPROOT.'core/mutex.class.inc.php');
-require_once(APPROOT.'core/dict.class.inc.php');
-require_once(APPROOT.'setup/xmldataloader.class.inc.php');
-require_once(__DIR__.'/hubruntimeenvironment.class.inc.php');
-
-/**
- * Overload of DBBackup to handle logging
- */
-class DBBackupWithErrorReporting extends DBBackup
-{
-	protected $aInfos = [];
-
-	protected $aErrors = [];
-
-	protected function LogInfo($sMsg)
-	{
-		$aInfos[] = $sMsg;
-	}
-
-	protected function LogError($sMsg)
-	{
-		IssueLog::Error($sMsg);
-		$aErrors[] = $sMsg;
-	}
-
-	public function GetInfos()
-	{
-		return $this->aInfos;
-	}
-
-	public function GetErrors()
-	{
-		return $this->aErrors;
-	}
-}
-
-/**
- *
- * @param string $sTargetFile
- * @throws Exception
- * @return DBBackupWithErrorReporting
- */
-function DoBackup($sTargetFile)
-{
-	// Make sure the target directory exists
-	$sBackupDir = dirname($sTargetFile);
-	SetupUtils::builddir($sBackupDir);
-
-	$oBackup = new DBBackupWithErrorReporting();
-	$oBackup->SetMySQLBinDir(MetaModel::GetConfig()->GetModuleSetting('itop-backup', 'mysql_bindir', ''));
-	$sSourceConfigFile = APPCONF.utils::GetCurrentEnvironment().'/'.ITOP_CONFIG_FILE;
-
-	$oMutex = new iTopMutex('backup.'.utils::GetCurrentEnvironment());
-	$oMutex->Lock();
-	try {
-		$oBackup->CreateCompressedBackup($sTargetFile, $sSourceConfigFile);
-	} catch (Exception $e) {
-		$oMutex->Unlock();
-		throw $e;
-	}
-	$oMutex->Unlock();
-	return $oBackup;
-}
-
-/**
- * Outputs the status of the current ajax execution (as a JSON structure)
- *
- * @param string $sMessage
- * @param bool $bSuccess
- * @param number $iErrorCode
- * @param array $aMoreFields
- *        	Extra fields to pass to the caller, if needed
- */
-function ReportStatus($sMessage, $bSuccess, $iErrorCode = 0, $aMoreFields = [])
-{
-	// Do not use AjaxPage during setup phases, because it uses InterfaceDiscovery in Twig compilation
-	$oPage = new JsonPage();
-	$aResult = [
-		'code' => $iErrorCode,
-		'message' => $sMessage,
-		'fields' => $aMoreFields,
-	];
-	$oPage->SetData($aResult);
-	$oPage->SetOutputDataOnly(true);
-	$oPage->output();
-}
-
-/**
- * Helper to output the status of a successful execution
- *
- * @param string $sMessage
- * @param array $aMoreFields
- *        	Extra fields to pass to the caller, if needed
- */
-function ReportSuccess($sMessage, $aMoreFields = [])
-{
-	ReportStatus($sMessage, true, 0, $aMoreFields);
-}
-
-/**
- * Helper to output the status of a failed execution
- *
- * @param string $sMessage
- * @param number $iErrorCode
- * @param array $aMoreFields
- *        	Extra fields to pass to the caller, if needed
- */
-function ReportError($sMessage, $iErrorCode, $aMoreFields = [])
-{
-	if ($iErrorCode == 0) {
-		// 0 means no error, so change it if no meaningful error code is supplied
-		$iErrorCode = -1;
-	}
-	ReportStatus($sMessage, false, $iErrorCode, $aMoreFields);
-}
+require_once(__DIR__.'/src/Controller/HubController.php');
 
 try {
 	SetupUtils::ExitMaintenanceMode(false); // Reset maintenance mode in case of problem
@@ -183,7 +67,7 @@ try {
 			foreach ($aChecks as $oCheckResult) {
 				if ($oCheckResult->iSeverity == CheckResult::ERROR) {
 					$bFailed = true;
-					ReportError($oCheckResult->sLabel, -2);
+					HubController::GetInstance()->ReportError($oCheckResult->sLabel, -2);
 				}
 			}
 			if (!$bFailed) {
@@ -191,169 +75,27 @@ try {
 				$fFreeSpace = SetupUtils::CheckDiskSpace($sDBBackupPath);
 				if ($fFreeSpace !== false) {
 					$sMessage = Dict::Format('iTopHub:BackupFreeDiskSpaceIn', SetupUtils::HumanReadableSize($fFreeSpace), dirname($sDBBackupPath));
-					ReportSuccess($sMessage);
+					HubController::GetInstance()->ReportSuccess($sMessage);
 				} else {
-					ReportError(Dict::S('iTopHub:FailedToCheckFreeDiskSpace'), -1);
+					HubController::GetInstance()->ReportError(Dict::S('iTopHub:FailedToCheckFreeDiskSpace'), -1);
 				}
 			}
 			break;
 
 		case 'do_backup':
-			require_once(APPROOT.'/application/startup.inc.php');
-			require_once(APPROOT.'/application/loginwebpage.class.inc.php');
-			LoginWebPage::DoLogin(true); // Check user rights and prompt if needed (must be admin)
-
-			try {
-				if (MetaModel::GetConfig()->Get('demo_mode')) {
-					throw new Exception('Sorry the installation of extensions is not allowed in demo mode');
-				}
-				SetupLog::Info('Backup starts...');
-				set_time_limit(0);
-				$sBackupPath = APPROOT.'/data/backups/manual/backup-';
-				$iSuffix = 1;
-				$sSuffix = '';
-				// Generate a unique name...
-				do {
-					$sBackupFile = $sBackupPath.date('Y-m-d-His').$sSuffix;
-					$sSuffix = '-'.$iSuffix;
-					$iSuffix++ ;
-				} while (file_exists($sBackupFile));
-
-				$oBackup = DoBackup($sBackupFile);
-				$aErrors = $oBackup->GetErrors();
-				if (count($aErrors) > 0) {
-					SetupLog::Error('Backup failed.');
-					SetupLog::Error(implode("\n", $aErrors));
-					ReportError(Dict::S('iTopHub:BackupFailed'), -1, $aErrors);
-				} else {
-					SetupLog::Info('Backup successfully completed.');
-					ReportSuccess(Dict::S('iTopHub:BackupOk'));
-				}
-			} catch (Exception $e) {
-				SetupLog::Error($e->getMessage());
-				ReportError($e->getMessage(), $e->getCode());
-			}
+			HubController::GetInstance()->LaunchBackup();
 			break;
 
 		case 'compile':
-			SetupLog::Info('Deployment starts...');
-			$sAuthent = utils::ReadParam('authent', '', false, 'raw_data');
-			if (!file_exists(utils::GetDataPath().'hub/compile_authent') || $sAuthent !== file_get_contents(utils::GetDataPath().'hub/compile_authent')) {
-				throw new SecurityException(Dict::S('iTopHub:FailAuthent'));
-			}
-			// First step: prepare the datamodel, if it fails, roll-back
-			$aSelectedExtensionCodes = utils::ReadParam('extension_codes', []);
-			$aSelectedExtensionDirs = utils::ReadParam('extension_dirs', []);
-
-			$oRuntimeEnv = new HubRunTimeEnvironment('production', false); // use a temp environment: production-build
-			$oRuntimeEnv->MoveSelectedExtensions(APPROOT.'/data/downloaded-extensions/', $aSelectedExtensionDirs);
-
-			$oConfig = new Config(APPCONF.'production/'.ITOP_CONFIG_FILE);
-			if ($oConfig->Get('demo_mode')) {
-				throw new Exception('Sorry the installation of extensions is not allowed in demo mode');
-			}
-
-			$aSelectModules = $oRuntimeEnv->CompileFrom('production', false); // WARNING symlinks does not seem to be compatible with manual Commit
-
-			$oRuntimeEnv->UpdateIncludes($oConfig);
-
-			$oRuntimeEnv->InitDataModel($oConfig, true /* model only */);
-
-			// Safety check: check the inter dependencies, will throw an exception in case of inconsistency
-			$oRuntimeEnv->AnalyzeInstallation($oConfig, $oRuntimeEnv->GetBuildDir(), true);
-
-			$oRuntimeEnv->CheckMetaModel(); // Will throw an exception if a problem is detected
-
-			// Everything seems Ok so far, commit in env-production!
-			$oRuntimeEnv->WriteConfigFileSafe($oConfig);
-			$oRuntimeEnv->Commit();
-
-			// Report the success in a way that will be detected by the ajax caller
-			SetupLog::Info('Compilation completed...');
-			ReportSuccess('Ok'); // No access to Dict::S here
+			HubController::GetInstance()->LaunchCompile();
 			break;
 
 		case 'move_to_production':
-			// Second step: update the schema and the data
-			// Everything happening below is based on env-production
-			$oRuntimeEnv = new RunTimeEnvironment('production', true);
-
-			try {
-				SetupLog::Info('Move to production starts...');
-				$sAuthent = utils::ReadParam('authent', '', false, 'raw_data');
-				if (!file_exists(utils::GetDataPath().'hub/compile_authent') || $sAuthent !== file_get_contents(utils::GetDataPath().'hub/compile_authent')) {
-					throw new SecurityException(Dict::S('iTopHub:FailAuthent'));
-				}
-				unlink(utils::GetDataPath().'hub/compile_authent');
-				// Load the "production" config file to clone & update it
-				$oConfig = new Config(APPCONF.'production/'.ITOP_CONFIG_FILE);
-				SetupUtils::EnterReadOnlyMode($oConfig);
-
-				$oRuntimeEnv->InitDataModel($oConfig, true /* model only */);
-
-				$aAvailableModules = $oRuntimeEnv->AnalyzeInstallation($oConfig, $oRuntimeEnv->GetBuildDir(), true);
-
-				$aSelectedModules = [];
-				foreach ($aAvailableModules as $sModuleId => $aModule) {
-					if (($sModuleId == ROOT_MODULE) || ($sModuleId == DATAMODEL_MODULE)) {
-						continue;
-					} else {
-						$aSelectedModules[] = $sModuleId;
-					}
-				}
-
-				$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'BeforeDatabaseCreation');
-
-				$oRuntimeEnv->CreateDatabaseStructure($oConfig, 'upgrade');
-
-				$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseCreation');
-
-				$oRuntimeEnv->UpdatePredefinedObjects();
-
-				$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseSetup');
-
-				$oRuntimeEnv->LoadData($aAvailableModules, $aSelectedModules, false /* no sample data*/);
-
-				$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDataLoad');
-
-				// Record the installation so that the "about box" knows about the installed modules
-				$sDataModelVersion = $oRuntimeEnv->GetCurrentDataModelVersion();
-
-				$oExtensionsMap = new iTopExtensionsMap();
-
-				// Default choices = as before
-				$oExtensionsMap->LoadChoicesFromDatabase($oConfig);
-				foreach ($oExtensionsMap->GetAllExtensions() as $oExtension) {
-					// Plus all "remote" extensions
-					if ($oExtension->sSource == iTopExtension::SOURCE_REMOTE) {
-						$oExtensionsMap->MarkAsChosen($oExtension->sCode);
-					}
-				}
-				$aSelectedExtensionCodes = [];
-				foreach ($oExtensionsMap->GetChoices() as $oExtension) {
-					$aSelectedExtensionCodes[] = $oExtension->sCode;
-				}
-				$aSelectedExtensions = $oExtensionsMap->GetChoices();
-				$oRuntimeEnv->RecordInstallation($oConfig, $sDataModelVersion, $aSelectedModules, $aSelectedExtensionCodes, 'Done by the iTop Hub Connector');
-
-				// Report the success in a way that will be detected by the ajax caller
-				SetupLog::Info('Deployment successfully completed.');
-				ReportSuccess(Dict::S('iTopHub:CompiledOK'));
-			} catch (Exception $e) {
-				if (file_exists(utils::GetDataPath().'hub/compile_authent')) {
-					unlink(utils::GetDataPath().'hub/compile_authent');
-				}
-				// Note: at this point, the dictionnary is not necessarily loaded
-				SetupLog::Error(get_class($e).': '.Dict::S('iTopHub:ConfigurationSafelyReverted')."\n".$e->getMessage());
-				SetupLog::Error('Debug trace: '.$e->getTraceAsString());
-				ReportError($e->getMessage(), $e->getCode());
-			} finally {
-				SetupUtils::ExitReadOnlyMode();
-			}
+			HubController::GetInstance()->LaunchDeploy();
 			break;
 
 		default:
-			ReportError("Invalid operation: '$sOperation'", -1);
+			HubController::GetInstance()->ReportError("Invalid operation: '$sOperation'", -1);
 	}
 } catch (Exception $e) {
 	SetupLog::Error(get_class($e).': '.Dict::S('iTopHub:ConfigurationSafelyReverted')."\n".$e->getMessage());
@@ -361,5 +103,5 @@ try {
 
 	utils::PopArchiveMode();
 
-	ReportError($e->getMessage(), $e->getCode());
+	HubController::GetInstance()->ReportError($e->getMessage(), $e->getCode());
 }

datamodels/2.x/itop-hub-connector/launch.php

@@ -186,7 +186,7 @@ function collect_configuration()
 
 	// iTop modules
 	$oConfig = MetaModel::GetConfig();
-	$aInstalledModules = ModuleInstallationService::GetInstance()->ReadFromDB($oConfig);
+	$aInstalledModules = ModuleInstallationRepository::GetInstance()->ReadFromDB($oConfig);
 
 	foreach ($aInstalledModules as $aDBInfo) {
 		$aConfiguration['itop_modules'][$aDBInfo['name']] = $aDBInfo['version'];

datamodels/2.x/itop-hub-connector/src/Controller/HubController.php

@@ -0,0 +1,300 @@
+<?php
+
+namespace Combodo\iTop\HubConnector\Controller;
+
+use Combodo\iTop\Application\WebPage\JsonPage;
+use Combodo\iTop\HubConnector\Model\DBBackupWithErrorReporting;
+use Combodo\iTop\HubConnector\setup\HubRunTimeEnvironment;
+use Config;
+use Dict;
+use Exception;
+use iTopExtension;
+use iTopExtensionsMap;
+use iTopMutex;
+use LoginWebPage;
+use MetaModel;
+use MFCompiler;
+use RunTimeEnvironment;
+use SecurityException;
+use SetupLog;
+use SetupUtils;
+use utils;
+
+require_once(APPROOT.'setup/runtimeenv.class.inc.php');
+require_once(APPROOT.'setup/backup.class.inc.php');
+require_once(APPROOT.'core/mutex.class.inc.php');
+require_once(APPROOT.'core/dict.class.inc.php');
+require_once(APPROOT.'setup/xmldataloader.class.inc.php');
+require_once(__DIR__.'/../setup/hubruntimeenvironment.class.inc.php');
+
+class HubController
+{
+	private static HubController $oInstance;
+	protected $bOutputHeaders = false;
+
+	protected function __construct()
+	{
+	}
+
+	final public static function GetInstance(): HubController
+	{
+		if (!isset(self::$oInstance)) {
+			self::$oInstance = new HubController();
+		}
+
+		return self::$oInstance;
+	}
+
+	final public static function SetInstance(?HubController $oInstance): void
+	{
+		self::$oInstance = $oInstance;
+	}
+
+	public function LaunchBackup()
+	{
+		require_once(APPROOT.'/application/startup.inc.php');
+		require_once(APPROOT.'/application/loginwebpage.class.inc.php');
+		LoginWebPage::DoLogin(true); // Check user rights and prompt if needed (must be admin)
+
+		try {
+			if (MetaModel::GetConfig()->Get('demo_mode')) {
+				throw new Exception('Sorry the installation of extensions is not allowed in demo mode');
+			}
+			SetupLog::Info('Backup starts...');
+			set_time_limit(0);
+			$sBackupPath = APPROOT.'/data/backups/manual/backup-';
+			$iSuffix = 1;
+			$sSuffix = '';
+			// Generate a unique name...
+			do {
+				$sBackupFile = $sBackupPath.date('Y-m-d-His').$sSuffix;
+				$sSuffix = '-'.$iSuffix;
+				$iSuffix++ ;
+			} while (file_exists($sBackupFile));
+
+			$oBackup = $this->DoBackup($sBackupFile);
+			$aErrors = $oBackup->GetErrors();
+			if (count($aErrors) > 0) {
+				SetupLog::Error('Backup failed.');
+				SetupLog::Error(implode("\n", $aErrors));
+				$this->ReportError(Dict::S('iTopHub:BackupFailed'), -1, $aErrors);
+			} else {
+				SetupLog::Info('Backup successfully completed.');
+				$this->ReportSuccess(Dict::S('iTopHub:BackupOk'));
+			}
+		} catch (Exception $e) {
+			SetupLog::Error($e->getMessage());
+			$this->ReportError($e->getMessage(), $e->getCode());
+		}
+	}
+
+	/**
+	 *
+	 * @param string $sTargetFile
+	 * @throws Exception
+	 * @return DBBackupWithErrorReporting
+	 */
+	public function DoBackup($sTargetFile): DBBackupWithErrorReporting
+	{
+		// Make sure the target directory exists
+		$sBackupDir = dirname($sTargetFile);
+		SetupUtils::builddir($sBackupDir);
+
+		$oBackup = new DBBackupWithErrorReporting();
+		$oBackup->SetMySQLBinDir(MetaModel::GetConfig()->GetModuleSetting('itop-backup', 'mysql_bindir', ''));
+		$sSourceConfigFile = APPCONF.utils::GetCurrentEnvironment().'/'.ITOP_CONFIG_FILE;
+
+		$oMutex = new iTopMutex('backup.'.utils::GetCurrentEnvironment());
+		$oMutex->Lock();
+		try {
+			$oBackup->CreateCompressedBackup($sTargetFile, $sSourceConfigFile);
+		} catch (Exception $e) {
+			$oMutex->Unlock();
+			throw $e;
+		}
+		$oMutex->Unlock();
+		return $oBackup;
+	}
+
+	public function LaunchCompile()
+	{
+		SetupLog::Info('Deployment starts...');
+		$sAuthent = utils::ReadParam('authent', '', false, 'raw_data');
+		if (!file_exists(utils::GetDataPath().'hub/compile_authent') || $sAuthent !== file_get_contents(utils::GetDataPath().'hub/compile_authent')) {
+			throw new SecurityException(Dict::S('iTopHub:FailAuthent'));
+		}
+		// First step: prepare the datamodel, if it fails, roll-back
+		$aSelectedExtensionCodes = utils::ReadParam('extension_codes', []);
+		$aSelectedExtensionDirs = utils::ReadParam('extension_dirs', []);
+
+		$oRuntimeEnv = new HubRunTimeEnvironment('production', false); // use a temp environment: production-build
+		$oRuntimeEnv->MoveSelectedExtensions(APPROOT.'/data/downloaded-extensions/', $aSelectedExtensionDirs);
+
+		$oConfig = new Config(APPCONF.'production/'.ITOP_CONFIG_FILE);
+		if ($oConfig->Get('demo_mode')) {
+			throw new Exception('Sorry the installation of extensions is not allowed in demo mode');
+		}
+
+		$aSelectModules = $oRuntimeEnv->CompileFrom('production'); // WARNING symlinks does not seem to be compatible with manual Commit
+
+		$oRuntimeEnv->UpdateIncludes($oConfig);
+
+		$oRuntimeEnv->InitDataModel($oConfig, true /* model only */);
+
+		// Safety check: check the inter dependencies, will throw an exception in case of inconsistency
+		$oRuntimeEnv->AnalyzeInstallation($oConfig, $oRuntimeEnv->GetBuildDir(), true);
+
+		$oRuntimeEnv->CheckMetaModel(); // Will throw an exception if a problem is detected
+
+		// Everything seems Ok so far, commit in env-production!
+		$oRuntimeEnv->WriteConfigFileSafe($oConfig);
+		$oRuntimeEnv->Commit();
+
+		// Report the success in a way that will be detected by the ajax caller
+		SetupLog::Info('Compilation completed...');
+
+		$this->ReportSuccess('Ok'); // No access to Dict::S here
+	}
+
+	public function LaunchDeploy()
+	{
+		// Second step: update the schema and the data
+		// Everything happening below is based on env-production
+		$oRuntimeEnv = new RunTimeEnvironment('production', true);
+
+		try {
+			SetupLog::Info('Move to production starts...');
+			$sAuthent = utils::ReadParam('authent', '', false, 'raw_data');
+			if (!file_exists(utils::GetDataPath().'hub/compile_authent') || $sAuthent !== file_get_contents(utils::GetDataPath().'hub/compile_authent')) {
+				throw new SecurityException(Dict::S('iTopHub:FailAuthent'));
+			}
+			unlink(utils::GetDataPath().'hub/compile_authent');
+			// Load the "production" config file to clone & update it
+			$oConfig = new Config(APPCONF.'production/'.ITOP_CONFIG_FILE);
+			SetupUtils::EnterReadOnlyMode($oConfig);
+
+			$oRuntimeEnv->InitDataModel($oConfig, true /* model only */);
+
+			$aAvailableModules = $oRuntimeEnv->AnalyzeInstallation($oConfig, $oRuntimeEnv->GetBuildDir(), true);
+
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'BeforeDatabaseCreation');
+
+			$oRuntimeEnv->CreateDatabaseStructure($oConfig, 'upgrade');
+
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseCreation');
+
+			$oRuntimeEnv->UpdatePredefinedObjects();
+
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseSetup');
+
+			$oRuntimeEnv->LoadData($aAvailableModules, false /* no sample data*/);
+
+			$oRuntimeEnv->CallInstallerHandlers($aAvailableModules, 'AfterDataLoad');
+
+			// Record the installation so that the "about box" knows about the installed modules
+			$sDataModelVersion = $oRuntimeEnv->GetCurrentDataModelVersion();
+
+			$oExtensionsMap = new iTopExtensionsMap();
+
+			// Default choices = as before
+			$oExtensionsMap->LoadChoicesFromDatabase($oConfig);
+			foreach ($oExtensionsMap->GetAllExtensions() as $oExtension) {
+				// Plus all "remote" extensions
+				if ($oExtension->sSource == iTopExtension::SOURCE_REMOTE) {
+					$oExtensionsMap->MarkAsChosen($oExtension->sCode);
+				}
+			}
+			$aSelectedExtensionCodes = [];
+			foreach ($oExtensionsMap->GetChoices() as $oExtension) {
+				$aSelectedExtensionCodes[] = $oExtension->sCode;
+			}
+			$aSelectedExtensions = $oExtensionsMap->GetChoices();
+			$oRuntimeEnv->RecordInstallation($oConfig, $sDataModelVersion, array_keys($aAvailableModules), $aSelectedExtensionCodes, 'Done by the iTop Hub Connector');
+
+			// Report the success in a way that will be detected by the ajax caller
+			SetupLog::Info('Deployment successfully completed.');
+			$this->ReportSuccess(Dict::S('iTopHub:CompiledOK'));
+		} catch (Exception $e) {
+			if (file_exists(utils::GetDataPath().'hub/compile_authent')) {
+				unlink(utils::GetDataPath().'hub/compile_authent');
+			}
+			// Note: at this point, the dictionnary is not necessarily loaded
+			SetupLog::Error(get_class($e).': '.Dict::S('iTopHub:ConfigurationSafelyReverted')."\n".$e->getMessage());
+			SetupLog::Error('Debug trace: '.$e->getTraceAsString());
+			$this->ReportError($e->getMessage(), $e->getCode());
+		} finally {
+			SetupUtils::ExitReadOnlyMode();
+		}
+	}
+
+	/**
+	 * Outputs the status of the current ajax execution (as a JSON structure)
+	 *
+	 * @param string $sMessage
+	 * @param bool $bSuccess
+	 * @param number $iErrorCode
+	 * @param array $aMoreFields
+	 *        	Extra fields to pass to the caller, if needed
+	 */
+	public function ReportStatus($sMessage, $bSuccess, $iErrorCode = 0, $aMoreFields = [])
+	{
+		// Do not use AjaxPage during setup phases, because it uses InterfaceDiscovery in Twig compilation
+		$this->oLastJsonPage = new JsonPage();
+		$this->oLastJsonPage->SetOutputHeaders($this->bOutputHeaders);
+		$aResult = [
+			'code' => $iErrorCode,
+			'message' => $sMessage,
+			'fields' => $aMoreFields,
+		];
+		$this->oLastJsonPage->SetData($aResult);
+		$this->oLastJsonPage->SetOutputDataOnly(true);
+		$this->oLastJsonPage->output();
+	}
+
+	private ?JsonPage $oLastJsonPage = null;
+
+	public function GetLastJsonPage(): ?JsonPage
+	{
+		return $this->oLastJsonPage;
+	}
+
+	/**
+	 * Helper to output the status of a successful execution
+	 *
+	 * @param string $sMessage
+	 * @param array $aMoreFields
+	 *        	Extra fields to pass to the caller, if needed
+	 */
+	public function ReportSuccess($sMessage, $aMoreFields = [])
+	{
+		$this->ReportStatus($sMessage, true, 0, $aMoreFields);
+	}
+
+	/**
+	 * Helper to output the status of a failed execution
+	 *
+	 * @param string $sMessage
+	 * @param number $iErrorCode
+	 * @param array $aMoreFields
+	 *        	Extra fields to pass to the caller, if needed
+	 */
+	public function ReportError($sMessage, $iErrorCode, $aMoreFields = [])
+	{
+		if ($iErrorCode == 0) {
+			// 0 means no error, so change it if no meaningful error code is supplied
+			$iErrorCode = -1;
+		}
+		$this->ReportStatus($sMessage, false, $iErrorCode, $aMoreFields);
+	}
+
+	/**
+	 * Dont print headers for testing purpose mainly
+	 * @param bool bOutputHeaders
+	 *
+	 * @return void
+	 */
+	public function SetOutputHeaders(bool $bOutputHeaders): void
+	{
+		$this->bOutputHeaders = $bOutputHeaders;
+	}
+}

datamodels/2.x/itop-hub-connector/src/Model/DBBackupWithErrorReporting.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace Combodo\iTop\HubConnector\Model;
+
+use DBBackup;
+use IssueLog;
+
+/**
+ * Overload of DBBackup to handle logging
+ */
+class DBBackupWithErrorReporting extends DBBackup
+{
+	protected $aInfos = [];
+
+	protected $aErrors = [];
+
+	protected function LogInfo($sMsg)
+	{
+		$this->aInfos[] = $sMsg;
+	}
+
+	protected function LogError($sMsg)
+	{
+		IssueLog::Error($sMsg);
+		$this->aErrors[] = $sMsg;
+	}
+
+	public function GetInfos(): array
+	{
+		return $this->aInfos;
+	}
+
+	public function GetErrors(): array
+	{
+		return $this->aErrors;
+	}
+}

datamodels/2.x/itop-hub-connector/src/setup/hubruntimeenvironment.class.inc.php

@@ -1,9 +1,17 @@
 <?php
 
+namespace Combodo\iTop\HubConnector\setup;
+
+use Config;
+use Exception;
+use RunTimeEnvironment;
+use SetupUtils;
+
 class HubRunTimeEnvironment extends RunTimeEnvironment
 {
 	/**
 	 * Constructor
+	 *
 	 * @param string $sEnvironment
 	 * @param string $bAutoCommit
 	 */
@@ -24,6 +32,7 @@ class HubRunTimeEnvironment extends RunTimeEnvironment
 
 	/**
 	 * Update the includes for the target environment
+	 *
 	 * @param Config $oConfig
 	 */
 	public function UpdateIncludes(Config $oConfig)
@@ -33,7 +42,9 @@ class HubRunTimeEnvironment extends RunTimeEnvironment
 
 	/**
 	 * Move an extension (path to folder of this extension) to the target environment
+	 *
 	 * @param string $sExtensionDirectory The folder of the extension
+	 *
 	 * @throws Exception
 	 */
 	public function MoveExtension($sExtensionDirectory)
@@ -57,8 +68,10 @@ class HubRunTimeEnvironment extends RunTimeEnvironment
 
 	/**
 	 * Move the selected extensions located in the given directory in data/<target-env>-modules
+	 *
 	 * @param string $sDownloadedExtensionsDir The directory to scan
 	 * @param string[] $aSelectedExtensionDirs The list of folders to move
+	 *
 	 * @throws Exception
 	 */
 	public function MoveSelectedExtensions($sDownloadedExtensionsDir, $aSelectedExtensionDirs)

datamodels/2.x/itop-tickets/module.itop-tickets.php

@@ -12,8 +12,7 @@ SetupWebPage::AddModule(
 		// Setup
 		//
 		'dependencies' => [
-			'itop-structure/2.7.1',
-			'itop-portal/3.0.0', // module_design_itop_design->module_designs->itop-portal
+			'itop-structure/2.7.1 || itop-portal/3.0.0', // itop-portal : module_design_itop_design->module_designs->itop-portal
 		],
 		'mandatory' => false,
 		'visible' => true,

setup/AnalyzeInstallation.php

@@ -1,12 +1,11 @@
 <?php
 
-require_once __DIR__.'/ModuleInstallationService.php';
+require_once __DIR__.'/ModuleInstallationRepository.php';
 
 class AnalyzeInstallation
 {
 	private static AnalyzeInstallation $oInstance;
 	private ?array $aAvailableModules = null;
-	private ?array $aSelectInstall = null;
 
 	protected function __construct()
 	{
@@ -23,7 +22,7 @@ class AnalyzeInstallation
 
 	final public static function SetInstance(?AnalyzeInstallation $oInstance): void
 	{
-		static::$oInstance = $oInstance;
+		self::$oInstance = $oInstance;
 	}
 
 	/**
@@ -58,6 +57,7 @@ class AnalyzeInstallation
 	 * )
 	 * @throws \Exception
 	 */
+
 	public function AnalyzeInstallation(?Config $oConfig, mixed $modulesPath, bool $bAbortOnMissingDependency = false, ?array $aModulesToLoad = null)
 	{
 		$aRes = [
@@ -96,7 +96,7 @@ class AnalyzeInstallation
 			$aRes[$sModuleName] = $aModuleInfo;
 		}
 
-		$aCurrentlyInstalledModules = ModuleInstallationService::GetInstance()->ReadComputeInstalledModules($oConfig);
+		$aCurrentlyInstalledModules = ModuleInstallationRepository::GetInstance()->ReadComputeInstalledModules($oConfig);
 
 		// Adjust the list of proposed modules
 		foreach ($aCurrentlyInstalledModules as $sModuleName => $aModuleDB) {

setup/ModuleInstallationRepository.php

@@ -1,25 +1,25 @@
 <?php
 
-class ModuleInstallationService
+class ModuleInstallationRepository
 {
-	private static ModuleInstallationService $oInstance;
+	private static ModuleInstallationRepository $oInstance;
 
 	protected function __construct()
 	{
 	}
 
-	final public static function GetInstance(): ModuleInstallationService
+	final public static function GetInstance(): ModuleInstallationRepository
 	{
 		if (!isset(self::$oInstance)) {
-			self::$oInstance = new ModuleInstallationService();
+			self::$oInstance = new ModuleInstallationRepository();
 		}
 
 		return self::$oInstance;
 	}
 
-	final public static function SetInstance(?ModuleInstallationService $oInstance): void
+	final public static function SetInstance(?ModuleInstallationRepository $oInstance): void
 	{
-		static::$oInstance = $oInstance;
+		self::$oInstance = $oInstance;
 	}
 
 	private ?array $aSelectInstall = null;
@@ -95,8 +95,17 @@ SQL;
 			$aSelectInstall = CMDBSource::QueryToArray($sSQLQuery);
 		} catch (MySQLException $e) {
 			// No database or erroneous information
-			$this->log_error('Can not connect to the database: host: '.$oConfig->Get('db_host').', user:'.$oConfig->Get('db_user').', pwd:'.$oConfig->Get('db_pwd').', db name:'.$oConfig->Get('db_name'));
-			$this->log_error('Exception '.$e->getMessage());
+			SetupLog::Error(
+				'Can not connect to the database',
+				null,
+				[
+					'host'    => $oConfig->Get('db_host'),
+					'user'    => $oConfig->Get('db_user'),
+					'pwd:'    => $oConfig->Get('db_pwd'),
+					'db name' => $oConfig->Get('db_name'),
+					'msg'     => $e->getMessage(),
+				]
+			);
 			return false;
 		}
 
@@ -129,8 +138,10 @@ SQL;
 			// so assume that the datamodel version is equal to the application version
 			$aResult['datamodel_version'] = $aResult['product_version'];
 		}
-		$this->log_info("GetApplicationVersion returns: product_name: ".$aResult['product_name'].', product_version: '.$aResult['product_version']);
-		return empty($aResult) ? false : $aResult;
+
+		SetupLog::Info(__METHOD__, null, ["product_name" => $aResult['product_name'], "product_version" => $aResult['product_version']]);
+
+		return count($aResult) == 0 ? false : $aResult;
 	}
 
 	private function ComputeInstalledModules(array $aSelectInstall): array
@@ -181,4 +192,47 @@ SQL;
 
 		return $aInstallByModule;
 	}
+
+	/**
+	 * Return previous module installation. offset is applied on parent_id.
+	 * @param $iOffset: by default (offset=0) returns current installation
+	 * @return array
+	 */
+	public static function GetPreviousModuleInstallationsByOffset(int $iOffset = 0): array
+	{
+		$oFilter = DBObjectSearch::FromOQL('SELECT ModuleInstallation AS mi WHERE mi.parent_id=0 AND mi.name!="datamodel"');
+		$oSet = new DBObjectSet($oFilter, ['installed' => false]); // Most recent first
+		$oSet->SetLimit($iOffset + 1);
+
+		$iParentId = 0;
+		while (!is_null($oModuleInstallation = $oSet->Fetch())) {
+			/** @var \DBObject $oModuleInstallation */
+			if ($iOffset == 0) {
+				$iParentId = $oModuleInstallation->Get('id');
+				break;
+			}
+			$iOffset--;
+		}
+
+		if ($iParentId === 0) {
+			IssueLog::Error("no ITOP_APPLICATION ModuleInstallation found", null, ['offset' => $iOffset]);
+			throw new \Exception("no ITOP_APPLICATION ModuleInstallation found");
+		}
+
+		$oFilter = DBObjectSearch::FromOQL("SELECT ModuleInstallation AS mi WHERE mi.id=$iParentId OR mi.parent_id=$iParentId");
+		$oSet = new DBObjectSet($oFilter); // Most recent first
+		$aRawValues = $oSet->ToArrayOfValues();
+		$aValues = [];
+		foreach ($aRawValues as $aRawValue) {
+			$aValue = [];
+			foreach ($aRawValue as $sAliasAttCode => $sValue) {
+				// remove 'mi.' from AttCode
+				$sAttCode = substr($sAliasAttCode, 3);
+				$aValue[$sAttCode] = $sValue;
+			}
+
+			$aValues[] = $aValue;
+		}
+		return $aValues;
+	}
 }

setup/ajax.dataloader.php

@@ -127,7 +127,7 @@ header("Expires: Fri, 17 Jul 1970 05:00:00 GMT");    // Date in the past
 /**
  * Main program
  */
-$sOperation = Utils::ReadParam('operation', '');
+$sOperation = utils::ReadParam('operation', '');
 try {
 	SetupUtils::CheckSetupToken();
 
@@ -164,7 +164,7 @@ try {
 			break;
 
 		case 'toggle_use_symbolic_links':
-			$sUseSymbolicLinks = Utils::ReadParam('bUseSymbolicLinks', false);
+			$sUseSymbolicLinks = utils::ReadParam('bUseSymbolicLinks', false);
 			$bUseSymbolicLinks = ($sUseSymbolicLinks === 'true');
 			MFCompiler::SetUseSymbolicLinksFlag($bUseSymbolicLinks);
 			echo "toggle useSymbolicLinks flag : $bUseSymbolicLinks";

setup/applicationinstaller.class.inc.php

@@ -17,9 +17,13 @@
 //   You should have received a copy of the GNU Affero General Public License
 //   along with iTop. If not, see <http://www.gnu.org/licenses/>
 
+use Combodo\iTop\Setup\FeatureRemoval\InplaceSetupAudit;
+use Combodo\iTop\Setup\FeatureRemoval\ModelReflectionSerializer;
+
 require_once(APPROOT.'setup/parameters.class.inc.php');
 require_once(APPROOT.'setup/xmldataloader.class.inc.php');
 require_once(APPROOT.'setup/backup.class.inc.php');
+require_once APPROOT.'setup/feature_removal/InplaceSetupAudit.php';
 
 /**
  * The base class for the installation process.
@@ -46,6 +50,8 @@ class ApplicationInstaller
 	protected $oParams;
 	protected static $bMetaModelStarted = false;
 
+	protected Config $oConfig;
+
 	/**
 	 * @param \Parameters $oParams
 	 *
@@ -57,9 +63,9 @@ class ApplicationInstaller
 		$this->oParams = $oParams;
 
 		$aParamValues = $oParams->GetParamForConfigArray();
-		$oConfig = new Config();
-		$oConfig->UpdateFromParams($aParamValues, null);
-		utils::SetConfig($oConfig);
+		$this->oConfig = new Config();
+		$this->oConfig->UpdateFromParams($aParamValues);
+		utils::SetConfig($this->oConfig);
 	}
 
 	/**
@@ -215,7 +221,7 @@ class ApplicationInstaller
 					$aPreinstall = $this->oParams->Get('preinstall');
 					$aCopies = $aPreinstall['copies'] ?? [];
 
-					self::DoCopy($aCopies);
+					$this->DoCopy($aCopies);
 					$sReport = "Copying...";
 
 					$aResult = [
@@ -238,11 +244,8 @@ class ApplicationInstaller
 					// __DB__-%Y-%m-%d
 					$sDestination = $aPreinstall['backup']['destination'];
 					$sSourceConfigFile = $aPreinstall['backup']['configuration_file'];
-					$aDBParams = $this->oParams->GetParamForConfigArray();
-					$oTempConfig = new Config();
-					$oTempConfig->UpdateFromParams($aDBParams);
 					$sMySQLBinDir = $this->oParams->Get('mysql_bindir', null);
-					self::DoBackup($oTempConfig, $sDestination, $sSourceConfigFile, $sMySQLBinDir);
+					$this->DoBackup($sDestination, $sSourceConfigFile, $sMySQLBinDir);
 
 					$aResult = [
 						'status' => self::OK,
@@ -257,9 +260,9 @@ class ApplicationInstaller
 					$aSelectedModules = $this->oParams->Get('selected_modules');
 					$sSourceDir = $this->oParams->Get('source_dir', 'datamodels/latest');
 					$sExtensionDir = $this->oParams->Get('extensions_dir', 'extensions');
-					$sTargetEnvironment = $this->GetTargetEnv();
-					$sTargetDir = $this->GetTargetDir();
 					$aMiscOptions = $this->oParams->Get('options', []);
+					$aRemovedExtensionCodes = $this->oParams->Get('removed_extensions', []);
+					$sSkipDataAudit = $this->oParams->Get('skip-data-audit', '');
 
 					$bUseSymbolicLinks = null;
 					if ((isset($aMiscOptions['symlinks']) && $aMiscOptions['symlinks'])) {
@@ -272,40 +275,49 @@ class ApplicationInstaller
 					}
 
 					$aParamValues = $this->oParams->GetParamForConfigArray();
-					self::DoCompile(
+					$bIsSetupDataAuditEnabled = $this->IsSetupDataAuditEnabled($sSkipDataAudit, $aParamValues);
+					$this->DoCompile(
+						$aRemovedExtensionCodes,
 						$aSelectedModules,
 						$sSourceDir,
 						$sExtensionDir,
-						$sTargetDir,
-						$sTargetEnvironment,
-						$bUseSymbolicLinks,
-						$aParamValues
+						$bIsSetupDataAuditEnabled,
+						$bUseSymbolicLinks
 					);
 
+					if ($bIsSetupDataAuditEnabled) {
+						$sNextStep = 'setup-audit';
+						$sNextStepLabel = 'Checking data consistency with the new data model';
+					} else {
+						$sNextStep = 'db-schema';
+						$sNextStepLabel = 'Updating database schema';
+					}
+
+					$aResult = [
+						'status' => self::OK,
+						'message' => '',
+						'next-step' => $sNextStep,
+						'next-step-label' => $sNextStepLabel,
+						'percentage-completed' => 40,
+					];
+					break;
+
+				case 'setup-audit':
+					$this->DoSetupAudit();
 					$aResult = [
 						'status' => self::OK,
 						'message' => '',
 						'next-step' => 'db-schema',
 						'next-step-label' => 'Updating database schema',
-						'percentage-completed' => 40,
+						'percentage-completed' => 50,
 					];
 					break;
 
 				case 'db-schema':
 					$aSelectedModules = $this->oParams->Get('selected_modules', []);
-					$sTargetEnvironment = $this->GetTargetEnv();
-					$sTargetDir = $this->GetTargetDir();
-					$aParamValues = $this->oParams->GetParamForConfigArray();
-					$bOldAddon = $this->oParams->Get('old_addon', false);
-					$sUrl = $this->oParams->Get('url', '');
 
-					self::DoUpdateDBSchema(
-						$aSelectedModules,
-						$sTargetDir,
-						$aParamValues,
-						$sTargetEnvironment,
-						$bOldAddon,
-						$sUrl
+					$this->DoUpdateDBSchema(
+						$aSelectedModules
 					);
 
 					$aResult = [
@@ -318,25 +330,17 @@ class ApplicationInstaller
 					break;
 
 				case 'after-db-create':
-					$sTargetEnvironment = $this->GetTargetEnv();
-					$sTargetDir = $this->GetTargetDir();
-					$aParamValues = $this->oParams->GetParamForConfigArray();
 					$aAdminParams = $this->oParams->Get('admin_account');
 					$sAdminUser = $aAdminParams['user'];
 					$sAdminPwd = $aAdminParams['pwd'];
 					$sAdminLanguage = $aAdminParams['language'];
 					$aSelectedModules = $this->oParams->Get('selected_modules', []);
-					$bOldAddon = $this->oParams->Get('old_addon', false);
 
-					self::AfterDBCreate(
-						$sTargetDir,
-						$aParamValues,
+					$this->AfterDBCreate(
 						$sAdminUser,
 						$sAdminPwd,
 						$sAdminLanguage,
-						$aSelectedModules,
-						$sTargetEnvironment,
-						$bOldAddon
+						$aSelectedModules
 					);
 
 					$aResult = [
@@ -350,18 +354,10 @@ class ApplicationInstaller
 
 				case 'load-data':
 					$aSelectedModules = $this->oParams->Get('selected_modules');
-					$sTargetEnvironment = $this->GetTargetEnv();
-					$sTargetDir = $this->GetTargetDir();
-					$aParamValues = $this->oParams->GetParamForConfigArray();
-					$bOldAddon = $this->oParams->Get('old_addon', false);
 					$bSampleData = ($this->oParams->Get('sample_data', 0) == 1);
 
-					self::DoLoadFiles(
+					$this->DoLoadFiles(
 						$aSelectedModules,
-						$sTargetDir,
-						$aParamValues,
-						$sTargetEnvironment,
-						$bOldAddon,
 						$bSampleData
 					);
 
@@ -375,24 +371,16 @@ class ApplicationInstaller
 					break;
 
 				case 'create-config':
-					$sTargetEnvironment = $this->GetTargetEnv();
-					$sTargetDir = $this->GetTargetDir();
 					$sPreviousConfigFile = $this->oParams->Get('previous_configuration_file', '');
 					$sDataModelVersion = $this->oParams->Get('datamodel_version', '0.0.0');
-					$bOldAddon = $this->oParams->Get('old_addon', false);
 					$aSelectedModuleCodes = $this->oParams->Get('selected_modules', []);
 					$aSelectedExtensionCodes = $this->oParams->Get('selected_extensions', []);
-					$aParamValues = $this->oParams->GetParamForConfigArray();
 
-					self::DoCreateConfig(
-						$sTargetDir,
+					$this->DoCreateConfig(
 						$sPreviousConfigFile,
-						$sTargetEnvironment,
 						$sDataModelVersion,
-						$bOldAddon,
 						$aSelectedModuleCodes,
 						$aSelectedExtensionCodes,
-						$aParamValues,
 						$sInstallComment
 					);
 
@@ -473,7 +461,7 @@ class ApplicationInstaller
 		SetupUtils::ExitReadOnlyMode();
 	}
 
-	protected static function DoCopy($aCopies)
+	protected function DoCopy($aCopies)
 	{
 		$aReports = [];
 		foreach ($aCopies as $aCopy) {
@@ -494,7 +482,6 @@ class ApplicationInstaller
 	}
 
 	/**
-	 * @param Config $oConfig
 	 * @param string $sBackupFileFormat
 	 * @param string $sSourceConfigFile
 	 * @param string $sMySQLBinDir
@@ -504,26 +491,25 @@ class ApplicationInstaller
 	 * @throws \MySQLException
 	 * @since 2.5.0 uses a {@link Config} object to store DB parameters
 	 */
-	protected static function DoBackup($oConfig, $sBackupFileFormat, $sSourceConfigFile, $sMySQLBinDir = null)
+	protected function DoBackup($sBackupFileFormat, $sSourceConfigFile, $sMySQLBinDir = null)
 	{
-		$oBackup = new SetupDBBackup($oConfig);
+		$oBackup = new SetupDBBackup($this->oConfig);
 		$sTargetFile = $oBackup->MakeName($sBackupFileFormat);
 		if (!empty($sMySQLBinDir)) {
 			$oBackup->SetMySQLBinDir($sMySQLBinDir);
 		}
 
-		CMDBSource::InitFromConfig($oConfig);
+		CMDBSource::InitFromConfig($this->oConfig);
 		$oBackup->CreateCompressedBackup($sTargetFile, $sSourceConfigFile);
 	}
 
 	/**
+	 * @param array $aRemovedExtensionCodes
 	 * @param array $aSelectedModules
 	 * @param string $sSourceDir
 	 * @param string $sExtensionDir
-	 * @param string $sTargetDir
-	 * @param string $sEnvironment
+	 * @param bool $bIsSetupDataAuditEnabled
 	 * @param boolean $bUseSymbolicLinks
-	 * @param array $aParamValues
 	 *
 	 * @return void
 	 * @throws \ConfigException
@@ -531,14 +517,24 @@ class ApplicationInstaller
 	 *
 	 * @since 3.1.0 N°2013 added the aParamValues param
 	 */
-	protected static function DoCompile($aSelectedModules, $sSourceDir, $sExtensionDir, $sTargetDir, $sEnvironment, $bUseSymbolicLinks = null, $aParamValues = [])
+	protected function DoCompile($aRemovedExtensionCodes, $aSelectedModules, $sSourceDir, $sExtensionDir, $bIsSetupDataAuditEnabled, $bUseSymbolicLinks = null)
 	{
+		/**
+	 * @since 3.2.0 move the ContextTag init at the very beginning of the method
+	 * @noinspection PhpUnusedLocalVariableInspection
+	 */
+		$oContextTag = new ContextTag(ContextTag::TAG_SETUP);
+
 		SetupLog::Info("Compiling data model.");
 
 		require_once(APPROOT.'setup/modulediscovery.class.inc.php');
 		require_once(APPROOT.'setup/modelfactory.class.inc.php');
 		require_once(APPROOT.'setup/compiler.class.inc.php');
 
+		$aParamValues = $this->oParams->GetParamForConfigArray();
+		$sEnvironment = $this->GetTargetEnv();
+		$sTargetDir = $this->GetTargetDir();
+
 		if (empty($sSourceDir) || empty($sTargetDir)) {
 			throw new Exception("missing parameter source_dir and/or target_dir");
 		}
@@ -560,20 +556,25 @@ class ApplicationInstaller
 		if (!is_dir($sSourcePath)) {
 			throw new Exception("Failed to find the source directory '$sSourcePath', please check the rights of the web server");
 		}
+
 		$bIsAlreadyInMaintenanceMode = SetupUtils::IsInMaintenanceMode();
+		if ($bIsSetupDataAuditEnabled) {
+			if ($bIsAlreadyInMaintenanceMode) {
+				//required to read DM before calling SaveModelInfo
+				SetupUtils::ExitMaintenanceMode();
+				$bIsAlreadyInMaintenanceMode = false;
+			}
+
+			$this->SaveModelInfo($sEnvironment);
+		}
+
 		if (($sEnvironment == 'production') && !$bIsAlreadyInMaintenanceMode) {
 			$sConfigFilePath = utils::GetConfigFilePath($sEnvironment);
 			if (is_file($sConfigFilePath)) {
 				$oConfig = new Config($sConfigFilePath);
-			} else {
-				$oConfig = null;
-			}
-
-			if (false === is_null($oConfig)) {
 				$oConfig->UpdateFromParams($aParamValues);
+				SetupUtils::EnterMaintenanceMode($oConfig);
 			}
-
-			SetupUtils::EnterMaintenanceMode($oConfig);
 		}
 
 		if (!is_dir($sTargetPath)) {
@@ -589,6 +590,9 @@ class ApplicationInstaller
 			SetupUtils::tidydir($sTargetPath);
 		}
 
+		$oExtensionsMap = new iTopExtensionsMap('production', $aDirsToScan);
+		$oExtensionsMap->DeclareExtensionAsRemoved($aRemovedExtensionCodes);
+
 		$oFactory = new ModelFactory($aDirsToScan);
 
 		$oDictModule = new MFDictModule('dictionaries', 'iTop Dictionaries', APPROOT.'dictionaries');
@@ -655,20 +659,86 @@ class ApplicationInstaller
 		}
 	}
 
+	private function GetModelInfoPath(string $sEnv): string
+	{
+		return APPROOT."data/beforecompilation_".$sEnv."_modelinfo.json";
+	}
+
+	private function SaveModelInfo(string $sEnvironment): void
+	{
+		$aModelInfo = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($sEnvironment);
+		$sModelInfoPath = $this->GetModelInfoPath($sEnvironment);
+		file_put_contents($sModelInfoPath, json_encode($aModelInfo));
+	}
+
+	private function GetPreviousModelInfo(string $sEnvironment): array
+	{
+		$sContent = file_get_contents($this->GetModelInfoPath($sEnvironment));
+		$aModelInfo = json_decode($sContent, true);
+
+		if (false === $aModelInfo) {
+			throw new Exception("Could not read (before compilation) previous model to audit data");
+		}
+
+		return $aModelInfo;
+	}
+
+	protected function DoSetupAudit()
+	{
+		/**
+		 * @since 3.2.0 move the ContextTag init at the very beginning of the method
+		 * @noinspection PhpUnusedLocalVariableInspection
+		 */
+		$oContextTag = new ContextTag(ContextTag::TAG_SETUP);
+
+		$sTargetEnvironment = $this->GetTargetEnv();
+		$aPreviousCompilationModelInfo = $this->GetPreviousModelInfo($sTargetEnvironment);
+
+		$oSetupAudit = new InplaceSetupAudit($aPreviousCompilationModelInfo, $sTargetEnvironment);
+
+		$oSetupAudit->GetIssues(true);
+		$iCount = $oSetupAudit->GetDataToCleanupCount();
+
+		if ($iCount > 0) {
+			throw new Exception("$iCount elements require data adjustments or cleanup in the backoffice prior to upgrading iTop");
+		}
+	}
+
+	private function IsSetupDataAuditEnabled($sSkipDataAudit, array $aParamValues): bool
+	{
+		if ($sSkipDataAudit === "checked") {
+			SetupLog::Info("Setup data audit disabled", null, ['skip-data-audit' => $sSkipDataAudit]);
+			return false;
+		}
+
+		$sMode = $aParamValues['mode'];
+		if ($sMode !== "upgrade") {
+			//first install
+			return false;
+		}
+
+		$sPath = APPROOT.$this->GetTargetDir();
+		if (!is_dir($sPath)) {
+			SetupLog::Info("Reinstallation of an iTop from a backup (No ".$this->GetTargetDir()." found). Setup data audit disabled", null, ['skip-data-audit' => $sSkipDataAudit]);
+
+			return false;
+		}
+
+		return true;
+	}
+
 	/**
 	 * @param $aSelectedModules
-	 * @param $sModulesDir
-	 * @param $aParamValues
-	 * @param string $sTargetEnvironment
-	 * @param bool $bOldAddon
-	 * @param string $sAppRootUrl
 	 *
 	 * @throws \ConfigException
 	 * @throws \CoreException
 	 * @throws \MySQLException
 	 */
-	protected static function DoUpdateDBSchema($aSelectedModules, $sModulesDir, $aParamValues, $sTargetEnvironment = '', $bOldAddon = false, $sAppRootUrl = '')
+	protected function DoUpdateDBSchema($aSelectedModules)
 	{
+		$sTargetEnvironment = $this->GetTargetEnv();
+		$sModulesDir = $this->GetTargetDir();
+		$aParamValues = $this->oParams->GetParamForConfigArray();
 		/**
 		 * @since 3.2.0 move the ContextTag init at the very beginning of the method
 		 * @noinspection PhpUnusedLocalVariableInspection
@@ -682,11 +752,6 @@ class ApplicationInstaller
 		$oConfig = new Config();
 		$oConfig->UpdateFromParams($aParamValues, $sModulesDir);
 
-		if ($bOldAddon) {
-			// Old version of the add-on for backward compatibility with pre-2.0 data models
-			$oConfig->SetAddons([]);
-		}
-
 		$oProductionEnv = new RunTimeEnvironment($sTargetEnvironment);
 		$oProductionEnv->InitDataModel($oConfig, true);  // load data model only
 
@@ -741,9 +806,8 @@ class ApplicationInstaller
 		}
 
 		// Module specific actions (migrate the data)
-		//
 		$aAvailableModules = $oProductionEnv->AnalyzeInstallation(MetaModel::GetConfig(), APPROOT.$sModulesDir);
-		$oProductionEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'BeforeDatabaseCreation');
+		$oProductionEnv->CallInstallerHandlers($aAvailableModules, 'BeforeDatabaseCreation', $aSelectedModules);
 
 		if (!$oProductionEnv->CreateDatabaseStructure(MetaModel::GetConfig(), $sMode)) {
 			throw new Exception("Failed to create/upgrade the database structure for environment '$sTargetEnvironment'");
@@ -839,16 +903,16 @@ class ApplicationInstaller
 		ModuleInstallerAPI::MoveColumnInDB($sDBPrefix.'priv_query', 'fields', $sDBPrefix.'priv_query_oql', 'fields');
 	}
 
-	protected static function AfterDBCreate(
-		$sModulesDir,
-		$aParamValues,
+	protected function AfterDBCreate(
 		$sAdminUser,
 		$sAdminPwd,
 		$sAdminLanguage,
-		$aSelectedModules,
-		$sTargetEnvironment,
-		$bOldAddon
+		$aSelectedModules
 	) {
+		$aParamValues = $this->oParams->GetParamForConfigArray();
+		$sTargetEnvironment = $this->GetTargetEnv();
+		$sModulesDir = $this->GetTargetDir();
+
 		/**
 		 * @since 3.2.0 move the ContextTag init at the very beginning of the method
 		 * @noinspection PhpUnusedLocalVariableInspection
@@ -860,11 +924,6 @@ class ApplicationInstaller
 		$oConfig = new Config();
 		$oConfig->UpdateFromParams($aParamValues, $sModulesDir);
 
-		if ($bOldAddon) {
-			// Old version of the add-on for backward compatibility with pre-2.0 data models
-			$oConfig->SetAddons([]);
-		}
-
 		$oProductionEnv = new RunTimeEnvironment($sTargetEnvironment);
 		$oProductionEnv->InitDataModel($oConfig, true);  // load data model and connect to the database
 		$oContextTag = new ContextTag(ContextTag::TAG_SETUP);
@@ -873,7 +932,7 @@ class ApplicationInstaller
 		// Perform here additional DB setup... profiles, etc...
 		//
 		$aAvailableModules = $oProductionEnv->AnalyzeInstallation(MetaModel::GetConfig(), APPROOT.$sModulesDir);
-		$oProductionEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseCreation');
+		$oProductionEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseCreation', $aSelectedModules);
 
 		$oProductionEnv->UpdatePredefinedObjects();
 
@@ -887,7 +946,7 @@ class ApplicationInstaller
 
 		// Perform final setup tasks here
 		//
-		$oProductionEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDatabaseSetup');
+		$oProductionEnv->CallInstallerHandlers($aAvailableModules, 'AfterDatabaseSetup', $aSelectedModules);
 	}
 
 	/**
@@ -905,14 +964,14 @@ class ApplicationInstaller
 		}
 	}
 
-	protected static function DoLoadFiles(
+	protected function DoLoadFiles(
 		$aSelectedModules,
-		$sModulesDir,
-		$aParamValues,
-		$sTargetEnvironment = 'production',
-		$bOldAddon = false,
 		$bSampleData = false
 	) {
+		$aParamValues = $this->oParams->GetParamForConfigArray();
+		$sTargetEnvironment = $this->GetTargetEnv();
+		$sModulesDir = $this->GetTargetDir();
+
 		/**
 		 * @since 3.2.0 move the ContextTag init at the very beginning of the method
 		 * @noinspection PhpUnusedLocalVariableInspection
@@ -922,11 +981,6 @@ class ApplicationInstaller
 		$oConfig = new Config();
 		$oConfig->UpdateFromParams($aParamValues, $sModulesDir);
 
-		if ($bOldAddon) {
-			// Old version of the add-on for backward compatibility with pre-2.0 data models
-			$oConfig->SetAddons([]);
-		}
-
 		$oProductionEnv = new RunTimeEnvironment($sTargetEnvironment);
 
 		//Load the MetaModel if needed (asynchronous mode)
@@ -937,22 +991,19 @@ class ApplicationInstaller
 		}
 
 		$aAvailableModules = $oProductionEnv->AnalyzeInstallation($oConfig, APPROOT.$sModulesDir);
-		$oProductionEnv->LoadData($aAvailableModules, $aSelectedModules, $bSampleData);
+		$oProductionEnv->LoadData($aAvailableModules, $bSampleData, $aSelectedModules);
 
 		// Perform after dbload setup tasks here
 		//
-		$oProductionEnv->CallInstallerHandlers($aAvailableModules, $aSelectedModules, 'AfterDataLoad');
+		$oProductionEnv->CallInstallerHandlers($aAvailableModules, 'AfterDataLoad', $aSelectedModules);
 	}
 
 	/**
-	 * @param string $sModulesDir
 	 * @param string $sPreviousConfigFile
-	 * @param string $sTargetEnvironment
 	 * @param string $sDataModelVersion
-	 * @param boolean $bOldAddon
 	 * @param array $aSelectedModuleCodes
 	 * @param array $aSelectedExtensionCodes
-	 * @param array $aParamValues parameters array used to create config file using {@see Config::UpdateFromParams}
+	 * @param string|null $sInstallComment
 	 *
 	 * @param null $sInstallComment
 	 *
@@ -960,17 +1011,17 @@ class ApplicationInstaller
 	 * @throws \CoreException
 	 * @throws \Exception
 	 */
-	protected static function DoCreateConfig(
-		$sModulesDir,
+	protected function DoCreateConfig(
 		$sPreviousConfigFile,
-		$sTargetEnvironment,
 		$sDataModelVersion,
-		$bOldAddon,
 		$aSelectedModuleCodes,
 		$aSelectedExtensionCodes,
-		$aParamValues,
 		$sInstallComment = null
 	) {
+		$aParamValues = $this->oParams->GetParamForConfigArray();
+		$sTargetEnvironment = $this->GetTargetEnv();
+		$sModulesDir = $this->GetTargetDir();
+
 		/**
 		 * @since 3.2.0 move the ContextTag init at the very beginning of the method
 		 * @noinspection PhpUnusedLocalVariableInspection
@@ -980,12 +1031,10 @@ class ApplicationInstaller
 		$aParamValues['selected_modules'] = implode(',', $aSelectedModuleCodes);
 		$sMode = $aParamValues['mode'];
 
-		$bPreserveModuleSettings = false;
 		if ($sMode == 'upgrade') {
 			try {
 				$oOldConfig = new Config($sPreviousConfigFile);
 				$oConfig = clone($oOldConfig);
-				$bPreserveModuleSettings = true;
 			} catch (Exception $e) {
 				// In case the previous configuration is corrupted... start with a blank new one
 				$oConfig = new Config();
@@ -999,11 +1048,7 @@ class ApplicationInstaller
 
 		$oConfig->Set('access_mode', ACCESS_FULL);
 		// Final config update: add the modules
-		$oConfig->UpdateFromParams($aParamValues, $sModulesDir, $bPreserveModuleSettings);
-		if ($bOldAddon) {
-			// Old version of the add-on for backward compatibility with pre-2.0 data models
-			$oConfig->SetAddons([]);
-		}
+		$oConfig->UpdateFromParams($aParamValues, $sModulesDir);
 
 		// Record which modules are installed...
 		$oProductionEnv = new RunTimeEnvironment($sTargetEnvironment);

setup/compiler.class.inc.php

@@ -21,8 +21,8 @@
 use Combodo\iTop\Application\Branding;
 use Combodo\iTop\Application\WebPage\iTopWebPage;
 use Combodo\iTop\Application\WebPage\Page;
-use Combodo\iTop\DesignElement;
 use Combodo\iTop\DesignDocument;
+use Combodo\iTop\DesignElement;
 
 require_once(APPROOT.'setup/setuputils.class.inc.php');
 require_once(APPROOT.'setup/modelfactory.class.inc.php');
@@ -3359,6 +3359,8 @@ EOF;
 
 		$bDataXmlPrecompiledFileExists = false;
 		clearstatcache();
+
+		$iDataXmlFileLastModified = 0;
 		if (!empty($sPrecompiledFileUri)) {
 			$sDataXmlProvidedPrecompiledFile = $sTempTargetDir.DIRECTORY_SEPARATOR.$sPrecompiledFileUri;
 			$bDataXmlPrecompiledFileExists = file_exists($sDataXmlProvidedPrecompiledFile) ;

setup/email.test.php

@@ -34,7 +34,7 @@ require_once(APPROOT.'/application/startup.inc.php');
 require_once(APPROOT.'/application/loginwebpage.class.inc.php');
 LoginWebPage::DoLogin(true); // Check user rights and prompt if needed (must be admin)
 
-$sOperation = Utils::ReadParam('operation', 'step1');
+$sOperation = utils::ReadParam('operation', 'step1');
 $oP = new SetupPage('iTop email test utility');
 
 // Although this page doesn't expose sensitive info, with it we can send multiple emails
@@ -208,7 +208,7 @@ function DisplayStep2(SetupPage $oP, $sFrom, $sTo)
 	$oP->add("<p>Sending an email to '".htmlentities($sTo, ENT_QUOTES, 'utf-8')."'... (From: '".htmlentities($sFrom, ENT_QUOTES, 'utf-8')."')</p>\n");
 	$oP->add("<form method=\"post\">\n");
 
-	$oEmail = new Email();
+	$oEmail = new EMail();
 	$oEmail->SetRecipientTO($sTo);
 	$oEmail->SetRecipientFrom($sFrom);
 	$oEmail->SetSubject("Test iTop");
@@ -256,8 +256,8 @@ try {
 
 		case 'step2':
 			$oP->no_cache();
-			$sTo = Utils::ReadParam('to', '', false, 'raw_data');
-			$sFrom = Utils::ReadParam('from', '', false, 'raw_data');
+			$sTo = utils::ReadParam('to', '', false, 'raw_data');
+			$sFrom = utils::ReadParam('from', '', false, 'raw_data');
 			DisplayStep2($oP, $sFrom, $sTo);
 			break;
 

setup/extensionsmap.class.inc.php

@@ -3,143 +3,11 @@
 use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReader;
 use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReaderException;
 
+require_once(APPROOT.'/setup/itopextension.class.inc.php');
 require_once(APPROOT.'/setup/parameters.class.inc.php');
 require_once(APPROOT.'/core/cmdbsource.class.inc.php');
 require_once(APPROOT.'/setup/modulediscovery.class.inc.php');
 require_once(APPROOT.'/setup/moduleinstaller.class.inc.php');
-/**
- * Basic helper class to describe an extension, with some characteristics and a list of modules
- */
-class iTopExtension
-{
-	public const SOURCE_WIZARD = 'datamodels';
-	public const SOURCE_MANUAL = 'extensions';
-	public const SOURCE_REMOTE = 'data';
-
-	/**
-	 * @var string
-	 */
-	public $sCode;
-
-	/**
-	 * @var string
-	 */
-	public $sVersion;
-
-	/**
-	 * @var string
-	 */
-	public $sInstalledVersion;
-
-	/**
-	 * @var string
-	 */
-	public $sLabel;
-
-	/**
-	 * @var string
-	 */
-	public $sDescription;
-
-	/**
-	 * @var string
-	 */
-	public $sSource;
-
-	/**
-	 * @var bool
-	 */
-	public $bMandatory;
-
-	/**
-	 * @var string
-	 */
-	public $sMoreInfoUrl;
-
-	/**
-	 * @var bool
-	 */
-	public $bMarkedAsChosen;
-	/**
-	 * If null, check if at least one module cannot be uninstalled
-	 * @var bool|null
-	 */
-	public ?bool $bCanBeUninstalled = null;
-
-	/**
-	 * @var bool
-	 */
-	public $bVisible;
-
-	/**
-	 * @var string[]
-	 */
-	public $aModules;
-
-	/**
-	 * @var string[]
-	 */
-	public $aModuleVersion;
-
-	/**
-	 * @var string[]
-	 */
-	public $aModuleInfo;
-
-	/**
-	 * @var string
-	 */
-	public $sSourceDir;
-
-	/**
-	 *
-	 * @var string[]
-	 */
-	public $aMissingDependencies;
-	/**
-	 * @var bool
-	 */
-	public bool $bInstalled = false;
-	/**
-	 * @var bool
-	 */
-	public bool $bRemovedFromDisk = false;
-
-	public function __construct()
-	{
-		$this->sCode = '';
-		$this->sLabel = '';
-		$this->sDescription = '';
-		$this->sSource = self::SOURCE_WIZARD;
-		$this->bMandatory = false;
-		$this->sMoreInfoUrl = '';
-		$this->bMarkedAsChosen = false;
-		$this->sVersion = ITOP_VERSION;
-		$this->sInstalledVersion = '';
-		$this->aModules = [];
-		$this->aModuleVersion = [];
-		$this->aModuleInfo = [];
-		$this->sSourceDir = '';
-		$this->bVisible = true;
-		$this->aMissingDependencies = [];
-	}
-
-	/**
-	 * @since 3.3.0
-	 * @return bool
-	 */
-	public function CanBeUninstalled(): bool
-	{
-		if (!is_null($this->bCanBeUninstalled)) {
-			return $this->bCanBeUninstalled;
-		}
-		foreach ($this->aModuleInfo as $sModuleCode => $aModuleInfo) {
-			$this->bCanBeUninstalled = $aModuleInfo['uninstallable'] === 'yes';
-			return $this->bCanBeUninstalled;
-		}
-		return true;
-	}
-}
 
 /**
  * Helper class to discover all available extensions on a given iTop system
@@ -308,28 +176,26 @@ class iTopExtensionsMap
 		return $this->aExtensionsByCode[$sExtensionCode] ?? null;
 	}
 
-	/*public function GetMissingExtensions(array $aSelectedExtensions)
+	/**
+	* @param array<string> $aExtensionCodes
+	* @return void
+	*/
+	public function DeclareExtensionAsRemoved(array $aExtensionCodes): void
 	{
-		\SetupLog::Info(__METHOD__, null, ['selected' => $aSelectedExtensions]);
-		$aExtensionsFromDb = array_keys($this->aExtensionsByCode);
-		sort($aExtensionsFromDb);
-		\SetupLog::Info(__METHOD__, null, ['found' => $aExtensionsFromDb]);
-
-		$aRes = [];
-		foreach (array_diff($aExtensionsFromDb, $aSelectedExtensions) as $sExtensionCode) {
-			$oExtension = $this->GetFromExtensionCode($sExtensionCode);
-			if (!is_null($oExtension) && $oExtension->bVisible && $oExtension->sSource != iTopExtension::SOURCE_WIZARD) {
-
-				\SetupLog::Info(__METHOD__."$sExtensionCode", null, ['visible' => $oExtension->bVisible, 'mandatory' => $oExtension->bMandatory]);
-				$aRes [] = $sExtensionCode;
+		$aRemovedExtension = [];
+		foreach ($aExtensionCodes as $sCode) {
+			/** @var \iTopExtension $oExtension */
+			$oExtension = $this->GetFromExtensionCode($sCode);
+			if (!is_null($oExtension)) {
+				$aRemovedExtension [] = $oExtension;
+				\IssueLog::Info(__METHOD__.": remove extension locally", null, ['extension_code' => $oExtension->sCode]);
 			} else {
-				\SetupLog::Info(__METHOD__." MISSING $sExtensionCode");
+				\IssueLog::Warning(__METHOD__." cannot find extensions", null, ['code' => $sCode]);
 			}
 		}
-		\SetupLog::Info(__METHOD__, null, $aRes);
 
-		return $aRes;
-	}*/
+		\ModuleDiscovery::DeclareRemovedExtensions($aRemovedExtension);
+	}
 
 	/**
 	 * Read (recursively) a directory to find if it contains extensions (or modules)

setup/feature_removal/AbstractSetupAudit.php

@@ -0,0 +1,105 @@
+<?php
+
+namespace Combodo\iTop\Setup\FeatureRemoval;
+
+use ContextTag;
+use DBObjectSearch;
+use DBObjectSet;
+use IssueLog;
+use MetaModel;
+use SetupLog;
+
+require_once APPROOT.'setup/feature_removal/ModelReflectionSerializer.php';
+
+abstract class AbstractSetupAudit
+{
+	protected bool $bClassesInitialized = false;
+	protected array $aClassesBefore = [];
+	protected array $aClassesAfter = [];
+	protected array $aRemovedClasses = [];
+	protected array $aFinalClassesToCleanup = [];
+
+	public function __construct()
+	{
+	}
+
+	abstract public function ComputeClasses(): void;
+
+	public function GetRemovedClasses(): array
+	{
+		$this->ComputeClasses();
+
+		if (count($this->aRemovedClasses) == 0) {
+			if (count($this->aClassesBefore) == 0) {
+				return $this->aRemovedClasses;
+			}
+
+			if (count($this->aClassesAfter) == 0) {
+				return $this->aRemovedClasses;
+			}
+
+			$aExtensionsNames = array_diff($this->aClassesBefore, $this->aClassesAfter);
+			$this->aRemovedClasses = [];
+			$aClasses = array_values($aExtensionsNames);
+			sort($aClasses);
+
+			foreach ($aClasses as $i => $sClass) {
+				$this->aRemovedClasses[] = $sClass;
+			}
+		}
+
+		return $this->aRemovedClasses;
+	}
+
+	public function GetIssues(bool $bStopDataCheckAtFirstIssue = false): array
+	{
+		$this->aFinalClassesToCleanup = [];
+
+		foreach ($this->GetRemovedClasses() as $sClass) {
+			if (MetaModel::IsAbstract($sClass)) {
+				continue;
+			}
+
+			if (!MetaModel::IsStandaloneClass($sClass)) {
+				$iCount = $this->Count($sClass);
+				$this->aFinalClassesToCleanup[$sClass] = $iCount;
+				if ($bStopDataCheckAtFirstIssue && $iCount > 0) {
+					//setup envt: should raise issue ASAP
+					$this->LogInfoWithProperLogger("Setup audit found data to cleanup", null, $this->aFinalClassesToCleanup);
+					return $this->aFinalClassesToCleanup;
+				}
+			}
+		}
+
+		$this->LogInfoWithProperLogger("Setup audit found data to cleanup", null, ['data_to_cleanup' => $this->aFinalClassesToCleanup]);
+		return $this->aFinalClassesToCleanup;
+	}
+
+	public function GetDataToCleanupCount(): int
+	{
+		$res = 0;
+		foreach ($this->aFinalClassesToCleanup as $sClass => $iCount) {
+			$res += $iCount;
+		}
+		return $res;
+	}
+
+	private function Count($sClass): int
+	{
+		$oSearch = DBObjectSearch::FromOQL("SELECT $sClass", []);
+		$oSearch->AllowAllData();
+		$oSet = new DBObjectSet($oSearch);
+
+		return $oSet->Count();
+	}
+
+	//could be shared with others in log APIs ?
+	private function LogInfoWithProperLogger($sMessage, $sChannel = null, $aContext = []): void
+	{
+		if (ContextTag::Check(ContextTag::TAG_SETUP)) {
+			SetupLog::Info($sMessage, $sChannel, $aContext);
+		} else {
+			IssueLog::Info($sMessage, $sChannel, $aContext);
+		}
+	}
+}

setup/feature_removal/DryRemovalRuntimeEnvironment.php

@@ -2,6 +2,7 @@
 
 namespace Combodo\iTop\Setup\FeatureRemoval;
 
+use iTopExtensionsMap;
 use MetaModel;
 use RunTimeEnvironment;
 use SetupUtils;
@@ -11,7 +12,6 @@ class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 	public const DRY_REMOVAL_AUDIT_ENV = "extension-removal";
 
 	protected array $aExtensionsByCode;
-	private bool $bExtensionMapModified = false;
 
 	/**
 	 * Toolset for building a run-time environment
@@ -41,29 +41,16 @@ class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 		$this->Cleanup();
 		SetupUtils::copydir(APPROOT."/data/$sSourceEnv-modules", APPROOT."/data/$sEnv-modules");
 
-		if (count($aExtensionCodesToRemove) > 0) {
-			$this->RemoveExtensionsLocally($aExtensionCodesToRemove);
-		}
+		$this->DeclareExtensionAsRemoved($aExtensionCodesToRemove);
 		$oDryRemovalConfig = clone(MetaModel::GetConfig());
 		$oDryRemovalConfig->ChangeModulesPath($sSourceEnv, $this->sFinalEnv);
 		$this->WriteConfigFileSafe($oDryRemovalConfig);
 	}
 
-	private function RemoveExtensionsLocally(array $aExtensionCodes): void
+	private function DeclareExtensionAsRemoved(array $aExtensionCodes): void
 	{
-		$oExtensionsMap = new \iTopExtensionsMap($this->sFinalEnv);
-
-		foreach ($aExtensionCodes as $sCode) {
-			/** @var \iTopExtension $oExtension */
-			$oExtension = $oExtensionsMap->GetFromExtensionCode($sCode);
-			if (!is_null($oExtension)) {
-				$sDir = $oExtension->sSourceDir;
-				\IssueLog::Info(__METHOD__.": remove extension locally", null, [$oExtension->sCode => $sDir]);
-				SetupUtils::rrmdir($sDir);
-			} else {
-				\IssueLog::Warning(__METHOD__." cannot find extensions", null, ['env' => $this->sFinalEnv, 'code' => $sCode]);
-			}
-		}
+		$oExtensionsMap = new iTopExtensionsMap($this->sFinalEnv);
+		$oExtensionsMap->DeclareExtensionAsRemoved($aExtensionCodes);
 	}
 
 	public function Cleanup()
@@ -75,23 +62,4 @@ class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 		SetupUtils::rrmdir(APPROOT."/conf/$sEnv");
 		@unlink(APPROOT."/data/datamodel-$sEnv.xml");
 	}
-
-	/**
-	 * @return \iTopExtensionsMap|null
-	 */
-	/*protected function GetExtensionMap(): ?iTopExtensionsMap
-	{
-		if (is_null(parent::GetExtensionMap())) {
-			return null;
-		}
-
-		if (!$this->bExtensionMapModified) {
-			$this->bExtensionMapModified = true;
-			foreach ($this->aExtensionsByCode as $sCode) {
-				parent::GetExtensionMap()->RemoveExtension($sCode);
-			}
-		}
-
-		return parent::GetExtensionMap();
-	}*/
 }

setup/feature_removal/InplaceSetupAudit.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace Combodo\iTop\Setup\FeatureRemoval;
+
+use MetaModel;
+
+require_once __DIR__.'/AbstractSetupAudit.php';
+require_once APPROOT.'setup/feature_removal/ModelReflectionSerializer.php';
+
+class InplaceSetupAudit extends AbstractSetupAudit
+{
+	//file used when present to trigger audit exception when testing specific setups
+	public const GETISSUE_ERROR_MSG_FILE_FORTESTONLY = '.setup_audit_error_msg.txt';
+
+	private string $sEnvAfter;
+
+	public function __construct(array $aClassesBefore, string $sEnvAfter)
+	{
+		parent::__construct();
+		$this->aClassesBefore = $aClassesBefore;
+		$this->sEnvAfter = $sEnvAfter;
+	}
+
+	public function ComputeClasses(): void
+	{
+		if ($this->bClassesInitialized) {
+			return;
+		}
+
+		$sCurrentEnvt = MetaModel::GetEnvironment();
+
+		if ($sCurrentEnvt === $this->sEnvAfter) {
+			$this->aClassesAfter = MetaModel::GetClasses();
+		} else {
+			$this->aClassesAfter = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($this->sEnvAfter);
+		}
+
+		$this->bClassesInitialized = true;
+	}
+}

setup/feature_removal/ModelReflectionSerializer.php

@@ -2,8 +2,12 @@
 
 namespace Combodo\iTop\Setup\FeatureRemoval;
 
+use ContextTag;
 use CoreException;
 use Exception;
+use IssueLog;
+use SetupLog;
+use utils;
 
 class ModelReflectionSerializer
 {
@@ -29,27 +33,38 @@ class ModelReflectionSerializer
 
 	public function GetModelFromEnvironment(string $sEnv): array
 	{
-		\IssueLog::Info(__METHOD__, null, ['env' => $sEnv]);
-		$sPHPExec = trim(\MetaModel::GetConfig()->Get('php_path'));
+		IssueLog::Info(__METHOD__, null, ['env' => $sEnv]);
+		$sPHPExec = trim(utils::GetConfig()->Get('php_path'));
 		$sOutput = "";
 		$iRes = 0;
+
 		exec(sprintf("$sPHPExec %s/get_model_reflection.php --env='%s'", __DIR__, $sEnv), $sOutput, $iRes);
 		if ($iRes != 0) {
-			\IssueLog::Error("Cannot get classes", null, ['env' => $sEnv, 'code' => $iRes, "output" => $sOutput]);
+			$this->LogErrorWithProperLogger("Cannot get classes", null, ['env' => $sEnv, 'code' => $iRes, "output" => $sOutput]);
 			throw new CoreException("Cannot get classes");
 		}
 
 		$aClasses = json_decode($sOutput[0] ?? null, true);
 		if (false === $aClasses) {
-			\IssueLog::Error("Invalid JSON", null, ["output" => $sOutput]);
+			$this->LogErrorWithProperLogger("Invalid JSON", null, ['env' => $sEnv, "output" => $sOutput]);
 			throw new Exception("cannot get classes");
 		}
 
 		if (!is_array($aClasses)) {
-			\IssueLog::Error("not an array", null, ["classes" => $aClasses]);
-			throw new Exception("cannot get classes");
+			$this->LogErrorWithProperLogger("not an array", null, ['env' => $sEnv, "classes" => $aClasses, "output" => $sOutput]);
+			throw new Exception("cannot get classes from $sEnv");
 		}
 
 		return $aClasses;
 	}
+
+	//could be shared with others in log APIs ?
+	private function LogErrorWithProperLogger($sMessage, $sChannel = null, $aContext = []): void
+	{
+		if (ContextTag::Check(ContextTag::TAG_SETUP)) {
+			SetupLog::Error($sMessage, $sChannel, $aContext);
+		} else {
+			IssueLog::Error($sMessage, $sChannel, $aContext);
+		}
+	}
 }

setup/feature_removal/SetupAudit.php

@@ -2,45 +2,46 @@
 
 namespace Combodo\iTop\Setup\FeatureRemoval;
 
-use DBObjectSearch;
-use DBObjectSet;
 use MetaModel;
 
+require_once __DIR__.'/AbstractSetupAudit.php';
 require_once APPROOT.'setup/feature_removal/ModelReflectionSerializer.php';
 
-class SetupAudit
+class SetupAudit extends AbstractSetupAudit
 {
 	//file used when present to trigger audit exception when testing specific setups
 	public const GETISSUE_ERROR_MSG_FILE_FORTESTONLY = '.setup_audit_error_msg.txt';
 
-	private string $sEnvBeforeExtensionRemoval;
-	private string $sEnvAfterExtensionRemoval;
+	private string $sEnvBefore;
+	private string $sEnvAfter;
 
-	private array $aClassesBeforeRemoval;
-	private array $aClassesAfterRemoval;
-	private array $aRemovedClasses;
-	private array $aFinalClassesRemoved;
-
-	public function __construct(string $sEnvBeforeExtensionRemoval, string $sEnvAfterExtensionRemoval = DryRemovalRuntimeEnvironment::DRY_REMOVAL_AUDIT_ENV)
+	public function __construct(string $sEnvBefore, string $sEnvAfter)
 	{
-		$this->sEnvBeforeExtensionRemoval = $sEnvBeforeExtensionRemoval;
-		$this->sEnvAfterExtensionRemoval = $sEnvAfterExtensionRemoval;
+		parent::__construct();
+		$this->sEnvBefore = $sEnvBefore;
+		$this->sEnvAfter = $sEnvAfter;
+	}
+
+	public function ComputeClasses(): void
+	{
+		if ($this->bClassesInitialized) {
+			return;
+		}
 
 		$sCurrentEnvt = MetaModel::GetEnvironment();
-		if ($sCurrentEnvt === $this->sEnvBeforeExtensionRemoval) {
-			$this->aClassesBeforeRemoval = MetaModel::GetClasses();
+		if ($sCurrentEnvt === $this->sEnvBefore) {
+			$this->aClassesBefore = MetaModel::GetClasses();
 		} else {
-			$this->aClassesBeforeRemoval = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($this->sEnvBeforeExtensionRemoval);
+			$this->aClassesBefore = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($this->sEnvBefore);
 		}
 
-		if ($sCurrentEnvt === $this->sEnvAfterExtensionRemoval) {
-			$this->aClassesAfterRemoval = MetaModel::GetClasses();
+		if ($sCurrentEnvt === $this->sEnvAfter) {
+			$this->aClassesAfter = MetaModel::GetClasses();
 		} else {
-			$this->aClassesAfterRemoval = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($this->sEnvAfterExtensionRemoval);
+			$this->aClassesAfter = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($this->sEnvAfter);
 		}
 
-		$this->aRemovedClasses = [];
-		$this->aFinalClassesRemoved = [];
+		$this->bClassesInitialized = true;
 	}
 
 	/*public function SetSelectedExtensions(Config $oConfig, array $aSelectedExtensions)
@@ -56,16 +57,18 @@ class SetupAudit
 
 	public function GetRemovedClasses(): array
 	{
+		$this->ComputeClasses();
+
 		if (count($this->aRemovedClasses) == 0) {
-			if (count($this->aClassesBeforeRemoval) == 0) {
+			if (count($this->aClassesBefore) == 0) {
 				return $this->aRemovedClasses;
 			}
 
-			if (count($this->aClassesAfterRemoval) == 0) {
+			if (count($this->aClassesAfter) == 0) {
 				return $this->aRemovedClasses;
 			}
 
-			$aExtensionsNames = array_diff($this->aClassesBeforeRemoval, $this->aClassesAfterRemoval);
+			$aExtensionsNames = array_diff($this->aClassesBefore, $this->aClassesAfter);
 			$this->aRemovedClasses = [];
 			$aClasses = array_values($aExtensionsNames);
 			sort($aClasses);
@@ -77,50 +80,4 @@ class SetupAudit
 
 		return $this->aRemovedClasses;
 	}
-
-	/** test only: return file path that force audit error being raised
-	 *
-	 * @return string
-	 */
-	public static function GetErrorMessageFilePathForTestOnly(): string
-	{
-		return APPROOT."/data/".self::GETISSUE_ERROR_MSG_FILE_FORTESTONLY;
-	}
-
-	public function GetIssues(bool $bThrowExceptionAtFirstIssue = false): array
-	{
-		$sErrorMessageFilePath = self::GetErrorMessageFilePathForTestOnly();
-		if ($bThrowExceptionAtFirstIssue && is_file($sErrorMessageFilePath)) {
-			$sMsg = file_get_contents($sErrorMessageFilePath);
-			throw new \Exception($sMsg);
-		}
-
-		$this->aFinalClassesRemoved = [];
-
-		foreach ($this->GetRemovedClasses() as $sClass) {
-			if (MetaModel::IsAbstract($sClass)) {
-				continue;
-			}
-
-			if (!MetaModel::IsStandaloneClass($sClass)) {
-				$iCount = $this->Count($sClass);
-				$this->aFinalClassesRemoved[$sClass] = $iCount;
-				if ($bThrowExceptionAtFirstIssue && $iCount > 0) {
-					//setup envt: should raise issue ASAP
-					throw new \Exception($sClass);
-				}
-			}
-		}
-
-		return $this->aFinalClassesRemoved;
-	}
-
-	private function Count($sClass): int
-	{
-		$oSearch = DBObjectSearch::FromOQL("SELECT $sClass", []);
-		$oSearch->AllowAllData();
-		$oSet = new DBObjectSet($oSearch);
-
-		return $oSet->Count();
-	}
 }

setup/itopextension.class.inc.php

@@ -0,0 +1,143 @@
+<?php
+
+use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReader;
+use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReaderException;
+
+require_once(APPROOT.'/setup/parameters.class.inc.php');
+require_once(APPROOT.'/core/cmdbsource.class.inc.php');
+require_once(APPROOT.'/setup/modulediscovery.class.inc.php');
+require_once(APPROOT.'/setup/moduleinstaller.class.inc.php');
+
+/**
+ * Basic helper class to describe an extension, with some characteristics and a list of modules
+ */
+class iTopExtension
+{
+	public const SOURCE_WIZARD = 'datamodels';
+	public const SOURCE_MANUAL = 'extensions';
+	public const SOURCE_REMOTE = 'data';
+
+	/**
+	 * @var string
+	 */
+	public $sCode;
+
+	/**
+	 * @var string
+	 */
+	public $sVersion;
+
+	/**
+	 * @var string
+	 */
+	public $sInstalledVersion;
+
+	/**
+	 * @var string
+	 */
+	public $sLabel;
+
+	/**
+	 * @var string
+	 */
+	public $sDescription;
+
+	/**
+	 * @var string
+	 */
+	public $sSource;
+
+	/**
+	 * @var bool
+	 */
+	public $bMandatory;
+
+	/**
+	 * @var string
+	 */
+	public $sMoreInfoUrl;
+
+	/**
+	 * @var bool
+	 */
+	public $bMarkedAsChosen;
+	/**
+	 * If null, check if at least one module cannot be uninstalled
+	 * @var bool|null
+	 */
+	public ?bool $bCanBeUninstalled = null;
+
+	/**
+	 * @var bool
+	 */
+	public $bVisible;
+
+	/**
+	 * @var string[]
+	 */
+	public $aModules;
+
+	/**
+	 * @var string[]
+	 */
+	public $aModuleVersion;
+
+	/**
+	 * @var string[]
+	 */
+	public $aModuleInfo;
+
+	/**
+	 * @var string
+	 */
+	public $sSourceDir;
+
+	/**
+	 *
+	 * @var string[]
+	 */
+	public $aMissingDependencies;
+	/**
+	 * @var bool
+	 */
+	public bool $bInstalled = false;
+	/**
+	 * @var bool
+	 */
+	public bool $bRemovedFromDisk = false;
+
+	public function __construct()
+	{
+		$this->sCode = '';
+		$this->sLabel = '';
+		$this->sDescription = '';
+		$this->sSource = self::SOURCE_WIZARD;
+		$this->bMandatory = false;
+		$this->sMoreInfoUrl = '';
+		$this->bMarkedAsChosen = false;
+		$this->sVersion = ITOP_VERSION;
+		$this->sInstalledVersion = '';
+		$this->aModules = [];
+		$this->aModuleVersion = [];
+		$this->aModuleInfo = [];
+		$this->sSourceDir = '';
+		$this->bVisible = true;
+		$this->aMissingDependencies = [];
+	}
+
+	/**
+	 * @since 3.3.0
+	 * @return bool
+	 */
+	public function CanBeUninstalled(): bool
+	{
+		if (!is_null($this->bCanBeUninstalled)) {
+			return $this->bCanBeUninstalled;
+		}
+		foreach ($this->aModuleInfo as $sModuleCode => $aModuleInfo) {
+			$this->bCanBeUninstalled = $aModuleInfo['uninstallable'] === 'yes';
+			return $this->bCanBeUninstalled;
+		}
+		return true;
+	}
+}

setup/moduledependency/dependencyexpression.class.inc.php

@@ -5,7 +5,7 @@ namespace Combodo\iTop\Setup\ModuleDependency;
 require_once(APPROOT.'/setup/runtimeenv.class.inc.php');
 
 use Combodo\iTop\PhpParser\Evaluation\PhpExpressionEvaluator;
-use ModuleFileReaderException;
+use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReaderException;
 use RunTimeEnvironment;
 
 /**
@@ -63,17 +63,17 @@ class DependencyExpression
 
 	private static function GetPhpExpressionEvaluator(): PhpExpressionEvaluator
 	{
-		if (!isset(static::$oPhpExpressionEvaluator)) {
-			static::$oPhpExpressionEvaluator = new PhpExpressionEvaluator([], RunTimeEnvironment::STATIC_CALL_AUTOSELECT_WHITELIST);
+		if (!isset(self::$oPhpExpressionEvaluator)) {
+			self::$oPhpExpressionEvaluator = new PhpExpressionEvaluator([], RunTimeEnvironment::STATIC_CALL_AUTOSELECT_WHITELIST);
 		}
 
-		return static::$oPhpExpressionEvaluator;
+		return self::$oPhpExpressionEvaluator;
 	}
 
 	/**
 	 * Return module names potentially required by current dependency
 	 *
-	 * @return array
+	 * @return array<string>
 	 */
 	public function GetRemainingModuleNamesToResolve(): array
 	{

setup/moduledependency/module.class.inc.php

@@ -114,7 +114,7 @@ class Module
 	}
 
 	/**
-	 * @return array: list of unique module names
+	 * @return array<string> list of unique module names
 	 */
 	public function GetUnresolvedDependencyModuleNames(): array
 	{

setup/moduledependency/moduledependencysort.class.inc.php

@@ -19,16 +19,16 @@ class ModuleDependencySort
 
 	final public static function GetInstance(): ModuleDependencySort
 	{
-		if (!isset(static::$oInstance)) {
-			static::$oInstance = new static();
+		if (!isset(self::$oInstance)) {
+			self::$oInstance = new ModuleDependencySort();
 		}
 
-		return static::$oInstance;
+		return self::$oInstance;
 	}
 
 	final public static function SetInstance(?ModuleDependencySort $oInstance): void
 	{
-		static::$oInstance = $oInstance;
+		self::$oInstance = $oInstance;
 	}
 
 	/**
@@ -168,7 +168,7 @@ class ModuleDependencySort
 			foreach ($aCountDepsByModuleId as $sModuleId => $iInDegreeCounter) {
 				$oModule = $aUnresolvedDependencyModules[$sModuleId];
 
-				if ($bOneLoopAtLeast && $iInDegreeCounter > 0) {
+				if ($bOneLoopAtLeast && ($iInDegreeCounter > 0)) {
 					break;
 				}
 

setup/modulediscovery.class.inc.php

@@ -22,13 +22,13 @@
 
 use Combodo\iTop\PhpParser\Evaluation\PhpExpressionEvaluator;
 use Combodo\iTop\Setup\ModuleDependency\Module;
+use Combodo\iTop\Setup\ModuleDependency\ModuleDependencySort;
 use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReader;
 use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReaderException;
 
 require_once(APPROOT.'setup/modulediscovery/ModuleFileReader.php');
 require_once(__DIR__.'/moduledependency/moduledependencysort.class.inc.php');
-
-use Combodo\iTop\Setup\ModuleDependency\ModuleDependencySort;
+require_once(__DIR__.'/itopextension.class.inc.php');
 
 class MissingDependencyException extends CoreException
 {
@@ -95,6 +95,9 @@ class ModuleDiscovery
 	protected static $m_aModules = [];
 	protected static $m_aModuleVersionByName = [];
 
+	/** @var array<\iTopExtension> $m_aRemovedExtensions */
+	protected static $m_aRemovedExtensions = [];
+
 	// All the entries below are list of file paths relative to the module directory
 	protected static $m_aFilesList = ['datamodel', 'webservice', 'dictionary', 'data.struct', 'data.sample'];
 
@@ -131,6 +134,10 @@ class ModuleDiscovery
 
 		list($sModuleName, $sModuleVersion) = static::GetModuleName($sId);
 
+		if (self::IsModuleInExtensionList(self::$m_aRemovedExtensions, $sModuleName, $sModuleVersion, $aArgs)) {
+			return;
+		}
+
 		if (array_key_exists($sModuleName, self::$m_aModuleVersionByName)) {
 			if (version_compare($sModuleVersion, self::$m_aModuleVersionByName[$sModuleName]['version'], '>')) {
 				// Newer version, let's upgrade
@@ -214,29 +221,97 @@ class ModuleDiscovery
 	*/
 	public static function OrderModulesByDependencies($aModules, $bAbortOnMissingDependency = false, $aModulesToLoad = null)
 	{
-		if (is_null($aModulesToLoad)) {
+		if (is_null($aModulesToLoad) && count(self::$m_aRemovedExtensions) === 0) {
 			$aFilteredModules = $aModules;
 		} else {
 			$aFilteredModules = [];
-			foreach ($aModules as $sModuleId => $aModule) {
+			foreach ($aModules as $sModuleId => $aModuleInfo) {
 				$oModule = new Module($sModuleId);
 				$sModuleName = $oModule->GetModuleName();
-				if (in_array($sModuleName, $aModulesToLoad)) {
-					$aFilteredModules[$sModuleId] = $aModule;
+
+				if (self::IsModuleInExtensionList(self::$m_aRemovedExtensions, $sModuleName, $oModule->GetVersion(), $aModuleInfo)) {
+					continue;
+				}
+
+				if (is_null($aModulesToLoad) || in_array($sModuleName, $aModulesToLoad)) {
+					$aFilteredModules[$sModuleId] = $aModuleInfo;
 				}
 			}
 		}
-
 		return ModuleDependencySort::GetInstance()->GetModulesOrderedForInstallation($aFilteredModules, $bAbortOnMissingDependency);
 	}
 
+	/**
+	* @param array<\iTopExtension> $aRemovedExtension
+	* @return void
+	 */
+	public static function DeclareRemovedExtensions(array $aRemovedExtension)
+	{
+		if (self::$m_aRemovedExtensions != $aRemovedExtension) {
+			self::ResetCache();
+		}
+		self::$m_aRemovedExtensions = $aRemovedExtension;
+	}
+
+	/**
+	 * @param array<\iTopExtension> $aExtensions
+	 * @param string $sModuleName
+	 * @param string $sModuleVersion
+	 * @param array $aModuleInfo
+	 *
+	 * @return bool
+	 */
+	private static function IsModuleInExtensionList(array $aExtensions, string $sModuleName, string $sModuleVersion, array $aModuleInfo): bool
+	{
+		if (count($aExtensions) === 0) {
+			return false;
+		}
+		$aNonMatchingPaths = [];
+		$sModuleFilePath = $aModuleInfo[ModuleFileReader::MODULE_FILE_PATH];
+
+		/** @var \iTopExtension $oExtension */
+		foreach ($aExtensions as $oExtension) {
+			$sCurrentVersion = $oExtension->aModuleVersion[$sModuleName] ?? null;
+			if (is_null($sCurrentVersion)) {
+				continue;
+			}
+
+			if ($sModuleVersion !== $sCurrentVersion) {
+				continue;
+			}
+
+			$aCurrentModuleInfo = $oExtension->aModuleInfo[$sModuleName] ?? null;
+			if (is_null($aCurrentModuleInfo)) {
+				SetupLog::Warning("Missing $sModuleName in ".$oExtension->sLabel.". it should not happen");
+				continue;
+			}
+
+			// use case: same module coming from 2 different extensions
+			// we remove only the one coming from removed extensions
+			$sCurrentModuleFilePath = $aCurrentModuleInfo[ModuleFileReader::MODULE_FILE_PATH];
+			if (realpath($sModuleFilePath) !== realpath($sCurrentModuleFilePath)) {
+				$aNonMatchingPaths[] = $sCurrentModuleFilePath;
+				continue;
+			}
+
+			SetupLog::Info("Module considered as removed", null, ['extension_code' => $oExtension->sCode, 'module_name' => $sModuleName, 'module_version' => $sModuleVersion, ModuleFileReader::MODULE_FILE_PATH => $sCurrentModuleFilePath]);
+			return true;
+		}
+
+		if (count($aNonMatchingPaths) > 0) {
+			//add log for support
+			SetupLog::Info("Module kept as it came from non removed extensions", null, ['module_name' => $sModuleName, 'module_version' => $sModuleVersion, ModuleFileReader::MODULE_FILE_PATH => $sModuleFilePath, 'non_matching_paths' => $aNonMatchingPaths]);
+		}
+		return false;
+	}
+
 	private static function GetPhpExpressionEvaluator(): PhpExpressionEvaluator
 	{
-		if (!isset(static::$oPhpExpressionEvaluator)) {
-			static::$oPhpExpressionEvaluator = new PhpExpressionEvaluator([], RunTimeEnvironment::STATIC_CALL_AUTOSELECT_WHITELIST);
+		if (!isset(self::$oPhpExpressionEvaluator)) {
+			self::$oPhpExpressionEvaluator = new PhpExpressionEvaluator([], RunTimeEnvironment::STATIC_CALL_AUTOSELECT_WHITELIST);
 		}
 
-		return static::$oPhpExpressionEvaluator;
+		return self::$oPhpExpressionEvaluator;
 	}
 
 	/**
@@ -285,10 +360,12 @@ class ModuleDiscovery
 
 	/**
 	 * Helper function to interpret the name of a module
+	 *
 	 * @param $sModuleId string Identifier of the module, in the form 'name/version'
-	 * @return array(name, version)
+	 *
+	 * @return array of 2 elements (name, version)
 	 */
-	public static function GetModuleName($sModuleId)
+	public static function GetModuleName($sModuleId): array
 	{
 		$aMatches = [];
 		if (preg_match('!^(.*)/(.*)$!', $sModuleId, $aMatches)) {

setup/modulediscovery/ModuleFileReader.php

@@ -7,15 +7,15 @@ use CoreException;
 use Exception;
 use ParseError;
 use PhpParser\Error;
+use PhpParser\Node\Arg;
+use PhpParser\Node\Expr\Array_;
+use PhpParser\Node\Expr\Assign;
 use PhpParser\Node\Expr\StaticCall;
+use PhpParser\Node\Scalar\String_;
+use PhpParser\Node\Stmt\ElseIf_;
 use PhpParser\Node\Stmt\Expression;
 use PhpParser\Node\Stmt\If_;
 use PhpParser\ParserFactory;
-use PhpParser\Node\Expr\Assign;
-use PhpParser\Node\Stmt\ElseIf_;
-use PhpParser\Node\Expr\Array_;
-use PhpParser\Node\Scalar\String_;
-use PhpParser\Node\Arg;
 
 require_once __DIR__.'/ModuleFileReaderException.php';
 require_once APPROOT.'sources/PhpParser/Evaluation/PhpExpressionEvaluator.php';
@@ -36,6 +36,7 @@ class ModuleFileReader
 	public const MODULE_INFO_PATH = 0;
 	public const MODULE_INFO_ID = 1;
 	public const MODULE_INFO_CONFIG = 2;
+	public const MODULE_FILE_PATH = "module_file_path";
 
 	public const STATIC_CALLWHITELIST = [
 		"utils::GetItopVersionWikiSyntax",
@@ -48,21 +49,23 @@ class ModuleFileReader
 
 	final public static function GetInstance(): ModuleFileReader
 	{
-		if (!isset(static::$oInstance)) {
-			static::$oInstance = new static();
+		if (!isset(self::$oInstance)) {
+			self::$oInstance = new ModuleFileReader();
 		}
 
-		return static::$oInstance;
+		return self::$oInstance;
 	}
 
 	final public static function SetInstance(?ModuleFileReader $oInstance): void
 	{
-		static::$oInstance = $oInstance;
+		self::$oInstance = $oInstance;
 	}
 
 	/**
 	 * Read the information from a module file (module.xxx.php)
-	 * @param string $sModuleFile
+	 *
+	 * @param string $sModuleFilePath
+	 *
 	 * @return array
 	 * @throws ModuleFileReaderException
 	 */
@@ -108,7 +111,9 @@ class ModuleFileReader
 	 * Read the information from a module file (module.xxx.php)
 	 * Warning: this method is using eval() function to load the ModuleInstallerAPI classes.
 	 * Current method is never called at design/runtime. It is acceptable to use it during setup only.
-	 * @param string $sModuleFile
+	 *
+	 * @param string $sModuleFilePath
+	 *
 	 * @return array
 	 * @throws ModuleFileReaderException
 	 */
@@ -164,7 +169,7 @@ class ModuleFileReader
 	private function CompleteModuleInfoWithFilePath(array &$aModuleInfo)
 	{
 		if (count($aModuleInfo) == 3) {
-			$aModuleInfo[static::MODULE_INFO_CONFIG]['module_file_path'] = $aModuleInfo[static::MODULE_INFO_PATH];
+			$aModuleInfo[static::MODULE_INFO_CONFIG][self::MODULE_FILE_PATH] = $aModuleInfo[static::MODULE_INFO_PATH];
 		}
 	}
 
@@ -175,15 +180,21 @@ class ModuleFileReader
 		}
 
 		$sModuleInstallerClass = $aModuleInfo['installer'];
+		if (strlen($sModuleInstallerClass) === 0) {
+			return null;
+		}
+
 		if (!class_exists($sModuleInstallerClass)) {
-			$sModuleFilePath = $aModuleInfo['module_file_path'];
+			$sModuleFilePath = $aModuleInfo[self::MODULE_FILE_PATH];
 			$this->ReadModuleFileInformationUnsafe($sModuleFilePath);
 		}
 
 		if (!class_exists($sModuleInstallerClass)) {
+			\IssueLog::Error(__METHOD__, null, $aModuleInfo);
 			throw new CoreException("Wrong installer class: '$sModuleInstallerClass' is not a PHP class - Module: ".$aModuleInfo['label']);
 		}
 		if (!is_subclass_of($sModuleInstallerClass, 'ModuleInstallerAPI')) {
+			\IssueLog::Error(__METHOD__, null, $aModuleInfo);
 			throw new CoreException("Wrong installer class: '$sModuleInstallerClass' is not derived from 'ModuleInstallerAPI' - Module: ".$aModuleInfo['label']);
 		}
 
@@ -192,7 +203,7 @@ class ModuleFileReader
 
 	/**
 	 * @param string $sModuleFilePath
-	 * @param \PhpParser\Node\Expr\Assign $oAssignation
+	 * @param \PhpParser\Node\Stmt\Expression $oExpression
 	 *
 	 * @return array|null
 	 * @throws ModuleFileReaderException

setup/moduleinstaller.class.inc.php

@@ -130,6 +130,7 @@ abstract class ModuleInstallerAPI
 					if (in_array($sTo, $aNewValues)) {
 						$sEnumCol = $oAttDef->Get("sql");
 						$aFields = CMDBSource::QueryToArray("SHOW COLUMNS FROM `$sTableName` WHERE Field = '$sEnumCol'");
+						$aCurrentValues = [];
 						if (isset($aFields[0]['Type'])) {
 							$sColType = $aFields[0]['Type'];
 							// Note: the parsing should rely on str_getcsv (requires PHP 5.3) to cope with escaped string

setup/parameters.class.inc.php

@@ -7,6 +7,7 @@ class InvalidParameterException extends Exception
 abstract class Parameters
 {
 	public $aData = null;
+	private ?array $aParamValues = null;
 
 	public function __construct()
 	{
@@ -26,24 +27,26 @@ abstract class Parameters
 	 */
 	public function GetParamForConfigArray()
 	{
-		$aDBParams = $this->Get('database');
-		$aParamValues = [
-			'mode' => $this->Get('mode'),
-			'db_server' => $aDBParams['server'],
-			'db_user' => $aDBParams['user'],
-			'db_pwd' => $aDBParams['pwd'],
-			'db_name' => $aDBParams['name'],
-			'new_db_name' => $aDBParams['name'],
-			'db_prefix' => $aDBParams['prefix'],
-			'db_tls_enabled' => $aDBParams['db_tls_enabled'],
-			'db_tls_ca' => $aDBParams['db_tls_ca'],
-			'application_path' => $this->Get('url', ''),
-			'language' => $this->Get('language', ''),
-			'graphviz_path' => $this->Get('graphviz_path', ''),
-			'source_dir' => $this->Get('source_dir', ''),
-		];
+		if (is_null($this->aParamValues)) {
+			$aDBParams = $this->Get('database');
+			$this->aParamValues = [
+				'mode' => $this->Get('mode'),
+				'db_server' => $aDBParams['server'],
+				'db_user' => $aDBParams['user'],
+				'db_pwd' => $aDBParams['pwd'],
+				'db_name' => $aDBParams['name'],
+				'new_db_name' => $aDBParams['name'],
+				'db_prefix' => $aDBParams['prefix'],
+				'db_tls_enabled' => $aDBParams['db_tls_enabled'],
+				'db_tls_ca' => $aDBParams['db_tls_ca'],
+				'application_path' => $this->Get('url', ''),
+				'language' => $this->Get('language', ''),
+				'graphviz_path' => $this->Get('graphviz_path', ''),
+				'source_dir' => $this->Get('source_dir', ''),
+			];
+		}
 
-		return $aParamValues;
+		return $this->aParamValues;
 	}
 
 	public function Set($sCode, $value)
@@ -104,7 +107,9 @@ class PHPParameters extends Parameters
 	{
 		if ($this->aData == null) {
 			require_once($sParametersFile);
-			$this->aData = $ITOP_PARAMS; // Defined in the file loaded just above
+			if (isset($ITOP_PARAMS)) {
+				$this->aData = $ITOP_PARAMS; // Defined in the file loaded just above
+			}
 		}
 	}
 }

setup/runtimeenv.class.inc.php

@@ -126,9 +126,8 @@ class RunTimeEnvironment
 	 * from the given file
 	 * @param $oConfig object The configuration (volatile, not necessarily already on disk)
 	 * @param $bModelOnly boolean Whether or not to allow loading a data model with no corresponding DB
-	 * @return none
 	 */
-	public function InitDataModel($oConfig, $bModelOnly = true, $bUseCache = false)
+	public function InitDataModel($oConfig, $bModelOnly = true, $bUseCache = false): void
 	{
 		require_once APPROOT.'/setup/moduleinstallation.class.inc.php';
 
@@ -348,6 +347,7 @@ class RunTimeEnvironment
 		//
 		$oFactory = new ModelFactory($sSourceDirFull);
 		$aModulesToCompile = $this->GetMFModulesToCompile($sSourceEnv, $sSourceDir);
+		$oModule = null;
 		foreach ($aModulesToCompile as $oModule) {
 			if ($oModule instanceof MFDeltaModule) {
 				// Just before loading the delta, let's save an image of the datamodel
@@ -357,7 +357,7 @@ class RunTimeEnvironment
 			$oFactory->LoadModule($oModule);
 		}
 
-		if ($oModule instanceof MFDeltaModule) {
+		if (!is_null($oModule) && ($oModule instanceof MFDeltaModule)) {
 			// A delta was loaded, let's save a second copy of the datamodel
 			$oFactory->SaveToFile(utils::GetDataPath().'datamodel-'.$this->sTargetEnv.'-with-delta.xml');
 		} else {
@@ -549,9 +549,14 @@ class RunTimeEnvironment
 		//
 		$aAvailableModules = $this->AnalyzeInstallation($oConfig, $this->GetBuildDir());
 		foreach ($aSelectedModuleCodes as $sModuleId) {
+			if (($sModuleId == ROOT_MODULE) || ($sModuleId == DATAMODEL_MODULE)) {
+				continue;
+			}
+
 			if (!array_key_exists($sModuleId, $aAvailableModules)) {
 				continue;
 			}
+
 			$aModuleData = $aAvailableModules[$sModuleId];
 			$sName = $sModuleId;
 			$sVersion = $aModuleData['available_version'];
@@ -625,7 +630,7 @@ class RunTimeEnvironment
 	public function GetApplicationVersion(Config $oConfig)
 	{
 		try {
-			$aSelectInstall = ModuleInstallationService::GetInstance()->ReadFromDB($oConfig);
+			$aSelectInstall = ModuleInstallationRepository::GetInstance()->ReadFromDB($oConfig);
 		} catch (MySQLException $e) {
 			// No database or erroneous information
 			$this->log_error('Can not connect to the database: host: '.$oConfig->Get('db_host').', user:'.$oConfig->Get('db_user').', pwd:'.$oConfig->Get('db_pwd').', db name:'.$oConfig->Get('db_name'));
@@ -663,7 +668,8 @@ class RunTimeEnvironment
 			$aResult['datamodel_version'] = $aResult['product_version'];
 		}
 		$this->log_info("GetApplicationVersion returns: product_name: ".$aResult['product_name'].', product_version: '.$aResult['product_version']);
-		return empty($aResult) ? false : $aResult;
+
+		return count($aResult) == 0 ? false : $aResult;
 	}
 
 	public static function MakeDirSafe($sDir)
@@ -851,16 +857,20 @@ class RunTimeEnvironment
 	/**
 	 * Call the given handler method for all selected modules having an installation handler
 	 * @param array[] $aAvailableModules
-	 * @param string[] $aSelectedModules
 	 * @param string $sHandlerName
+	 *  @param string[]|null $aSelectedModules
 	 * @throws CoreException
 	 */
-	public function CallInstallerHandlers($aAvailableModules, $aSelectedModules, $sHandlerName)
+	public function CallInstallerHandlers($aAvailableModules, $sHandlerName, $aSelectedModules = null)
 	{
 		foreach ($aAvailableModules as $sModuleId => $aModule) {
-			if (($sModuleId != ROOT_MODULE) && in_array($sModuleId, $aSelectedModules)) {
+			if (($sModuleId == ROOT_MODULE) || ($sModuleId == DATAMODEL_MODULE)) {
+				continue;
+			}
+
+			if (is_null($aSelectedModules) || in_array($sModuleId, $aSelectedModules)) {
 				$aArgs = [MetaModel::GetConfig(), $aModule['installed_version'], $aModule['available_version']];
-				RunTimeEnvironment::CallInstallerHandler($aAvailableModules[$sModuleId], $sHandlerName, $aArgs);
+				RunTimeEnvironment::CallInstallerHandler($aModule, $sHandlerName, $aArgs);
 			}
 		}
 	}
@@ -887,6 +897,7 @@ class RunTimeEnvironment
 			try {
 				call_user_func_array($aCallSpec, $aArgs);
 			} catch (Exception $e) {
+				$sModuleId = $aModuleInfo[ModuleFileReader::MODULE_INFO_ID] ?? "";
 				$sErrorMessage = "Module $sModuleId : error when calling module installer class $sModuleInstallerClass for $sHandlerName handler";
 				$aExceptionContextData = [
 					'ModulelId' => $sModuleId,
@@ -903,10 +914,10 @@ class RunTimeEnvironment
 	/**
 	 * Load data from XML files for the selected modules (structural data and/or sample data)
 	 * @param array[] $aAvailableModules All available modules and their definition
-	 * @param string[] $aSelectedModules List of selected modules
 	 * @param bool $bSampleData Wether or not to load sample data
+	 *  @param null|string[] $aSelectedModules List of selected modules
 	 */
-	public function LoadData($aAvailableModules, $aSelectedModules, $bSampleData)
+	public function LoadData($aAvailableModules, $bSampleData, $aSelectedModules = null)
 	{
 		$oDataLoader = new XMLDataLoader();
 
@@ -919,30 +930,33 @@ class RunTimeEnvironment
 		$aFiles = [];
 		$aPreviouslyLoadedFiles = [];
 		foreach ($aAvailableModules as $sModuleId => $aModule) {
-			if (($sModuleId != ROOT_MODULE)) {
-				$sRelativePath = 'env-'.$this->sTargetEnv.'/'.basename($aModule['root_dir']);
-				// Load data only for selected AND newly installed modules
-				if (in_array($sModuleId, $aSelectedModules)) {
-					if ($aModule['installed_version'] != '') {
-						// Simulate the load of the previously loaded XML files to get the mapping of the keys
-						if ($bSampleData) {
-							$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
-							$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.sample']);
-						} else {
-							// Load only structural data
-							$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
-						}
+			if (($sModuleId == ROOT_MODULE) || ($sModuleId == DATAMODEL_MODULE)) {
+				continue;
+			}
+
+			$sRelativePath = 'env-'.$this->sTargetEnv.'/'.basename($aModule['root_dir']);
+			// Load data only for selected AND newly installed modules
+			if (is_null($aSelectedModules) || in_array($sModuleId, $aSelectedModules)) {
+				if ($aModule['installed_version'] != '') {
+					// Simulate the load of the previously loaded XML files to get the mapping of the keys
+					if ($bSampleData) {
+						$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
+						$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.sample']);
 					} else {
-						if ($bSampleData) {
-							$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
-							$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.sample']);
-						} else {
-							// Load only structural data
-							$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
-						}
+						// Load only structural data
+						$aPreviouslyLoadedFiles = static::MergeWithRelativeDir($aPreviouslyLoadedFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
+					}
+				} else {
+					if ($bSampleData) {
+						$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
+						$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.sample']);
+					} else {
+						// Load only structural data
+						$aFiles = static::MergeWithRelativeDir($aFiles, $sRelativePath, $aAvailableModules[$sModuleId]['data.struct']);
 					}
 				}
 			}
+
 		}
 
 		// Simulate the load of the previously loaded files, in order to initialize
@@ -951,7 +965,7 @@ class RunTimeEnvironment
 		foreach ($aPreviouslyLoadedFiles as $sFileRelativePath) {
 			$sFileName = APPROOT.$sFileRelativePath;
 			SetupLog::Info("Loading file: $sFileName (just to get the keys mapping)");
-			if (empty($sFileName) || !file_exists($sFileName)) {
+			if (!file_exists($sFileName)) {
 				throw(new Exception("File $sFileName does not exist"));
 			}
 
@@ -963,7 +977,7 @@ class RunTimeEnvironment
 		foreach ($aFiles as $sFileRelativePath) {
 			$sFileName = APPROOT.$sFileRelativePath;
 			SetupLog::Info("Loading file: $sFileName");
-			if (empty($sFileName) || !file_exists($sFileName)) {
+			if (!file_exists($sFileName)) {
 				throw(new Exception("File $sFileName does not exist"));
 			}
 

setup/setuputils.class.inc.php

@@ -254,32 +254,23 @@ class SetupUtils
 
 		if (!utils::IsModeCLI()) {
 			$sUploadTmpDir = self::GetUploadTmpDir();
-			if (empty($sUploadTmpDir)) {
-				$sUploadTmpDir = '/tmp';
-				$aResult[] = new CheckResult(
-					CheckResult::WARNING,
-					"Temporary directory for files upload is not defined (upload_tmp_dir), assuming that $sUploadTmpDir is used."
-				);
-			}
 			// check that the upload directory is indeed writable from PHP
-			if (!empty($sUploadTmpDir)) {
-				if (!file_exists($sUploadTmpDir)) {
+			if (!file_exists($sUploadTmpDir)) {
+				$aResult[] = new CheckResult(
+					CheckResult::ERROR,
+					"Temporary directory for files upload ($sUploadTmpDir) does not exist or cannot be read by PHP."
+				);
+			} else {
+				if (!is_writable($sUploadTmpDir)) {
 					$aResult[] = new CheckResult(
 						CheckResult::ERROR,
-						"Temporary directory for files upload ($sUploadTmpDir) does not exist or cannot be read by PHP."
+						"Temporary directory for files upload ($sUploadTmpDir) is not writable."
 					);
 				} else {
-					if (!is_writable($sUploadTmpDir)) {
-						$aResult[] = new CheckResult(
-							CheckResult::ERROR,
-							"Temporary directory for files upload ($sUploadTmpDir) is not writable."
-						);
-					} else {
-						$aResult[] = new CheckResult(
-							CheckResult::TRACE,
-							"Info - Temporary directory for files upload ($sUploadTmpDir) is writable."
-						);
-					}
+					$aResult[] = new CheckResult(
+						CheckResult::TRACE,
+						"Info - Temporary directory for files upload ($sUploadTmpDir) is writable."
+					);
 				}
 			}
 		}
@@ -599,7 +590,7 @@ class SetupUtils
 		// create and test destination location
 		//
 		$sDestDir = dirname($sDBBackupPath);
-		setuputils::builddir($sDestDir);
+		SetupUtils::builddir($sDestDir);
 		if (!is_dir($sDestDir)) {
 			$aResult[] = new CheckResult(CheckResult::ERROR, "$sDestDir does not exist and could not be created.");
 		}
@@ -1555,7 +1546,7 @@ JS
 		return $sHtml;
 	}
 
-	public static function GetConfig($oWizard)
+	public static function GetConfig(WizardController $oWizard)
 	{
 		$oConfig = new Config();
 		$sSourceDir = $oWizard->GetParameter('source_dir', '');
@@ -1570,7 +1561,7 @@ JS
 
 		$aParamValues = $oWizard->GetParamForConfigArray();
 		$aParamValues['source_dir'] = $sRelativeSourceDir;
-		$oConfig->UpdateFromParams($aParamValues, null);
+		$oConfig->UpdateFromParams($aParamValues);
 
 		return $oConfig;
 	}
@@ -1602,6 +1593,10 @@ JS
 			$aDirsToScan[] = $sExtraDir;
 		}
 		$oProductionEnv = new RunTimeEnvironment();
+		$aRemovedExtensionCodes = json_decode($oWizard->GetParameter('removed_extensions'), true) ?? [];
+		$oExtensionsMap = new iTopExtensionsMap('production', $aDirsToScan);
+		$oExtensionsMap->DeclareExtensionAsRemoved($aRemovedExtensionCodes);
+
 		$aAvailableModules = $oProductionEnv->AnalyzeInstallation($oConfig, $aDirsToScan, $bAbortOnMissingDependency, $aModulesToLoad);
 
 		foreach ($aAvailableModules as $key => $aModule) {
@@ -1627,7 +1622,7 @@ JS
 
 		$aParamValues = $oWizard->GetParamForConfigArray();
 		$aParamValues['source_dir'] = '';
-		$oConfig->UpdateFromParams($aParamValues, null);
+		$oConfig->UpdateFromParams($aParamValues);
 
 		$oProductionEnv = new RunTimeEnvironment();
 		return $oProductionEnv->GetApplicationVersion($oConfig);

setup/unattended-install/InstallationFileService.php

@@ -71,12 +71,12 @@ class InstallationFileService
 		return $this->aAfterComputationSelectedExtensions;
 	}
 
-	public function SetItopExtensionsMap(ItopExtensionsMap $oItopExtensionsMap): void
+	public function SetItopExtensionsMap(iTopExtensionsMap $oItopExtensionsMap): void
 	{
 		$this->oItopExtensionsMap = $oItopExtensionsMap;
 	}
 
-	public function GetItopExtensionsMap(): ItopExtensionsMap
+	public function GetItopExtensionsMap(): iTopExtensionsMap
 	{
 		if (is_null($this->oItopExtensionsMap)) {
 			$this->oItopExtensionsMap = new iTopExtensionsMap($this->sTargetEnvironment);

setup/wizardcontroller.class.inc.php

@@ -16,7 +16,6 @@
 //
 //   You should have received a copy of the GNU Affero General Public License
 //   along with iTop. If not, see <http://www.gnu.org/licenses/>
-use Combodo\iTop\Application\UI\Base\Component\Html\Html;
 use Combodo\iTop\Application\WebPage\WebPage;
 
 /**
@@ -402,18 +401,6 @@ abstract class WizardStep
 	 * @return void
 	 */
 	abstract public function Display(WebPage $oPage);
-	/**
-	 * Displays the wizard page for the current class/state
-	 * return UIBlock
-	 * The name of the input fields (and their id if one is supplied) MUST NOT start with "_"
-	 * (this is reserved for the wizard's own parameters)
-	 * @return \Combodo\iTop\Application\UI\Base\UIBlock
-	 * @since 3.0.0
-	 */
-	public function DisplayBlock(WebPage $oPage)
-	{
-		return new Html($this->Display($oPage));
-	}
 
 	/**
 	 * Processes the page's parameters and (if moving forward) returns the next step/state to be displayed

setup/wizardsteps.class.inc.php

@@ -50,7 +50,6 @@ require_once(APPROOT.'setup/applicationinstaller.class.inc.php');
 require_once(APPROOT.'setup/parameters.class.inc.php');
 require_once(APPROOT.'core/mutex.class.inc.php');
 require_once(APPROOT.'setup/extensionsmap.class.inc.php');
-require_once APPROOT.'setup/feature_removal/SetupAudit.php';
 
 /**
  * First step of the iTop Installation Wizard: Welcome screen, requirements
@@ -672,9 +671,13 @@ class WizStepLicense extends WizardStep
 	private function NeedsGdprConsent()
 	{
 		$sMode = $this->oWizard->GetParameter('install_mode');
-		$aModules = SetupUtils::AnalyzeInstallation($this->oWizard);
 
-		return (($sMode === 'install') && SetupUtils::IsConnectableToITopHub($aModules));
+		if ($sMode !== 'install') {
+			return false;
+		}
+
+		$aModules = SetupUtils::AnalyzeInstallation($this->oWizard);
+		return SetupUtils::IsConnectableToITopHub($aModules);
 	}
 
 	/**
@@ -1330,6 +1333,9 @@ class WizStepModulesChoice extends WizardStep
 	 */
 	protected bool $bChoicesFromDatabase;
 
+	private array $aAnalyzeInstallationModules;
+	private ?MissingDependencyException $oMissingDependencyException = null;
+
 	public function __construct(WizardController $oWizard, $sCurrentState)
 	{
 		parent::__construct($oWizard, $sCurrentState);
@@ -1354,6 +1360,14 @@ class WizStepModulesChoice extends WizardStep
 			$this->oExtensionsMap->LoadChoicesFromDatabase($this->oConfig);
 			$this->bChoicesFromDatabase = true;
 		}
+
+		// Sanity check (not stopper, to let developers go further...)
+		try {
+			$this->aAnalyzeInstallationModules = SetupUtils::AnalyzeInstallation($this->oWizard, true);
+		} catch (MissingDependencyException $e) {
+			$this->oMissingDependencyException = $e;
+			$this->aAnalyzeInstallationModules = SetupUtils::AnalyzeInstallation($this->oWizard);
+		}
 	}
 
 	public function GetTitle()
@@ -1424,7 +1438,7 @@ class WizStepModulesChoice extends WizardStep
 				$this->oWizard->SetParameter('selected_extensions', json_encode($aExtensions));
 				$this->oWizard->SetParameter('display_choices', $sDisplayChoices);
 				$this->oWizard->SetParameter('extensions_added', json_encode($aExtensionsAdded));
-				$this->oWizard->SetParameter('extensions_removed', json_encode($aExtensionsRemoved));
+				$this->oWizard->SetParameter('removed_extensions', json_encode($aExtensionsRemoved));
 				$this->oWizard->SetParameter('extensions_not_uninstallable', json_encode(array_keys($aExtensionsNotUninstallable)));
 				return ['class' => 'WizStepSummary', 'state' => ''];
 			}
@@ -1445,10 +1459,8 @@ class WizStepModulesChoice extends WizardStep
 	protected function DisplayStep($oPage)
 	{
 		// Sanity check (not stopper, to let developers go further...)
-		try {
-			SetupUtils::AnalyzeInstallation($this->oWizard, true);
-		} catch (MissingDependencyException $e) {
-			$oPage->warning($e->getHtmlDesc(), $e->getMessage());
+		if (! is_null($this->oMissingDependencyException)) {
+			$oPage->warning($this->oMissingDependencyException->getHtmlDesc(), $this->oMissingDependencyException->getMessage());
 		}
 
 		$this->bUpgrade = ($this->oWizard->GetParameter('install_mode') != 'install');
@@ -1459,9 +1471,8 @@ class WizStepModulesChoice extends WizardStep
 		$oPage->add_style(".choice-disabled { color: #999; }");
 		$oPage->add_style("input.unremovable { accent-color: orangered;}");
 
-		$aModules = SetupUtils::AnalyzeInstallation($this->oWizard);
 		$sManualInstallError = SetupUtils::CheckManualInstallDirEmpty(
-			$aModules,
+			$this->aAnalyzeInstallationModules,
 			$this->oWizard->GetParameter('extensions_dir', 'extensions')
 		);
 		if ($sManualInstallError !== '') {
@@ -1487,7 +1498,7 @@ class WizStepModulesChoice extends WizardStep
 		$oPage->add('</div>');
 
 		// Build the default choices
-		$aDefaults = $this->GetDefaults($aStepInfo, $aModules);
+		$aDefaults = $this->GetDefaults($aStepInfo, $this->aAnalyzeInstallationModules);
 		$index = $this->GetStepIndex();
 
 		// retrieve the saved selection
@@ -1747,7 +1758,7 @@ EOF
 	{
 		if ($sParentId == '') {
 			// Check once (before recursing) that the hidden modules are selected
-			foreach (SetupUtils::AnalyzeInstallation($this->oWizard) as $sModuleId => $aModule) {
+			foreach ($this->aAnalyzeInstallationModules as $sModuleId => $aModule) {
 				if (($sModuleId != ROOT_MODULE) && !isset($aModules[$sModuleId])) {
 					if (($aModule['category'] == 'authentication') || (!$aModule['visible'] && !isset($aModule['auto_select']))) {
 						$aModules[$sModuleId] = true;
@@ -1837,11 +1848,10 @@ EOF
 		if ($sParentId == '') {
 			// Last pass (after all the user's choices are turned into "selected" modules):
 			// Process 'auto_select' modules for modules that are not already selected
-			$aAvailableModules = SetupUtils::AnalyzeInstallation($this->oWizard);
 			do {
 				// Loop while new modules are added...
 				$bModuleAdded = false;
-				foreach ($aAvailableModules as $sModuleId => $aModule) {
+				foreach ($this->aAnalyzeInstallationModules as $sModuleId => $aModule) {
 					if (($sModuleId != ROOT_MODULE) && !array_key_exists($sModuleId, $aModules) && isset($aModule['auto_select'])) {
 						try {
 							SetupInfo::SetSelectedModules($aModules);
@@ -2167,18 +2177,6 @@ class WizStepSummary extends WizardStep
 			$this->bDependencyCheck = true;
 			try {
 				SetupUtils::AnalyzeInstallation($this->oWizard, true, $aSelectedModules);
-
-				/*$sInstallMode = utils::ReadParam('install_mode');
-				\SetupLog::Info(__METHOD__, null, ['$sInstallMode' => $sInstallMode]);
-				//if ($sInstallMode === "upgrade") {
-				$aExtensions = json_decode($this->oWizard->GetParameter('selected_extensions'), true);
-				$oSetupAudit = new SetupAudit([]);
-
-				$oConfig = SetupUtils::GetConfig($this->oWizard);
-				$oSetupAudit->SetSelectedExtensions($oConfig, $aExtensions);
-				//$oSetupAudit->AuditExtensionsCleanupRules(true);
-				//}
-				*/
 			} catch (MissingDependencyException $e) {
 				$this->bDependencyCheck = false;
 				$this->sDependencyIssue = $e->getHtmlDesc();
@@ -2261,7 +2259,7 @@ class WizStepSummary extends WizardStep
 		$oPage->add('</div>');
 		$oPage->add('<div class="closed"><span class="title ibo-setup-summary-title">Extensions to be uninstalled</span>');
 
-		$aExtensionsRemoved = json_decode($this->oWizard->GetParameter('extensions_removed'), true);
+		$aExtensionsRemoved = json_decode($this->oWizard->GetParameter('removed_extensions'), true) ?? [];
 		$aExtensionsNotUninstallable = json_decode($this->oWizard->GetParameter('extensions_not_uninstallable'));
 		$sExtensionsRemoved = '';
 		if (count($aExtensionsRemoved) > 0) {

setup/xmldataloader.class.inc.php

@@ -230,6 +230,7 @@ class XMLDataLoader
 						} else {
 							$iDstObj = (int)($oSubNode);
 							// Attempt to find the object in the list of loaded objects
+							/** @var \Combodo\iTop\Core\AttributeDefinition\AttributeExternalKey $oAttDef */
 							$iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj);
 							if ($iExtKey == 0) {
 								$iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative !
@@ -353,8 +354,10 @@ class XMLDataLoader
 			foreach ($oObjList as $oTargetObj) {
 				$bChanged = false;
 				$sClass = get_class($oTargetObj);
+				$iExtKey = -1;
 				foreach (MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef) {
 					if (($oAttDef->IsExternalKey()) && ($oTargetObj->Get($sAttCode) < 0)) { // Convention unresolved key = negative
+						/** @var \Combodo\iTop\Core\AttributeDefinition\AttributeExternalKey $oAttDef */
 						$sTargetClass = $oAttDef->GetTargetClass();
 						$iTempKey = $oTargetObj->Get($sAttCode);
 

sources/Application/WebPage/JsonPage.php

@@ -25,6 +25,7 @@ class JsonPage extends WebPage
 	 * This can be useful when feeding response to a third party lib that doesn't understand the structured format.
 	 */
 	protected $bOutputDataOnly = false;
+	protected $bOutputHeaders = true;
 
 	/**
 	 * JsonPage constructor.
@@ -82,6 +83,19 @@ class JsonPage extends WebPage
 		return $this;
 	}
 
+	/**
+	 * @see static::$bOutputHeaders
+	 * @param bool $bFlag
+	 *
+	 * @return $this
+	 */
+	public function SetOutputHeaders(bool $bFlag)
+	{
+		$this->bOutputHeaders = $bFlag;
+
+		return $this;
+	}
+
 	/**
 	 * Output the headers
 	 *
@@ -119,7 +133,10 @@ class JsonPage extends WebPage
 	public function output()
 	{
 		$oKpi = new ExecutionKPI();
-		$this->OutputHeaders();
+		if ($this->bOutputHeaders) {
+			$this->OutputHeaders();
+		}
+
 		$sContent = $this->ComputeContent();
 		$oKpi->ComputeAndReport(get_class($this).' output');
 

sources/Service/InterfaceDiscovery/InterfaceDiscovery.php

@@ -178,12 +178,19 @@ class InterfaceDiscovery
 					continue;
 				}
 				$aTmpClassMap = include $sAutoloadFile;
+				if (! is_array($aTmpClassMap)) {
+					//can happen when setup compilation broken in the middle
+					//ex: $sAutoloadFile could be empty and $aTmpClassMap is a int
+					$aAutoloaderErrors[] = $sAutoloadFile;
+					continue;
+				}
+
 				/** @noinspection SlowArrayOperationsInLoopInspection we are getting an associative array so the documented workarounds cannot be used */
 				$aClassMap = array_merge($aClassMap, $aTmpClassMap);
 			}
 			if (count($aAutoloaderErrors) > 0) {
 				IssueLog::Debug(
-					__METHOD__." cannot load some of the autoloader files",
+					__METHOD__." cannot load some of the autoloader files: missing or corrupted",
 					LogChannels::CORE,
 					['autoloader_errors' => $aAutoloaderErrors]
 				);

tests/php-unit-tests/integration-tests/itop-hub-connector/HubControllerTest.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace Combodo\iTop\Test\UnitTest\HubConnector;
+
+use Combodo\iTop\DBTools\Service\DBToolsUtils;
+use Combodo\iTop\HubConnector\Controller\HubController;
+use Combodo\iTop\Test\UnitTest\ItopDataTestCase;
+use DOMFormatException;
+use JsonPage;
+use MFCompiler;
+use SetupLog;
+use SetupUtils;
+use utils;
+
+/**
+* @runClassInSeparateProcess
+ */
+class HubControllerTest extends ItopDataTestCase
+{
+	public const USE_TRANSACTION      = false;
+	public const AUTHENTICATION_TOKEN = '14b5da9d092f84044187421419a0347e7317bc8cd2b486fdda631be06b959269';
+	public const AUTHENTICATION_PASSWORD    = "tagada-Secret,007";
+
+	protected function setUp(): void
+	{
+		$this->SkipIfModuleNotPresent('itop-hub-connector');
+		parent::setUp();
+		$this->RequireOnceItopFile('env-production/itop-hub-connector/src/Controller/HubController.php');
+	}
+
+	public function testLaunchCompile(): void
+	{
+		$this->PrepareCompileAuthent();
+
+		$this->CopyProductionModulesIntoHubExtensionDir();
+
+		HubController::GetInstance()->SetOutputHeaders(false);
+
+		HubController::GetInstance()->LaunchCompile();
+
+		$this->CheckReport('{"code":0,"message":"Ok","fields":[]}');
+	}
+
+	public function testLaunchDeploy(): void
+	{
+		$this->testLaunchCompile();
+		HubController::GetInstance()->LaunchDeploy();
+		$this->CheckReport('{"code":0,"message":"Compilation successful.","fields":[]}');
+		$this->AssertPreviousAndCurrentInstallationAreEquivalent();
+	}
+
+	private function CheckReport($sExpected)
+	{
+		$oJsonPage = HubController::GetInstance()->GetLastJsonPage();
+		$this->assertEquals($sExpected, $this->InvokeNonPublicMethod(JsonPage::class, 'ComputeContent', $oJsonPage, []));
+
+		//keep line below to avoid: Test code or tested code did not (only) close its own output buffers
+		$this->InvokeNonPublicMethod(JsonPage::class, 'RenderContent', $oJsonPage, []);
+	}
+
+	private function PrepareCompileAuthent()
+	{
+		$sUUID = 'hub_'.uniqid();
+		$_REQUEST['authent'] = $sUUID;
+		$sPath = utils::GetDataPath().'hub/compile_authent';
+		file_put_contents($sPath, $sUUID);
+		$this->aFileToClean[] = $sPath;
+	}
+
+	private function CopyProductionModulesIntoHubExtensionDir()
+	{
+		$sProdModules = APPROOT.'/data/production-modules';
+		$sExtensionDir = APPROOT.'/data/downloaded-extensions/';
+		$this->aFileToClean[] = $sExtensionDir;
+
+		SetupUtils::rrmdir($sExtensionDir);
+		@mkdir($sExtensionDir);
+		SetupUtils::copydir($sExtensionDir, $sProdModules);
+	}
+}

tests/php-unit-tests/src/BaseTestCase/ItopDataTestCase.php

@@ -17,6 +17,7 @@ namespace Combodo\iTop\Test\UnitTest;
 use ArchivedObjectException;
 use CMDBObject;
 use CMDBSource;
+use Combodo\iTop\DBTools\Service\DBToolsUtils;
 use Combodo\iTop\Service\Events\EventService;
 use Config;
 use Contact;
@@ -34,6 +35,7 @@ use lnkContactToTicket;
 use lnkFunctionalCIToTicket;
 use MetaModel;
 use MissingQueryArgument;
+use ModuleInstallationRepository;
 use MySQLException;
 use MySQLHasGoneAwayException;
 use Person;
@@ -1556,4 +1558,31 @@ abstract class ItopDataTestCase extends ItopTestCase
 		@chmod($sConfigPath, 0440);
 		@unlink($this->sConfigTmpBackupFile);
 	}
+
+	public function AssertPreviousAndCurrentInstallationAreEquivalent()
+	{
+		$aPreviousInstallations = ModuleInstallationRepository::GetInstance()->GetPreviousModuleInstallationsByOffset(1);
+		$aInstallations = ModuleInstallationRepository::GetInstance()->GetPreviousModuleInstallationsByOffset();
+		$this->assertEquals($this->GetCanonicalComparableModuleInstallationArray($aPreviousInstallations), $this->GetCanonicalComparableModuleInstallationArray($aInstallations));
+	}
+
+	protected function GetCanonicalComparableModuleInstallationArray($aInstallations): array
+	{
+		$aRes = [];
+		$aIgnoredFields = ['id', 'parent_id', 'installed', 'comment'];
+		foreach ($aInstallations as $aData) {
+			$aNewData = [];
+			foreach ($aData as $sKey => $val) {
+				if (in_array($sKey, $aIgnoredFields)) {
+					continue;
+				}
+				$aNewData[$sKey] = $val;
+			}
+			$sName = $aNewData['name'];
+			$aRes[$sName] = $aNewData;
+		}
+
+		asort($aRes);
+		return $aRes;
+	}
 }

tests/php-unit-tests/src/BaseTestCase/ItopTestCase.php

@@ -698,4 +698,17 @@ abstract class ItopTestCase extends KernelTestCase
 
 		return $this->CallUrl($sUrl, $aPostFields, $aCurlOptions, $bXDebugEnabled);
 	}
+
+	/**
+	 * Return a temporary file path. that will be cleaned up by tearDown()
+	 *
+	 * @return string: temporary file path: file prefix include phpunit test method name
+	 */
+	public function GetTemporaryFilePath(): string
+	{
+		$sPrefix = $this->getName(false);
+		$sPath = tempnam(sys_get_temp_dir(), $sPrefix);
+		$this->aFileToClean[] = $sPath;
+		return $sPath;
+	}
 }

tests/php-unit-tests/src/Service/UnitTestRunTimeEnvironment.php

@@ -8,11 +8,13 @@
 namespace Combodo\iTop\Test\UnitTest\Service;
 
 use Combodo\iTop\Test\UnitTest\ItopCustomDatamodelTestCase;
+use DOMFormatException;
 use MFCoreModule;
 use RecursiveDirectoryIterator;
 use RecursiveIteratorIterator;
 use ReflectionClass;
 use RunTimeEnvironment;
+use SetupLog;
 use SetupUtils;
 use utils;
 
@@ -64,7 +66,13 @@ class UnitTestRunTimeEnvironment extends RunTimeEnvironment
 			}
 		}
 
-		parent::CompileFrom($sSourceEnv, $bUseSymLinks);
+		try {
+			parent::CompileFrom($sSourceEnv, $bUseSymLinks);
+		} catch (DOMFormatException $e) {
+			$sFileName = $sSourceEnv.'.delta.xml';
+			SetupLog::Error(__METHOD__, null, [$sFileName => @file_get_contents(APPROOT.'data/'.$sFileName)]);
+			throw $e;
+		}
 	}
 
 	public function IsUpToDate()

tests/php-unit-tests/unitary-tests/modulediscovery/ModuleDiscoveryTest.php

@@ -1,50 +0,0 @@
-<?php
-
-namespace Combodo\iTop\Test\UnitTest\Setup;
-
-use Combodo\iTop\Test\UnitTest\ItopTestCase;
-
-class ModuleDiscoveryTest extends ItopTestCase
-{
-	public function GetModuleNameProvider()
-	{
-		return [
-			'nominal' => [
-				'sModuleId' => 'a/1.2.3',
-				'name' => 'a',
-				'version' => '1.2.3',
-			],
-			'develop' => [
-				'sModuleId' => 'a/1.2.3-dev',
-				'name' => 'a',
-				'version' => '1.2.3-dev',
-			],
-			'missing version => 1.0.0' => [
-				'sModuleId' => 'a/',
-				'name' => 'a',
-				'version' => '1.0.0',
-			],
-			'missing everything except name' => [
-				'sModuleId' => 'a',
-				'name' => 'a',
-				'version' => '1.0.0',
-			],
-		];
-	}
-
-	protected function setUp(): void
-	{
-		parent::setUp();
-
-		$this->RequireOnceItopFile('setup/modulediscovery.class.inc.php');
-	}
-
-	/**
-	 * @dataProvider GetModuleNameProvider
-	 */
-	public function testGetModuleName($sModuleId, $expectedName, $expectedVersion)
-	{
-		$this->assertEquals([$expectedName, $expectedVersion], \ModuleDiscovery::GetModuleName($sModuleId));
-	}
-
-}

tests/php-unit-tests/unitary-tests/setup/AnalyzeInstallationTest.php

@@ -4,7 +4,7 @@ namespace Combodo\iTop\Test\UnitTest\Setup;
 
 use AnalyzeInstallation;
 use Combodo\iTop\Test\UnitTest\ItopTestCase;
-use ModuleInstallationService;
+use ModuleInstallationRepository;
 
 class AnalyzeInstallationTest extends ItopTestCase
 {
@@ -12,7 +12,7 @@ class AnalyzeInstallationTest extends ItopTestCase
 	{
 		parent::setUp();
 		$this->RequireOnceItopFile('setup/AnalyzeInstallation.php');
-		$this->RequireOnceItopFile('setup/ModuleInstallationService.php');
+		$this->RequireOnceItopFile('setup/ModuleInstallationRepository.php');
 		$this->RequireOnceItopFile('setup/modulediscovery.class.inc.php');
 		$this->RequireOnceItopFile('setup/runtimeenv.class.inc.php');
 	}
@@ -151,7 +151,7 @@ class AnalyzeInstallationTest extends ItopTestCase
 
 		$this->SetNonPublicProperty(AnalyzeInstallation::GetInstance(), "aAvailableModules", $aAvailableModules);
 		//$aModules = json_decode(file_get_contents(__DIR__.'/ressources/priv_modules2.json'), true);
-		$this->SetNonPublicProperty(ModuleInstallationService::GetInstance(), "aSelectInstall", $aInstalledModules);
+		$this->SetNonPublicProperty(ModuleInstallationRepository::GetInstance(), "aSelectInstall", $aInstalledModules);
 
 		$oConfig =  $this->createMock(\Config::class);
 

tests/php-unit-tests/unitary-tests/setup/ModuleDiscoveryTest.php

@@ -2,7 +2,9 @@
 
 namespace Combodo\iTop\Test\UnitTest\Setup;
 
+use Combodo\iTop\Setup\ModuleDiscovery\ModuleFileReader;
 use Combodo\iTop\Test\UnitTest\ItopTestCase;
+use iTopExtension;
 use MissingDependencyException;
 use ModuleDiscovery;
 
@@ -15,6 +17,12 @@ class ModuleDiscoveryTest extends ItopTestCase
 		$this->RequireOnceItopFile('setup/modulediscovery.class.inc.php');
 	}
 
+	protected function tearDown(): void
+	{
+		parent::tearDown();
+		ModuleDiscovery::DeclareRemovedExtensions([]);
+	}
+
 	public function testOrderModulesByDependencies_RealExample()
 	{
 		$aModules = json_decode(file_get_contents(__DIR__.'/ressources/reallife_discovered_modules.json'), true);
@@ -77,4 +85,126 @@ TXT;
 
 		ModuleDiscovery::OrderModulesByDependencies($aModules, true, $aChoices);
 	}
+
+	public function testOrderModulesByDependencies_FailWhenChoosenModuleDependsOnRemovedExtensionModule()
+	{
+		$aChoices = ['id1', 'id2'];
+
+		$sModuleFilePath = $this->GetTemporaryFilePath();
+		$sModuleFilePath2 = $this->GetTemporaryFilePath();
+
+		$aModules = [
+			"id1/1" => [
+				'dependencies' => [ 'id2/2'],
+				'label' => 'label1',
+				ModuleFileReader::MODULE_FILE_PATH => $sModuleFilePath,
+			],
+			"id2/2" => [
+				'dependencies' => [],
+				'label' => 'label2',
+				ModuleFileReader::MODULE_FILE_PATH => $sModuleFilePath2,
+			],
+		];
+
+		$oExtension = $this->GivenExtensionWithModule('id2', '2', $sModuleFilePath2);
+		ModuleDiscovery::DeclareRemovedExtensions([$oExtension]);
+
+		$sExpectedMessage = <<<TXT
+The following modules have unmet dependencies:
+label1 (id: id1/1) depends on: ❌ id2/2
+TXT;
+		$this->expectException(MissingDependencyException::class);
+		$this->expectExceptionMessage($sExpectedMessage);
+
+		ModuleDiscovery::OrderModulesByDependencies($aModules, true, $aChoices);
+	}
+
+	public function GetModuleNameProvider()
+	{
+		return [
+			'nominal' => [
+				'sModuleId' => 'a/1.2.3',
+				'name' => 'a',
+				'version' => '1.2.3',
+			],
+			'develop' => [
+				'sModuleId' => 'a/1.2.3-dev',
+				'name' => 'a',
+				'version' => '1.2.3-dev',
+			],
+			'missing version => 1.0.0' => [
+				'sModuleId' => 'a/',
+				'name' => 'a',
+				'version' => '1.0.0',
+			],
+			'missing everything except name' => [
+				'sModuleId' => 'a',
+				'name' => 'a',
+				'version' => '1.0.0',
+			],
+		];
+	}
+
+	/**
+	 * @dataProvider GetModuleNameProvider
+	 */
+	public function testGetModuleName($sModuleId, $expectedName, $expectedVersion)
+	{
+		$this->assertEquals([$expectedName, $expectedVersion], ModuleDiscovery::GetModuleName($sModuleId));
+	}
+
+	public function testIsModuleInExtensionList_NoRemovedExtension()
+	{
+		$this->assertFalse($this->InvokeNonPublicStaticMethod(ModuleDiscovery::class, "IsModuleInExtensionList", [[], 'module_name', '123', []]));
+	}
+
+	public function testIsModuleInExtensionList_ModuleWithAnotherVersionIncludedInRemoveExtension()
+	{
+		$sModuleFilePath = $this->GetTemporaryFilePath();
+		$aExtensionList = [$this->GivenExtensionWithModule('module_name', '456', $sModuleFilePath)];
+		$this->AssertModuleIsPartOfRemovedExtension($aExtensionList, 'module_name', '123', $sModuleFilePath, false);
+	}
+
+	public function testIsModuleInExtensionList_AnotherModuleWithSameVersionIncludedInRemoveExtension()
+	{
+		$sModuleFilePath = $this->GetTemporaryFilePath();
+		$aExtensionList = [$this->GivenExtensionWithModule('module_name', '456', $sModuleFilePath)];
+		$this->AssertModuleIsPartOfRemovedExtension($aExtensionList, 'another_module_name', '456', $sModuleFilePath, false);
+	}
+
+	public function testIsModuleInExtensionList_SameExtensionComingFromAnotherLocation()
+	{
+		$sModuleFilePath = $this->GetTemporaryFilePath();
+		$sModuleFilePath2 = $this->GetTemporaryFilePath();
+		$aExtensionList = [$this->GivenExtensionWithModule('module_name', '456', $sModuleFilePath2)];
+		$this->AssertModuleIsPartOfRemovedExtension($aExtensionList, 'module_name', '456', $sModuleFilePath, false);
+	}
+
+	public function testIsModuleInExtensionList_ModuleShouldBeExcluded()
+	{
+		$sModuleFilePath = $this->GetTemporaryFilePath();
+		$aExtensionList = [$this->GivenExtensionWithModule('module_name', '456', $sModuleFilePath)];
+		$this->AssertModuleIsPartOfRemovedExtension($aExtensionList, 'module_name', '456', $sModuleFilePath, true);
+	}
+
+	public function AssertModuleIsPartOfRemovedExtension($aExtensionList, $sModuleName, $sModuleVersion, $sModuleFilePath, $bExpected)
+	{
+		$aCurrentModuleInfo = [
+			ModuleFileReader::MODULE_FILE_PATH => $sModuleFilePath,
+		];
+		$this->assertEquals(
+			$bExpected,
+			$this->InvokeNonPublicStaticMethod(ModuleDiscovery::class, "IsModuleInExtensionList", [$aExtensionList, $sModuleName, $sModuleVersion, $aCurrentModuleInfo])
+		);
+	}
+
+	private function GivenExtensionWithModule(string $sModuleName, string $sVersion, bool|string $sModuleFilePath): iTopExtension
+	{
+		$oExt = new iTopExtension();
+		$oExt->aModuleVersion[$sModuleName] = $sVersion;
+		$oExt->aModuleInfo[$sModuleName] = [
+			ModuleFileReader::MODULE_FILE_PATH => $sModuleFilePath,
+		];
+		return $oExt;
+	}
 }

tests/php-unit-tests/unitary-tests/setup/feature_removal/SetupAuditTest.php

@@ -3,10 +3,12 @@
 namespace Combodo\iTop\Test\UnitTest\Setup\FeatureRemoval;
 
 use Combodo\iTop\Setup\FeatureRemoval\DryRemovalRuntimeEnvironment;
+use Combodo\iTop\Setup\FeatureRemoval\InplaceSetupAudit;
+use Combodo\iTop\Setup\FeatureRemoval\ModelReflectionSerializer;
 use Combodo\iTop\Setup\FeatureRemoval\SetupAudit;
 use Combodo\iTop\Test\UnitTest\ItopCustomDatamodelTestCase;
 use Combodo\iTop\Test\UnitTest\Service\UnitTestRunTimeEnvironment;
-use Exception;
+use MetaModel;
 
 class SetupAuditTest extends ItopCustomDatamodelTestCase
 {
@@ -38,6 +40,7 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		parent::setUp();
 
 		$this->RequireOnceItopFile('/setup/feature_removal/SetupAudit.php');
+		$this->RequireOnceItopFile('/setup/feature_removal/InplaceSetupAudit.php');
 		$this->RequireOnceItopFile('/setup/feature_removal/DryRemovalRuntimeEnvironment.php');
 	}
 
@@ -52,7 +55,7 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		$oDryRemovalRuntimeEnvt->Prepare($this->GetTestEnvironment(), ['nominal_ext1', 'finalclass_ext2']);
 		$oDryRemovalRuntimeEnvt->CompileFrom($this->GetTestEnvironment());
 
-		$oSetupAudit = new SetupAudit(\MetaModel::GetEnvironment());
+		$oSetupAudit = new SetupAudit(MetaModel::GetEnvironment(), DryRemovalRuntimeEnvironment::DRY_REMOVAL_AUDIT_ENV);
 
 		$expected = [
 			"Feature1Module1MyClass",
@@ -67,13 +70,25 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		$this->assertEqualsCanonicalizing($expected, $oSetupAudit->GetIssues());
 	}
 
+	public function testGetRemovedClassesFromSetupWizard()
+	{
+		$sEnv = MetaModel::GetEnvironment();
+
+		$aClassesBeforeRemoval = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($sEnv);
+		$aClassesBeforeRemoval[] = "GabuZomeu";
+
+		$oSetupAudit = new InplaceSetupAudit($aClassesBeforeRemoval, $sEnv);
+		$oSetupAudit->ComputeClasses();
+		$this->assertEquals(["GabuZomeu"], $oSetupAudit->GetRemovedClasses());
+	}
+
 	public function testGetIssues()
 	{
 		$sUID = "AuditExtensionsCleanupRules_".uniqid();
 		$oOrg = $this->CreateOrganization($sUID);
 		$this->createObject('FinalClassFeature1Module1MyFinalClassFromLocation', ['org_id' => $oOrg->GetKey(), 'name' => $sUID, 'name2' => uniqid()]);
 
-		$oSetupAudit = new SetupAudit(\MetaModel::GetEnvironment());
+		$oSetupAudit = new SetupAudit(MetaModel::GetEnvironment(), DryRemovalRuntimeEnvironment::DRY_REMOVAL_AUDIT_ENV);
 		$aRemovedClasses = [
 			"Feature1Module1MyClass",
 			"FinalClassFeature1Module1MyClass",
@@ -99,7 +114,7 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		$this->createObject('FinalClassFeature1Module1MyFinalClassFromLocation', ['org_id' => $oOrg->GetKey(), 'name' => $sUID, 'name2' => uniqid()]);
 		$this->createObject('FinalClassFeature2Module1MyFinalClassFromLocation', ['org_id' => $oOrg->GetKey(), 'name' => $sUID, 'name2' => uniqid()]);
 
-		$oSetupAudit = new SetupAudit(\MetaModel::GetEnvironment());
+		$oSetupAudit = new SetupAudit(MetaModel::GetEnvironment(), DryRemovalRuntimeEnvironment::DRY_REMOVAL_AUDIT_ENV);
 		$aRemovedClasses = [
 			"Feature1Module1MyClass",
 			"FinalClassFeature1Module1MyClass",
@@ -111,8 +126,9 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		//avoid setup dry computation
 		$this->SetNonPublicProperty($oSetupAudit, 'aRemovedClasses', $aRemovedClasses);
 
-		$this->expectException(Exception::class);
-		$this->expectExceptionMessage('FinalClassFeature1Module1MyFinalClassFromLocation');
-		$oSetupAudit->GetIssues(true);
+		$expected = [
+			"FinalClassFeature1Module1MyFinalClassFromLocation" => 1,
+		];
+		$this->assertEqualsCanonicalizing($expected, $oSetupAudit->GetIssues(true));
 	}
 }