N°8806 - improve GetMFModulesToCompile to handle extensions and installation choices on all setup that use compileFrom (MTP, CoreUpdate, Hub, DryRemoval)

datamodels/2.x/itop-core-update/src/Service/RunTimeEnvironmentCoreUpdater.php

@@ -11,7 +11,6 @@ require_once(APPROOT."setup/runtimeenv.class.inc.php");
 
 use Config;
 use Exception;
-use ModelFactory;
 use RunTimeEnvironment;
 use SetupUtils;
 
@@ -115,37 +114,4 @@ class RunTimeEnvironmentCoreUpdater extends RunTimeEnvironment
 		}
 		throw new Exception('No configuration file available');
 	}
-
-	protected function GetMFModulesToCompile($sSourceEnv, $sSourceDir)
-	{
-		$aRet =  parent::GetMFModulesToCompile($sSourceEnv, $sSourceDir);
-
-		// Add new mandatory modules from datamodel 2.x only
-		$sSourceDirFull = APPROOT.$sSourceDir;
-		if (!is_dir($sSourceDirFull)) {
-			throw new Exception("The source directory '$sSourceDirFull' does not exist (or could not be read)");
-		}
-		$aDirsToCompile = [$sSourceDirFull];
-
-		$oFactory = new ModelFactory($aDirsToCompile);
-		$aModules = $oFactory->FindModules();
-		$aAvailableModules = [];
-		/** @var \MFModule $oModule */
-		foreach ($aModules as $oModule) {
-			$aAvailableModules[$oModule->GetName()] = $oModule;
-		}
-		// TODO check the auto-selected modules here
-		foreach ($this->GetExtensionMap()->GetAllExtensions() as $oExtension) {
-			if ($oExtension->bMarkedAsChosen) {
-				foreach ($oExtension->aModules as $sModuleName) {
-					if (!isset($aRet[$sModuleName]) && isset($aAvailableModules[$sModuleName])) {
-						$aRet[$sModuleName] = $aAvailableModules[$sModuleName];
-					}
-				}
-			}
-		}
-
-		return $aRet;
-	}
-
 }

setup/extensionsmap.class.inc.php

@@ -35,18 +35,32 @@ class iTopExtensionsMap
 	/** @var bool $bHasXmlInstallationFile : false when legacy 1.x package with no installation.xml */
 	protected $bHasXmlInstallationFile = true;
 
+	//extension dirs apart from package
+	protected array $aExtraDirs = [];
+
 	/**
 	 * The list of all discovered extensions
+	 *
 	 * @param string $sFromEnvironment The environment to scan
-	 * @param bool $bNormailizeOldExtension true to "magically" convert some well-known old extensions (i.e. a set of modules) to the new iTopExtension format
+	 * @param array $aExtraDirs extensions dir to scan
+	 *
 	 * @return void
 	 */
-	public function __construct($sFromEnvironment = ITOP_DEFAULT_ENV, $aExtraDirs = [])
+	public function __construct(string $sFromEnvironment = ITOP_DEFAULT_ENV, array $aExtraDirs = [])
 	{
 		$this->aExtensions = [];
 		$this->aExtensionsByCode = [];
 		$this->aScannedDirs = [];
 		$this->ScanDisk($sFromEnvironment);
+
+		$this->aExtraDirs = $aExtraDirs;
+		if (is_dir(APPROOT.'extensions')) {
+			$this->aExtraDirs [] = APPROOT.'extensions';
+		}
+		if (is_dir(APPROOT.'data/'.$sFromEnvironment.'-modules')) {
+			$this->aExtraDirs [] = APPROOT.'data/'.$sFromEnvironment.'-modules';
+		}
+
 		foreach ($aExtraDirs as $sDir) {
 			$this->ReadDir($sDir, iTopExtension::SOURCE_REMOTE);
 		}
@@ -64,13 +78,13 @@ class iTopExtensionsMap
 		if (!$this->ReadInstallationWizard(APPROOT.'/datamodels/2.x')) {
 			$this->bHasXmlInstallationFile = false;
 			//no installation xml found in 2.x: let's read all extensions in 2.x first
-			if (!$this->ReadDir(APPROOT.'/datamodels/2.x', iTopExtension::SOURCE_WIZARD)) {
+			if (!$this->ReadDir(APPROOT.'datamodels/2.x', iTopExtension::SOURCE_WIZARD)) {
 				//nothing found in 2.x : fallback read in 1.x (flat structure)
-				$this->ReadDir(APPROOT.'/datamodels/1.x', iTopExtension::SOURCE_WIZARD);
+				$this->ReadDir(APPROOT.'datamodels/1.x', iTopExtension::SOURCE_WIZARD);
 			}
 		}
-		$this->ReadDir(APPROOT.'/extensions', iTopExtension::SOURCE_MANUAL);
-		$this->ReadDir(APPROOT.'/data/'.$sEnvironment.'-modules', iTopExtension::SOURCE_REMOTE);
+		$this->ReadDir(APPROOT.'extensions', iTopExtension::SOURCE_MANUAL);
+		$this->ReadDir(APPROOT.'data/'.$sEnvironment.'-modules', iTopExtension::SOURCE_REMOTE);
 	}
 
 	/**
@@ -375,9 +389,10 @@ class iTopExtensionsMap
 	}
 
 	/**
-	* @param bool $bKeepMissingDependencyExtensions
-	* @param bool $bRemoteExtensionsShouldBeMandatory
-	* @return array<\iTopExtension>>
+	 * @param bool $bKeepExtensionsHavingMissingDependencies
+	 * @param bool $bRemoteExtensionsShouldBeMandatory
+	 *
+	 * @return \iTopExtension[]
 	 */
 	public function GetAllExtensionsToDisplayInSetup(bool $bKeepExtensionsHavingMissingDependencies = false, bool $bRemoteExtensionsShouldBeMandatory = true): array
 	{
@@ -589,6 +604,11 @@ class iTopExtensionsMap
 		}
 	}
 
+	public function GetExtraDir(): array
+	{
+		return $this->aExtraDirs;
+	}
+
 	/**
 	 * Tells if the given module name is "chosen" since it is part of a "chosen" extension (in the specified source dir)
 	 * @param string $sModuleNameToFind

setup/feature_removal/DryRemovalRuntimeEnvironment.php

@@ -3,8 +3,6 @@
 namespace Combodo\iTop\Setup\FeatureRemoval;
 
 use iTopExtensionsMap;
-use MetaModel;
-use ModuleDiscovery;
 use RunTimeEnvironment;
 use SetupUtils;
 
@@ -37,18 +35,19 @@ class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 		SetupUtils::copydir(APPROOT."/conf/$sSourceEnv", APPROOT."/conf/$sBuildEnv");
 
 		$this->DeclareExtensionAsRemoved($this->aExtensionsByCode);
+		/*
+				$sSourceDir = MetaModel::GetConfig()->Get('source_dir');
+				$aSearchDirs = $this->GetExtraDirsToCompile($sSourceDir);
 
-		$sSourceDir = MetaModel::GetConfig()->Get('source_dir');
-		$aSearchDirs = $this->GetExtraDirsToCompile($sSourceDir);
+				$aModulesToLoad = $this->GetModulesToLoad($sSourceEnv, $aSearchDirs);
+				\SetupLog::Info(__METHOD__, null, ['module_to_load' => $aModulesToLoad]);
 
-		$aModulesToLoad = $this->GetModulesToLoad($sSourceEnv, $aSearchDirs);
-
-		try {
-			ModuleDiscovery::GetModulesOrderedByDependencies($aSearchDirs, true, $aModulesToLoad);
-		} catch (\MissingDependencyException $e) {
-			\IssueLog::Error("Cannot prepare setup due to dependency issue", null, ['msg' => $e->getMessage(), 'modules_to_load' => $aModulesToLoad]);
-			throw $e;
-		}
+				try {
+					ModuleDiscovery::GetModulesOrderedByDependencies($aSearchDirs, true, $aModulesToLoad);
+				} catch (\MissingDependencyException $e) {
+					\IssueLog::Error("Cannot prepare setup due to dependency issue", null, ['msg' => $e->getMessage(), 'modules_to_load' => $aModulesToLoad]);
+					throw $e;
+				}*/
 	}
 
 	private function DeclareExtensionAsRemoved(array $aExtensionCodes): void

setup/moduleinstallation/InstallationChoicesToModuleConverter.php

@@ -6,6 +6,7 @@
  */
 
 use Combodo\iTop\Setup\ModuleDependency\DependencyExpression;
+use Combodo\iTop\Setup\ModuleDependency\Module;
 
 require_once __DIR__.'/ModuleInstallationException.php';
 require_once(APPROOT.'/setup/moduledependency/module.class.inc.php');
@@ -36,11 +37,12 @@ class InstallationChoicesToModuleConverter
 	 * @param array<string> $aInstallationChoices
 	 * @param array<string> $aSearchDirs
 	 * @param string|null $sInstallationFilePath
+	 * @param array|null $aExtensionDirs : module/extension dirs to load if they are compliant with choices
 	 *
 	 * @return array<string>
 	 * @throws \ModuleInstallationException
 	 */
-	public function GetModules(array $aInstallationChoices, array $aSearchDirs, ?string $sInstallationFilePath = null): array
+	public function GetModules(array $aInstallationChoices, array $aSearchDirs, ?string $sInstallationFilePath = null, ?array $aExtensionDirs = null): array
 	{
 		$aPackageModules = $this->GetAllModules($aSearchDirs);
 
@@ -61,11 +63,28 @@ class InstallationChoicesToModuleConverter
 		foreach (array_keys($aPackageModules) as $sModuleId) {
 			list($sModuleName) = ModuleDiscovery::GetModuleName($sModuleId);
 			if (in_array($sModuleName, $aInstalledModuleNames)) {
-				$aInstalledModules[] = $sModuleId;
+				$aInstalledModules[$sModuleName] = $sModuleId;
 			}
 		}
 
-		return $aInstalledModules;
+		if (!is_null($aExtensionDirs)) {
+			foreach (array_keys($this->GetAllModules($aExtensionDirs)) as $sModuleId) {
+				$oModule = new Module($sModuleId);
+
+				$sPreviousModuleId = $aInstalledModules[$oModule->GetModuleName()] ?? null;
+				if (is_null($sPreviousModuleId)) {
+					$aInstalledModules[$oModule->GetModuleName()] = $sModuleId;
+					continue;
+				}
+
+				$oPreviousModule = new Module($sPreviousModuleId);
+				if (version_compare($oModule->GetVersion(), $oPreviousModule->GetVersion(), '>')) {
+					$aInstalledModules[$oModule->GetModuleName()] = $sModuleId;
+				}
+			}
+		}
+
+		return array_values($aInstalledModules);
 	}
 
 	/**

setup/runtimeenv.class.inc.php

@@ -50,8 +50,6 @@ class RunTimeEnvironment
 		"SetupInfo::ModuleIsSelected",
 	];
 
-	private static bool $bMetamodelStarted = false;
-
 	/**
 	 * The name of the environment that the caller wants to build
 	 * @var string sFinalEnv
@@ -150,10 +148,6 @@ class RunTimeEnvironment
 	 */
 	public function InitDataModel($oConfig, $bModelOnly = true, $bUseCache = false): void
 	{
-		//		if (self::$bMetamodelStarted && $bModelOnly) {
-		//			return;
-		//		}
-
 		$sConfigFile = $oConfig->GetLoadedFile();
 		if (strlen($sConfigFile) > 0) {
 			$this->log_info("MetaModel::Startup from $sConfigFile (ModelOnly = $bModelOnly)");
@@ -173,7 +167,6 @@ class RunTimeEnvironment
 		$_SESSION['itop_env'] = $this->sBuildEnv;
 
 		MetaModel::Startup($oConfig, $bModelOnly, $bUseCache, false, $this->sBuildEnv);
-		self::$bMetamodelStarted = true;
 
 		if ($this->oExtensionsMap === null) {
 			$this->oExtensionsMap = new iTopExtensionsMap($this->sBuildEnv);
@@ -465,13 +458,8 @@ class RunTimeEnvironment
 		$aExtraDirs = $this->GetExtraDirsToScan($aDirsToCompile);
 		$aDirsToCompile = array_merge($aDirsToCompile, $aExtraDirs);
 
-		// Determine the installed modules and extensions
-		//
 		$oSourceConfig = new Config(APPCONF.$sSourceEnv.'/'.ITOP_CONFIG_FILE);
 
-		$aModulesToLoad = $this->GetModulesToLoad($this->sFinalEnv, $aDirsToCompile);
-		$aAvailableModules = $this->AnalyzeInstallation($oSourceConfig, $aDirsToCompile, false, $aModulesToLoad);
-
 		// Actually read the modules available for the build environment,
 		// but get the selection from the source environment and finally
 		// mark as (automatically) chosen all the "remote" modules present in the
@@ -485,6 +473,10 @@ class RunTimeEnvironment
 			}
 		}
 
+		$aModulesToLoad = $this->GetModulesToLoad($this->sFinalEnv, $aDirsToCompile);
+		SetupLog::Info(__METHOD__, null, ['modules_to_load' => $aModulesToLoad]);
+		$aAvailableModules = $this->AnalyzeInstallation($oSourceConfig, $aDirsToCompile, false, $aModulesToLoad);
+
 		// Do load the required modules
 		//
 		$oDictModule = new MFDictModule('dictionaries', 'iTop Dictionaries', APPROOT.'dictionaries');
@@ -1597,10 +1589,24 @@ class RunTimeEnvironment
 		return substr_compare($sHaystack, $sNeedle, 0, strlen($sNeedle)) === 0;
 	}
 
-	protected function GetModulesToLoad(string $sSourceEnv, $aSearchDirs): ?array
+	/**
+	 * @param string $sSourceEnv
+	 * @param array<string> $aSearchDirs : module/extension dirs to load if they are included in choices
+	 *
+	 * @return array| null
+	 * @throws \ConfigException
+	 * @throws \CoreException
+	 * @throws \ModuleInstallationException
+	 */
+	protected function GetModulesToLoad(string $sSourceEnv, array $aSearchDirs): ?array
 	{
+		if (is_null($this->GetExtensionMap())) {
+			return null;
+		}
+
 		$oSourceConfig = new Config(utils::GetConfigFilePath($sSourceEnv));
-		$aChoices = iTopExtensionsMap::GetChoicesFromDatabase($oSourceConfig);
+
+		$aChoices = $this->GetExtensionMap()->GetChoicesFromDatabase($oSourceConfig);
 		if (false === $aChoices) {
 			return null;
 		}
@@ -1611,13 +1617,22 @@ class RunTimeEnvironment
 			$sInstallFilePath = null;
 		}
 
-		$aModuleIdsToLoad = InstallationChoicesToModuleConverter::GetInstance()->GetModules($aChoices, $aSearchDirs, $sInstallFilePath);
+		$aExtensionDirs = [];
+		foreach ($this->GetExtensionMap()->GetAllExtensions() as $oExtension) {
+			if ($oExtension->bMarkedAsChosen && is_dir($oExtension->sSourceDir)) {
+				$aExtensionDirs [] = $oExtension->sSourceDir;
+			}
+		}
+
+		SetupLog::Info(__METHOD__, null, ['ext_dirs' => $aExtensionDirs]);
+		$aModuleIdsToLoad = InstallationChoicesToModuleConverter::GetInstance()->GetModules($aChoices, $aSearchDirs, $sInstallFilePath, $aExtensionDirs);
 		$aModulesToLoad = [];
 		foreach ($aModuleIdsToLoad as $sModuleId) {
 			$oModule = new Module($sModuleId);
 			$sModuleName = $oModule->GetModuleName();
 			$aModulesToLoad[] = $sModuleName;
 		}
+
 		return $aModulesToLoad;
 	}
 }

tests/php-unit-tests/unitary-tests/setup/feature_removal/SetupAuditTest.php

@@ -54,7 +54,7 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		$oDryRemovalRuntimeEnvt = new DryRemovalRuntimeEnvironment($this->GetTestEnvironment(), ['nominal_ext1', 'finalclass_ext2']);
 		$oDryRemovalRuntimeEnvt->CompileFrom($this->GetTestEnvironment());
 
-		$oSetupAudit = new SetupAudit(MetaModel::GetEnvironment());
+		$oSetupAudit = new SetupAudit($this->GetTestEnvironment());
 
 		$expected = [
 			"Feature1Module1MyClass",