cleanup: remove cached stuff in runtimeenv

datamodels/2.x/combodo-data-feature-removal/src/Controller/DataFeatureRemovalController.php

@@ -145,7 +145,7 @@ class DataFeatureRemovalController extends Controller
 
 		$bForceCompilation = Session::Get('bForceCompilation', false);
 		try {
-			$this->Compile($aRemoveExtensionCodes, $bForceCompilation);
+			$this->Compile($aAddedExtensions, $aRemoveExtensionCodes, $bForceCompilation);
 		} catch (CoreException $e) {
 			$aParams['DataFeatureRemovalErrorMessage'] = $e->getHtmlDesc();
 			$this->DisplayPage($aParams, 'AnalysisResult');
@@ -162,7 +162,7 @@ class DataFeatureRemovalController extends Controller
 			$aSelectedExtensions = DataFeatureRemoverExtensionService::GetInstance()->GetExtensionMap()->GetSelectedExtensions($oConfig, array_keys($aAddedExtensions), array_keys($aRemovedExtensions));
 			$aHiddenInputs['selected_extensions'] = $this->ConvertIntoSetupFormat($aSelectedExtensions);
 
-			$oRunTimeEnvironment = $this->GetRuntimeEnvironment($aRemovedExtensions);
+			$oRunTimeEnvironment = $this->GetRuntimeEnvironment($aAddedExtensions, $aRemovedExtensions);
 			$aSearchDirs = [$oRunTimeEnvironment->GetBuildDir()];
 			$aSelectedModules = $oRunTimeEnvironment->GetModulesToLoadFromChoices($oConfig, $aSelectedExtensions, $aSearchDirs);
 			$aHiddenInputs['selected_modules'] = $this->ConvertIntoSetupFormat($aSelectedModules);
@@ -213,13 +213,14 @@ class DataFeatureRemovalController extends Controller
 	}
 
 	/**
+* @param array $aAddedExtensions
 * @param array $aRemovedExtensions
 * @param bool $bForceCompilation
 * @return void
 * @throws \ConfigException
 * @throws \CoreException
 	 */
-	private function Compile(array $aRemovedExtensions, bool $bForceCompilation = true): void
+	private function Compile(array $aAddedExtensions, array $aRemovedExtensions, bool $bForceCompilation = true): void
 	{
 		$sSourceEnv = MetaModel::GetEnvironment();
 		$sBuildDir = APPROOT."/env-$sSourceEnv-build";
@@ -234,15 +235,15 @@ class DataFeatureRemovalController extends Controller
 				null,
 				['sSourceEnv' => $sSourceEnv, 'sBuildDir' => $sBuildDir, 'bIsDirEmpty' => $bIsDirEmpty, glob("$sBuildDir/*")]
 			);
-			$this->GetRuntimeEnvironment($aRemovedExtensions)->CompileFrom($sSourceEnv);
+			$this->GetRuntimeEnvironment($aAddedExtensions, $aRemovedExtensions)->CompileFrom($sSourceEnv);
 		}
 	}
 
-	private function GetRuntimeEnvironment(array $aRemovedExtensions): RunTimeEnvironment
+	private function GetRuntimeEnvironment(array $aAddedExtensions, array $aRemovedExtensions): RunTimeEnvironment
 	{
 		if (is_null($this->oRuntimeEnvironment)) {
 			$sSourceEnv = MetaModel::GetEnvironment();
-			$this->oRuntimeEnvironment = new DryRemovalRuntimeEnvironment($sSourceEnv, $aRemovedExtensions);
+			$this->oRuntimeEnvironment = new DryRemovalRuntimeEnvironment($sSourceEnv, $aAddedExtensions, $aRemovedExtensions);
 		}
 
 		return $this->oRuntimeEnvironment;

setup/extensionsmap.class.inc.php

@@ -759,7 +759,7 @@ class iTopExtensionsMap
 	public function ModuleIsChosenAsPartOfAnExtension($sModuleNameToFind, $sInSourceOnly = iTopExtension::SOURCE_REMOTE)
 	{
 		foreach ($this->GetAllExtensions() as $oExtension) {
-			if (($oExtension->sSource == $sInSourceOnly) &&
+			if (/*($oExtension->sSource == $sInSourceOnly) &&*/
 				($oExtension->bMarkedAsChosen == true) &&
 				(array_key_exists($sModuleNameToFind, $oExtension->aModuleVersion))) {
 				return true;

setup/feature_removal/DryRemovalRuntimeEnvironment.php

@@ -2,6 +2,7 @@
 
 namespace Combodo\iTop\Setup\FeatureRemoval;
 
+use Config;
 use iTopExtensionsMap;
 use RunTimeEnvironment;
 use SetupUtils;
@@ -9,15 +10,17 @@ use SetupUtils;
 class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 {
 	protected array $aExtensionsToRemoveByCode;
+	protected array $aExtensionCodesToAddByCode;
 
 	/**
 	 * Toolset for building a run-time environment
 	 *
 	 *  @param string $sSourceEnv: environment from which setup is inspired to simulate extension removal and usee CompileFrom...
 	 */
-	public function __construct($sSourceEnv = ITOP_DEFAULT_ENV, array $aExtensionCodesToRemove = [])
+	public function __construct($sSourceEnv = ITOP_DEFAULT_ENV, array $aExtensionCodesToAdd = [], array $aExtensionCodesToRemove = [])
 	{
 		parent::__construct($sSourceEnv, false);
+		$this->aExtensionCodesToAddByCode = $aExtensionCodesToAdd;
 		$this->aExtensionsToRemoveByCode = $aExtensionCodesToRemove;
 		$this->Prepare($sSourceEnv, $this->sBuildEnv);
 	}
@@ -34,13 +37,18 @@ class DryRemovalRuntimeEnvironment extends RunTimeEnvironment
 		SetupUtils::copydir(APPROOT."/data/$sSourceEnv-modules", APPROOT."/data/$sBuildEnv-modules");
 		SetupUtils::copydir(APPROOT."/conf/$sSourceEnv", APPROOT."/conf/$sBuildEnv");
 
-		$this->DeclareExtensionAsRemoved($this->aExtensionsToRemoveByCode);
-	}
+		$oSourceConfig = new Config(APPCONF.$sSourceEnv.'/'.ITOP_CONFIG_FILE);
+		$sSourceDir = $oSourceConfig->Get('source_dir');
+		list($aExtraDirs, ) = $this->GetDirsToCompile($sSourceDir, $sSourceEnv);
 
-	private function DeclareExtensionAsRemoved(array $aExtensionCodes): void
-	{
-		$oExtensionsMap = new iTopExtensionsMap($this->sBuildEnv);
-		$oExtensionsMap->DeclareExtensionAsRemoved($aExtensionCodes);
+		$this->InitExtensionMap($aExtraDirs, $oSourceConfig);
+		$this->GetExtensionMap()->DeclareExtensionAsRemoved($this->aExtensionsToRemoveByCode);
+
+		foreach ($this->GetExtensionMap()->GetAllExtensions() as $oExtension) {
+			if (array_key_exists($oExtension->sCode, $this->aExtensionCodesToAddByCode)) {
+				$oExtension->bMarkedAsChosen = true;
+			}
+		}
 	}
 
 	public function Cleanup(): void

setup/feature_removal/SetupAudit.php

@@ -17,6 +17,11 @@ class SetupAudit extends AbstractSetupAudit
 		$this->sEnvAfter = $sEnvAfter ?? "$sEnvBefore-build";
 	}
 
+	public function GetEnvAfter(): string
+	{
+		return $this->sEnvAfter;
+	}
+
 	public function ComputeClasses(): void
 	{
 		if ($this->bClassesInitialized) {

setup/runtimeenv.class.inc.php

@@ -71,15 +71,37 @@ class RunTimeEnvironment
 		return $this->oExtensionsMap;
 	}
 
-	public function InitExtensionMap($aExtraDirs, $oSourceConfig)
+	protected function GetDirsToCompile(string $sSourceDir, string $sSourceEnv): array
 	{
-		// Actually read the modules available for the build environment,
-		// but get the selection from the source environment and finally
-		// mark as (automatically) chosen all the "remote" modules present in the
-		// build environment (data/<build-env>-modules)
-		// The actual choices will be recorded by RecordInstallation below
-		$this->oExtensionsMap = new iTopExtensionsMap($this->sBuildEnv, $aExtraDirs);
-		$this->oExtensionsMap->LoadChoicesFromDatabase($oSourceConfig);
+		$sSourceDirFull = APPROOT.$sSourceDir;
+		if (!is_dir($sSourceDirFull)) {
+			throw new Exception("The source directory '$sSourceDirFull' does not exist (or could not be read)");
+		}
+		$aDirsToCompile = [$sSourceDirFull];
+		if (is_dir(APPROOT.'extensions')) {
+			$aDirsToCompile[] = APPROOT.'extensions';
+		}
+		$sExtraDir = utils::GetDataPath().$this->sBuildEnv.'-modules/';
+		if (is_dir($sExtraDir)) {
+			$aDirsToCompile[] = $sExtraDir;
+		}
+
+		$aExtraDirs = $this->GetExtraDirsToScan($aDirsToCompile);
+		$aDirsToCompile = array_merge($aDirsToCompile, $aExtraDirs);
+		return [$aExtraDirs, $aDirsToCompile];
+	}
+
+	public function InitExtensionMap(array $aExtraDirs, Config $oSourceConfig)
+	{
+		if (is_null($this->oExtensionsMap)) {
+			// Actually read the modules available for the build environment,
+			// but get the selection from the source environment and finally
+			// mark as (automatically) chosen all the "remote" modules present in the
+			// build environment (data/<build-env>-modules)
+			// The actual choices will be recorded by RecordInstallation below
+			$this->oExtensionsMap = new iTopExtensionsMap($this->sBuildEnv, $aExtraDirs);
+			$this->oExtensionsMap->LoadChoicesFromDatabase($oSourceConfig);
+		}
 	}
 
 	/**
@@ -444,32 +466,12 @@ class RunTimeEnvironment
 
 	/**
 	 * Get the installed modules (only the installed ones)
+	 * @return \MFModule[]
 	 */
-	protected function GetMFModulesToCompile($sSourceEnv, $sSourceDir)
+	protected function GetMFModulesToCompile($sSourceEnv, $sSourceDir): array
 	{
-		$sSourceDirFull = APPROOT.$sSourceDir;
-		if (!is_dir($sSourceDirFull)) {
-			throw new Exception("The source directory '$sSourceDirFull' does not exist (or could not be read)");
-		}
-		$aDirsToCompile = [$sSourceDirFull];
-		if (is_dir(APPROOT.'extensions')) {
-			$aDirsToCompile[] = APPROOT.'extensions';
-		}
-		$sExtraDir = utils::GetDataPath().$this->sBuildEnv.'-modules/';
-		if (is_dir($sExtraDir)) {
-			$aDirsToCompile[] = $sExtraDir;
-		}
-
-		$aExtraDirs = $this->GetExtraDirsToScan($aDirsToCompile);
-		$aDirsToCompile = array_merge($aDirsToCompile, $aExtraDirs);
-
+		list($aExtraDirs, $aDirsToCompile) = $this->GetDirsToCompile($sSourceDir, $sSourceEnv);
 		$oSourceConfig = new Config(APPCONF.$sSourceEnv.'/'.ITOP_CONFIG_FILE);
-
-		// Actually read the modules available for the build environment,
-		// but get the selection from the source environment and finally
-		// mark as (automatically) chosen all the "remote" modules present in the
-		// build environment (data/<build-env>-modules)
-		// The actual choices will be recorded by RecordInstallation below
 		$this->InitExtensionMap($aExtraDirs, $oSourceConfig);
 		$this->GetExtensionMap()->LoadChoicesFromDatabase($oSourceConfig);
 		foreach ($this->GetExtensionMap()->GetAllExtensions() as $oExtension) {
@@ -477,12 +479,10 @@ class RunTimeEnvironment
 				$this->GetExtensionMap()->MarkAsChosen($oExtension->sCode);
 			}
 		}
-
 		$aModulesToLoad = $this->GetModulesToLoad($this->sFinalEnv, $aDirsToCompile);
 		$aAvailableModules = $this->AnalyzeInstallation($oSourceConfig, $aDirsToCompile, true, $aModulesToLoad);
 
 		// Do load the required modules
-		//
 		$oDictModule = new MFDictModule('dictionaries', 'iTop Dictionaries', APPROOT.'dictionaries');
 
 		$aRet = [];
@@ -1700,21 +1700,29 @@ class RunTimeEnvironment
 		}
 
 		$aExtensionDirs = [];
+		$aFromSelectedExtensionModules=[];
 		foreach ($this->GetExtensionMap()->GetAllExtensions() as $oExtension) {
 			if ($oExtension->bMarkedAsChosen && is_dir($oExtension->sSourceDir)) {
 				$aExtensionDirs [] = $oExtension->sSourceDir;
+				$aFromSelectedExtensionModules = array_merge($aFromSelectedExtensionModules, $oExtension->aModules);
 			}
 		}
 
 		SetupLog::Info(__METHOD__, null, ['ext_dirs' => $aExtensionDirs]);
 		$aModuleIdsToLoad = InstallationChoicesToModuleConverter::GetInstance()->GetModules($aChoices, $aSearchDirs, $sInstallFilePath, $aExtensionDirs);
 		$aModulesToLoad = [];
+
 		foreach ($aModuleIdsToLoad as $sModuleId) {
 			$oModule = new Module($sModuleId);
 			$sModuleName = $oModule->GetModuleName();
 			$aModulesToLoad[] = $sModuleName;
 		}
 
+		foreach ($aFromSelectedExtensionModules as $sModuleName){
+			if (! in_array($sModuleName, $aModulesToLoad)){
+				$aModulesToLoad[] = $sModuleName;
+			}
+		}
 		return $aModulesToLoad;
 	}
 

tests/php-unit-tests/src/Service/UnitTestRunTimeEnvironment.php

@@ -108,7 +108,7 @@ class UnitTestRunTimeEnvironment extends RunTimeEnvironment
 	/**
 	 * @inheritDoc
 	 */
-	protected function GetMFModulesToCompile($sSourceEnv, $sSourceDir)
+	protected function GetMFModulesToCompile($sSourceEnv, $sSourceDir): array
 	{
 		\SetupLog::Info(__METHOD__);
 		$aRet = parent::GetMFModulesToCompile($sSourceEnv, $sSourceDir);

tests/php-unit-tests/unitary-tests/setup/feature_removal/SetupAuditTest.php

@@ -9,11 +9,14 @@ use Combodo\iTop\Setup\FeatureRemoval\SetupAudit;
 use Combodo\iTop\Test\UnitTest\ItopCustomDatamodelTestCase;
 use Combodo\iTop\Test\UnitTest\Service\UnitTestRunTimeEnvironment;
 use MetaModel;
+use SetupUtils;
 
 class SetupAuditTest extends ItopCustomDatamodelTestCase
 {
 	public const ENVT = 'php-unit-extensionremoval-tests';
 
+	private ?string $sDirPathToRemoveFromExtensionFolder = null;
+
 	public function GetDatamodelDeltaAbsPath(): string
 	{
 		//no delta: empty path provided
@@ -44,6 +47,14 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 		$this->RequireOnceItopFile('/setup/feature_removal/DryRemovalRuntimeEnvironment.php');
 	}
 
+	protected function tearDown(): void
+	{
+		parent::tearDown();
+		if (!is_null($this->sDirPathToRemoveFromExtensionFolder) && is_dir($this->sDirPathToRemoveFromExtensionFolder)) {
+			SetupUtils::rrmdir($this->sDirPathToRemoveFromExtensionFolder);
+		}
+	}
+
 	public function GetTestEnvironment(): string
 	{
 		return self::ENVT;
@@ -51,7 +62,11 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 
 	public function testComputeDryRemoval()
 	{
-		$oDryRemovalRuntimeEnvt = new DryRemovalRuntimeEnvironment($this->GetTestEnvironment(), ['nominal_ext1', 'finalclass_ext2']);
+		$this->sDirPathToRemoveFromExtensionFolder = APPROOT.'/extensions/finalclass_ext3';
+		SetupUtils::copydir(__DIR__.'/other_features/finalclass_ext3', $this->sDirPathToRemoveFromExtensionFolder);
+		clearstatcache();
+
+		$oDryRemovalRuntimeEnvt = new DryRemovalRuntimeEnvironment($this->GetTestEnvironment(), ['finalclass_ext3' => 'Ext For Test3'], ['nominal_ext1', 'finalclass_ext2']);
 		$oDryRemovalRuntimeEnvt->CompileFrom($this->GetTestEnvironment());
 
 		$oSetupAudit = new SetupAudit($this->GetTestEnvironment());
@@ -67,6 +82,19 @@ class SetupAuditTest extends ItopCustomDatamodelTestCase
 			"FinalClassFeature2Module1MyFinalClassFromLocation" => 0,
 		];
 		$this->assertEqualsCanonicalizing($expected, $oSetupAudit->RunDataAudit());
+
+		$aClassesAfter = ModelReflectionSerializer::GetInstance()->GetModelFromEnvironment($oSetupAudit->GetEnvAfter());
+		$expected = [
+			"FinalClassFeature3Module1MyClass",
+			"FinalClassFeature3Module1MyFinalClassFromLocation",
+		];
+		foreach ($expected as $sAddedClass) {
+			$this->assertContains(
+				$sAddedClass,
+				$aClassesAfter,
+				"After DryRemoval compilation DM should contain classes coming from finalclass_ext3 extension"
+			);
+		}
 	}
 
 	public function testGetRemovedClassesFromSetupWizard()

tests/php-unit-tests/unitary-tests/setup/feature_removal/other_features/finalclass_ext3/extension.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<extension format="1.0">
+  <extension_code>finalclass_ext3</extension_code>
+  <company>Combodo SARL</company>
+  <author><![CDATA[Odain]]></author>
+  <label><![CDATA[Ext For Test3]]></label>
+  <description><![CDATA[Ext For Test]]></description>
+  <version>6.6.6</version>
+  <modules type="array">
+    <module>
+      <id>finalclass_ext3_module1</id>
+      <version>tags/6.6.6</version>
+    </module>
+  </modules>
+  <release_date>2023-07-19</release_date>
+  <version_description><![CDATA[
+      ]]></version_description>
+  <itop_version_min>3.2.0</itop_version_min>
+  <status></status>
+  <mandatory>false</mandatory>
+  <more_info_url></more_info_url>
+</extension>

tests/php-unit-tests/unitary-tests/setup/feature_removal/other_features/finalclass_ext3/finalclass_ext3_module1/datamodel.finalclass_ext3_module1.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<itop_design xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0">
+  <classes>
+    <class id="FinalClassFeature3Module1MyFinalClassFromLocation" _delta="define">
+      <properties>
+        <category>bizmodel,searchable</category>
+        <abstract>false</abstract>
+        <db_table>FinalClassFeature3Module1MyFinalClassFromLocation</db_table>
+        <naming>
+          <attributes>
+            <attribute id="name2"/>
+            <attribute id="finalclass"/>
+          </attributes>
+        </naming>
+        <reconciliation>
+          <attributes>
+            <attribute id="name2"/>
+            <attribute id="finalclass"/>
+          </attributes>
+        </reconciliation>
+        <order>
+          <columns>
+            <column id="name2" ascending="false"/>
+          </columns>
+        </order>
+      </properties>
+      <fields>
+        <field id="name2" xsi:type="AttributeString">
+          <sql>name2</sql>
+          <default_value/>
+          <is_null_allowed>false</is_null_allowed>
+        </field>
+      </fields>
+      <methods/>
+      <presentation/>
+      <parent>Location</parent>
+    </class>
+    <class id="FinalClassFeature3Module1MyClass" _delta="define">
+      <properties>
+        <category>bizmodel,searchable</category>
+        <abstract>false</abstract>
+        <db_table>FinalClassFeature3Module1MyClass</db_table>
+        <naming>
+          <attributes>
+            <attribute id="name"/>
+          </attributes>
+        </naming>
+        <reconciliation>
+          <attributes>
+            <attribute id="name"/>
+          </attributes>
+        </reconciliation>
+        <order>
+          <columns>
+            <column id="name" ascending="false"/>
+          </columns>
+        </order>
+      </properties>
+      <fields>
+        <field id="name" xsi:type="AttributeString">
+          <sql>name</sql>
+          <default_value/>
+          <is_null_allowed>false</is_null_allowed>
+          <validation_pattern/>
+        </field>
+      </fields>
+      <methods/>
+      <presentation/>
+      <parent>cmdbAbstractObject</parent>
+    </class>
+  </classes>
+  <menus/>
+  <user_rights/>
+  <module_parameters/>
+</itop_design>

tests/php-unit-tests/unitary-tests/setup/feature_removal/other_features/finalclass_ext3/finalclass_ext3_module1/model.finalclass_ext3_module1.php

@@ -0,0 +1,16 @@
+<?php
+
+//// PHP Data Model definition file
+//
+//// WARNING - WARNING - WARNING
+//// DO NOT EDIT THIS FILE (unless you know what you are doing)
+////
+//// If you use supply a datamodel.xxxx.xml file with your module
+//// the this file WILL BE overwritten by the compilation of the
+//// module (during the setup) if the datamodel.xxxx.xml file
+//// contains the definition of new classes or menus.
+////
+//// The recommended way to define new classes (for iTop 2.0) is via the XML definition.
+//// This file remains in the module's template only for the cases where there is:
+//// - either no new class or menu defined in the XML file
+//// - or no XML file at all supplied by the module

tests/php-unit-tests/unitary-tests/setup/feature_removal/other_features/finalclass_ext3/finalclass_ext3_module1/module.finalclass_ext3_module1.php

@@ -0,0 +1,51 @@
+<?php
+
+/*
+ * @copyright   Copyright (C) 2010-2021 Combodo SARL
+ * @license     http://opensource.org/licenses/AGPL-3.0
+ */
+
+//
+// iTop module definition file
+//
+
+SetupWebPage::AddModule(
+	__FILE__, // Path to the current file, all other file names are relative to the directory containing this file
+	'finalclass_ext3_module1/6.6.6',
+	[
+		// Identification
+		//
+		'label'        => 'Ext For Test',
+		'category'     => 'business',
+
+		// Setup
+		//
+		'dependencies' => [
+			'itop-structure/3.2.0',
+		],
+		'mandatory' => false,
+		'visible' => true,
+		'installer' => '',
+
+		// Components
+		//
+		'datamodel' => [
+			'model.finalclass_ext3_module1.php',
+		],
+		'webservice' => [],
+		'data.struct' => [// add your 'structure' definition XML files here,
+		],
+		'data.sample' => [// add your sample data XML files here,
+		],
+
+		// Documentation
+		//
+		'doc.manual_setup' => '', // hyperlink to manual setup documentation, if any
+		'doc.more_information' => '', // hyperlink to more information, if any
+
+		// Default settings
+		//
+		'settings' => [// Module specific settings go here, if any
+		],
+	]
+);