N°6716 - ✅ High memory Consomption and performance issue

tests/php-unit-tests/perf-tests.xml.dist

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<phpunit
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/8.5/phpunit.xsd"
+        bootstrap="unittestautoload.php"
+        backupGlobals="true"
+        colors="true"
+        columns="120"
+        convertErrorsToExceptions="true"
+        convertNoticesToExceptions="true"
+        convertWarningsToExceptions="true"
+        processIsolation="false"
+        stopOnError="false"
+        stopOnFailure="false"
+        stopOnIncomplete="false"
+        stopOnRisky="false"
+        stopOnSkipped="false"
+        verbose="true"
+        printerClass="\Sempro\PHPUnitPrettyPrinter\PrettyPrinterForPhpUnit9"
+>
+
+  <extensions>
+    <extension class="Combodo\iTop\Test\UnitTest\Hook\TestsRunStartHook" />
+  </extensions>
+
+  <php>
+    <ini name="memory_limit" value="512M"/>
+    <ini name="error_reporting" value="E_ALL"/>
+    <ini name="display_errors" value="On"/>
+    <ini name="log_errors" value="On"/>
+    <ini name="html_errors" value="Off"/>
+    <env name="PHPUNIT_PRETTY_PRINT_PROGRESS" value="true"/>
+  </php>
+
+  <testsuites>
+    <!-- Unitary tests -->
+    <testsuite name="Perf">
+      <directory>perf-tests</directory>
+    </testsuite>
+  </testsuites>
+
+  <!-- Code coverage white list  -->
+  <filter>
+    <whitelist>
+      <file>../../core/apc-emulation.php</file>
+      <file>../../core/ormlinkset.class.inc.php</file>
+      <file>../../datamodels/2.x/itop-tickets/main.itop-tickets.php</file>
+    </whitelist>
+  </filter>
+
+</phpunit>

tests/php-unit-tests/perf-tests/BulkDBObjectTest.php

@@ -0,0 +1,93 @@
+<?php
+
+use Combodo\iTop\Test\UnitTest\ItopDataTestCase;
+
+/**
+ * @copyright   Copyright (C) 2010-2023 Combodo SARL
+ * @license     http://opensource.org/licenses/AGPL-3.0
+ */
+
+class BulkDBObjectTest extends ItopDataTestCase
+{
+	const CREATE_TEST_ORG = true;
+
+	protected function setUp(): void
+	{
+		parent::setUp();
+	}
+
+	public function testInsertAndReadPersonObjects()
+	{
+		echo "Part 1: Insert Persons\n";
+
+		$sOrgId = $this->getTestOrgId();
+		$idx = 1;
+		$fStart = microtime(true);
+		$fMaxExecutionTimeAllowed = 40.0;
+		$iInitialPeak = 0;
+		$sInitialPeak = '';
+		$fStartLoop = $fStart;
+		for ($i = 0; $i < 3000; $i++) {
+			$oPerson = $this->CreateObject(Person::class, ['org_id' => $sOrgId, 'name' => "Person_$i", 'first_name' => 'John']);
+			if (0 == ($idx % 100)) {
+				$fDuration = microtime(true) - $fStartLoop;
+				$iMemoryPeakUsage = memory_get_peak_usage();
+				if ($iInitialPeak === 0) {
+					$iInitialPeak = $iMemoryPeakUsage;
+					$sInitialPeak = \utils::BytesToFriendlyFormat($iInitialPeak, 4);
+				}
+
+				$sCurrPeak = \utils::BytesToFriendlyFormat($iMemoryPeakUsage, 4);
+				echo "$idx ".sprintf('%.1f ms', $fDuration * 1000)." - Peak Memory Usage: $sCurrPeak\n";
+				$this->assertTrue($iMemoryPeakUsage === $iInitialPeak, "Peak memory changed from $sInitialPeak to $sCurrPeak after $idx insert loops");				$fStartLoop = microtime(true);
+
+				$fTotalDuration = microtime(true) - $fStart;
+				$sTotalDuration = sprintf('%.3f s', $fTotalDuration);
+				$this->assertTrue($fTotalDuration < $fMaxExecutionTimeAllowed, "execution time $sTotalDuration should be < $fMaxExecutionTimeAllowed ($idx insert loops)");
+			}
+			$idx++;
+		}
+
+		$fTotalDuration = microtime(true) - $fStart;
+		$sTotalDuration = sprintf('%.3f s', $fTotalDuration);
+		echo "Total duration: $sTotalDuration\n\n";
+		$this->assertTrue($fTotalDuration < $fMaxExecutionTimeAllowed, "Total execution time $sTotalDuration should be < $fMaxExecutionTimeAllowed");
+
+		//////////////////////
+		// Part 2 Fetch all the created persons
+		echo "Part 1: Fetch Persons\n";
+
+		$oSearch = DBSearch::FromOQL('SELECT Person WHERE org_id=:org_id');
+		$oSet = new DBObjectSet($oSearch, [], ['org_id' => $sOrgId]);
+		$idx = 1;
+		$iInitialPeak = 0;
+		$sInitialPeak = '';
+		$fMaxExecutionTimeAllowed = 0.5;
+		$fStart = microtime(true);
+		$fStartLoop = $fStart;
+		while ($oContact = $oSet->Fetch()) {
+			if (0 == ($idx % 100)) {
+				$fDuration = microtime(true) - $fStartLoop;
+				$iMemoryPeakUsage = memory_get_peak_usage();
+				if ($iInitialPeak === 0) {
+					$iInitialPeak = $iMemoryPeakUsage;
+					$sInitialPeak = \utils::BytesToFriendlyFormat($iInitialPeak, 4);
+				}
+
+				$sCurrPeak = \utils::BytesToFriendlyFormat($iMemoryPeakUsage, 4);
+				echo "$idx ".sprintf('%.1f ms', $fDuration * 1000)." - Peak Memory Usage: $sCurrPeak\n";
+				$this->assertTrue($iMemoryPeakUsage === $iInitialPeak, "Peak memory changed from $sInitialPeak to $sCurrPeak after $idx fetch loops");
+				$fStartLoop = microtime(true);
+
+				$fTotalDuration = microtime(true) - $fStart;
+				$sTotalDuration = sprintf('%.3f s', $fTotalDuration);
+				$this->assertTrue($fTotalDuration < $fMaxExecutionTimeAllowed, "Total execution time $sTotalDuration should be < $fMaxExecutionTimeAllowed ($idx fetch loops)");
+			}
+			$idx++;
+		}
+		$fTotalDuration = microtime(true) - $fStart;
+		$sTotalDuration = sprintf('%.3f s', $fTotalDuration);
+		echo "Total duration: $sTotalDuration\n\n";
+		$this->assertTrue($fTotalDuration < $fMaxExecutionTimeAllowed, "Total execution time $sTotalDuration should be < $fMaxExecutionTimeAllowed");
+	}
+}

tests/php-unit-tests/phpunit.xml.dist

@@ -34,6 +34,9 @@
 
   <testsuites>
     <!-- Unitary tests -->
+    <testsuite name="Perf">
+      <directory>perf-tests</directory>
+    </testsuite>
     <testsuite name="Application">
       <directory>unitary-tests/application</directory>
     </testsuite>

tests/php-unit-tests/src/BaseTestCase/ItopDataTestCase.php

@@ -267,7 +267,9 @@ abstract class ItopDataTestCase extends ItopTestCase
 		$oMyObj->DBInsert();
 		$iKey = $oMyObj->GetKey();
 		$this->debug("Created $sClass::$iKey");
-		$this->aCreatedObjects[] = $oMyObj;
+		if (!static::USE_TRANSACTION) {
+			$this->aCreatedObjects[] = $oMyObj;
+		}
 
 		return $oMyObj;
 	}

tests/php-unit-tests/unitary-tests/core/DBObjectTest.php

@@ -947,15 +947,14 @@ class DBObjectTest extends ItopDataTestCase
 	 */
 	public function testConstructorMemoryFootprint():void
 	{
-		$idx = 0;
+		$idx = 1;
 		$fStart = microtime(true);
 		$fStartLoop = $fStart;
 		$iInitialPeak = 0;
-		$iMaxAllowedMemoryIncrease = 1 * 1024 * 1024;
 
 		for ($i = 0; $i < 5000; $i++) {
 			/** @noinspection PhpUnusedLocalVariableInspection We intentionally use a reference that will disappear on each loop */
-			$oPerson = new \Person();
+			$oPerson = MetaModel::NewObject(\Person::class);
 			if (0 == ($idx % 100)) {
 				$fDuration = microtime(true) - $fStartLoop;
 				$iMemoryPeakUsage = memory_get_peak_usage();
@@ -967,7 +966,7 @@ class DBObjectTest extends ItopDataTestCase
 				$sCurrPeak = \utils::BytesToFriendlyFormat($iMemoryPeakUsage, 4);
 				echo "$idx ".sprintf('%.1f ms', $fDuration * 1000)." - Peak Memory Usage: $sCurrPeak\n";
 
-				$this->assertTrue(($iMemoryPeakUsage - $iInitialPeak) <= $iMaxAllowedMemoryIncrease , "Peak memory changed from $sInitialPeak to $sCurrPeak after $i loops");
+				$this->assertTrue($iMemoryPeakUsage === $iInitialPeak, "Peak memory changed from $sInitialPeak to $sCurrPeak after $i loops");
 
 				$fStartLoop = microtime(true);
 			}