(Retrofit from trunk)

r4961
N°1065 Fixed a regression introduced in r4965.
---------------------
r4960
N.1065 Fix performance issues. Limit APC emulation cache entries to avoid disk saturation. New configuration entry added: 'apc_cache_emulation.max_entries'.
---------------------
r4956
N.1065 Fix performance issues. ormLinkSet creates the objects on demand.
---------------------
r4954
APC emulation using files when APC or APCu is not installed.

SVN:2.4[4984]

core/apc-compat.php

@@ -89,4 +89,10 @@ function apc_cache_info_compat()
 		$aCacheUserData = @apc_cache_info('user');
 	}
 	return $aCacheUserData;
+}
+
+// Cache emulation
+if (!function_exists('apc_store'))
+{
+	require_once(APPROOT.'core/apc-emulation.php');
 }

core/apc-emulation.php

@@ -0,0 +1,261 @@
+<?php
+// Copyright (c) 2010-2017 Combodo SARL
+//
+//   This file is part of iTop.
+//
+//   iTop is free software; you can redistribute it and/or modify
+//   it under the terms of the GNU Affero General Public License as published by
+//   the Free Software Foundation, either version 3 of the License, or
+//   (at your option) any later version.
+//
+//   iTop is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU Affero General Public License for more details.
+//
+//   You should have received a copy of the GNU Affero General Public License
+//   along with iTop. If not, see <http://www.gnu.org/licenses/>
+//
+
+/**
+ * Date: 27/09/2017
+ */
+
+
+/**
+ * @param array|string $key
+ * @param $var
+ * @param int $ttl
+ * @return array|bool
+ */
+function apc_store($key, $var = NULL, $ttl = 0)
+{
+	if (is_array($key))
+	{
+		$aResult = array();
+		foreach($key as $sKey => $value)
+		{
+			$aResult[] = apc_emul_store_unit($sKey, $value, $ttl);
+		}
+		return $aResult;
+	}
+	return apc_emul_store_unit($key, $var, $ttl);
+}
+
+/**
+ * @param string $sKey
+ * @param $value
+ * @param int $iTTL time to live
+ * @return bool
+ */
+function apc_emul_store_unit($sKey, $value, $iTTL)
+{
+	if ($iTTL > 0)
+	{
+		// hint for ttl management
+		$sKey = '-'.$sKey;
+	}
+
+	$sFilename = apc_emul_get_cache_filename($sKey);
+	// try to create the folder
+	$sDirname = dirname($sFilename);
+	if (!file_exists($sDirname))
+	{
+		if (!@mkdir($sDirname, 0755, true))
+		{
+			return false;
+		}
+	}
+	$bRes = !(@file_put_contents($sFilename, serialize($value), LOCK_EX) === false);
+	apc_emul_manage_new_entry($sFilename);
+	return $bRes;
+}
+
+/**
+ * @param $key string|array
+ * @return mixed
+ */
+function apc_fetch($key)
+{
+	if (is_array($key))
+	{
+		$aResult = array();
+		foreach($key as $sKey)
+		{
+			$aResult[$sKey] = apc_emul_fetch_unit($sKey);
+		}
+		return $aResult;
+	}
+	return apc_emul_fetch_unit($key);
+}
+
+/**
+ * @param $sKey
+ * @return bool|mixed
+ */
+function apc_emul_fetch_unit($sKey)
+{
+	// Try the 'TTLed' version
+	$sValue = apc_emul_readcache_locked(apc_emul_get_cache_filename('-'.$sKey));
+	if ($sValue === false)
+	{
+		$sValue = apc_emul_readcache_locked(apc_emul_get_cache_filename($sKey));
+		if ($sValue === false)
+		{
+			return false;
+		}
+	}
+	$oRes = @unserialize($sValue);
+	return $oRes;
+}
+
+function apc_emul_readcache_locked($sFilename)
+{
+	$file = @fopen($sFilename, 'r');
+	if ($file === false)
+	{
+		return false;
+	}
+	flock($file, LOCK_SH);
+	$sContent = @fread($file, @filesize($sFilename));
+	flock($file, LOCK_UN);
+	fclose($file);
+	return $sContent;
+}
+
+/**
+ * @param string $cache_type
+ * @return bool
+ */
+function apc_clear_cache($cache_type = '')
+{
+	$sRootCacheDir = apc_emul_get_cache_filename('');
+	apc_emul_delete_entry($sRootCacheDir);
+	return true;
+}
+
+function apc_emul_delete_entry($sCache)
+{
+	if (is_dir($sCache))
+	{
+		$aFiles = array_diff(scandir($sCache), array('.', '..'));
+		foreach($aFiles as $sFile)
+		{
+			$sSubFile = $sCache.'/'.$sFile;
+			if (!apc_emul_delete_entry($sSubFile))
+			{
+				return false;
+			}
+		}
+		if (!@rmdir($sCache))
+		{
+			return false;
+		}
+	}
+	else
+	{
+		if (!@unlink($sCache))
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+/**
+ * @param $key
+ * @return bool|string[]
+ */
+function apc_delete($key)
+{
+	return apc_emul_delete_entry(apc_emul_get_cache_filename($key));
+}
+
+
+function apc_emul_get_cache_filename($sKey)
+{
+	$sPath = str_replace(array(' ', '/', '\\', '.'), '-', $sKey);
+	return utils::GetCachePath().'apc-emul/'.$sPath;
+}
+
+
+/** Manage the cache files when a new cache entry is added
+ * @param string $sNewFilename new cache file added
+ */
+function apc_emul_manage_new_entry($sNewFilename)
+{
+	// Check only once per request
+	static $aFilesByTime = null;
+	static $iFileCount = 0;
+	$iMaxFiles = MetaModel::GetConfig()->Get('apc_cache_emulation.max_entries');
+	if (!$aFilesByTime)
+	{
+		$sRootCacheDir = apc_emul_get_cache_filename('');
+		$aFilesByTime = apc_emul_list_files_time($sRootCacheDir);
+		$iFileCount = count($aFilesByTime);
+		if ($iMaxFiles !== 0)
+		{
+			asort($aFilesByTime);
+		}
+	}
+	else
+	{
+		$aFilesByTime[$sNewFilename] = time();
+		$iFileCount++;
+	}
+	if (($iMaxFiles !== 0) && ($iFileCount > $iMaxFiles))
+	{
+		$iFileNbToRemove = $iFileCount - $iMaxFiles;
+		foreach($aFilesByTime as $sFileToRemove => $iTime)
+		{
+			@unlink($sFileToRemove);
+			if ($iFileNbToRemove-- === 0)
+			{
+				break;
+			}
+		}
+		$aFilesByTime = array_slice($aFilesByTime, $iFileCount - $iMaxFiles, null, true);
+		$iFileCount = $iMaxFiles;
+	}
+}
+
+/** Get the list of files with their associated access time
+ * @param string $sCheck Directory to scan
+ * @param array $aFilesByTime used by recursion
+ * @return array
+ */
+function apc_emul_list_files_time($sCheck, &$aFilesByTime = array())
+{
+	// Garbage collection
+	$aFiles = array_diff(@scandir($sCheck), array('.', '..'));
+	foreach($aFiles as $sFile)
+	{
+		$sSubFile = $sCheck.'/'.$sFile;
+		if (is_dir($sSubFile))
+		{
+			apc_emul_list_files_time($sSubFile, $aFilesByTime);
+		}
+		else
+		{
+			$iTime = apc_emul_get_file_time($sSubFile);
+			if ($iTime !== false)
+			{
+				$aFilesByTime[$sSubFile] = $iTime;
+			}
+		}
+	}
+	return $aFilesByTime;
+}
+
+/** Get the file access time if TTL is managed
+ * @param string $sFilename
+ * @return bool|int returns the file atime or false if not relevant
+ */
+function apc_emul_get_file_time($sFilename)
+{
+	if (strpos(basename($sFilename), '-') === 0)
+	{
+		return @fileatime($sFilename);
+	}
+	return false;
+}

core/config.class.inc.php

@@ -435,6 +435,14 @@ class Config
 			'source_of_value' => '',
 			'show_in_conf_sample' => true,
 		),
+		'apc_cache_emulation.max_entries' => array(
+			'type' => 'integer',
+			'description' => 'Maximum number of cache entries (0 means no limit)',
+			'default' => 1000,
+			'value' => 1000,
+			'source_of_value' => '',
+			'show_in_conf_sample' => false,
+		),
 		'timezone' => array(
 			'type' => 'string',
 			'description' => 'Timezone (reference: http://php.net/manual/en/timezones.php). If empty, it will be left unchanged and MUST be explicitely configured in PHP',

core/ormlinkset.class.inc.php

@@ -52,7 +52,7 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 	 * Object from the original set, minus the removed objects
 	 * @var DBObject[] array of iObjectId => DBObject
 	 */
-	protected $aPreserved;
+	protected $aPreserved = array();
 
 	/**
 	 * @var DBObject[] New items
@@ -172,7 +172,7 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 		{
 			if ($this->oOriginalSet)
 			{
-				$this->aOriginalObjects = $this->oOriginalSet->ToArray();
+				$this->aOriginalObjects = $this->GetArrayOfIndex();
 				$this->aPreserved = $this->aOriginalObjects; // Copy (not effective until aPreserved gets modified)
                 foreach ($this->aRemoved as $iObjectId)
                 {
@@ -181,7 +181,7 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
                         unset($this->aPreserved[$iObjectId]);
                     }
                 }
-                foreach ($this->aModified as $iObjectId)
+                foreach ($this->aModified as $iObjectId => $oLink)
                 {
                     if (array_key_exists($iObjectId, $this->aPreserved))
                     {
@@ -199,6 +199,22 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 		}
 	}
 
+	/**
+	 * Note: After calling this method, the set cursor will be at the end of the set. You might want to rewind it.
+	 * @return array
+	 */
+	protected function GetArrayOfIndex()
+	{
+		$aRet = array();
+		$this->oOriginalSet->Rewind();
+		$iRow = 0;
+		while ($oObject = $this->oOriginalSet->Fetch())
+		{
+			$aRet[$oObject->GetKey()] = $iRow++;
+		}
+		return $aRet;
+	}
+
     /**
      * @param bool $bWithId
      * @return array
@@ -318,7 +334,9 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 		$iPreservedCount = count($this->aPreserved);
 		if ($this->iCursor < $iPreservedCount)
 		{
-			$oRet = current($this->aPreserved);
+			$iRet = current($this->aPreserved);
+			$this->oOriginalSet->Seek($iRet);
+			$oRet = $this->oOriginalSet->Fetch();
 		}
 		else
 		{
@@ -397,9 +415,7 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 	 */
 	public function rewind()
 	{
-		$this->LoadOriginalIds();
-
-		$this->iCursor = 0;
+	    $this->iCursor = 0;
 		reset($this->aPreserved);
         reset($this->aAdded);
         reset($this->aModified);
@@ -474,7 +490,7 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 			$this->aAdded = array();
 			$this->aRemoved = array();
 			$this->aModified = array();
-			$this->aPreserved = $this->aOriginalObjects;
+			$this->aPreserved = ($this->aOriginalObjects === null) ? array() : $this->aOriginalObjects;
 			$this->bHasDelta = false;
 
 			/** @var AttributeLinkedSet $oAttDef */
@@ -549,12 +565,15 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 
 		// Check for the existing links
 		//
+		/** @var DBObject[] $aExistingLinks */
+		$aExistingLinks = array();
+		/** @var Int[] $aExistingRemote */
+		$aExistingRemote = array();
 		if (count($aCheckLinks) > 0)
 		{
 			$oSearch = new DBObjectSearch($this->sClass);
 			$oSearch->AddCondition('id', $aCheckLinks, 'IN');
 			$oSet = new DBObjectSet($oSearch);
-			/** @var DBObject[] $aExistingLinks */
 			$aExistingLinks = $oSet->ToArray();
 		}
 
@@ -566,7 +585,6 @@ class ormLinkSet implements iDBObjectSetIterator, Iterator, SeekableIterator
 			$oSearch->AddCondition($sExtKeyToMe, $oHostObject->GetKey(), '=');
 			$oSearch->AddCondition($sExtKeyToRemote, $aCheckRemote, 'IN');
 			$oSet = new DBObjectSet($oSearch);
-			/** @var Int[] $aExistingRemote */
 			$aExistingRemote = $oSet->GetColumnAsArray($sExtKeyToRemote);
 		}