N°1188 - Backup needs too much disk space

SVN:trunk[5144]
This commit is contained in:
Eric Espié
2017-11-30 08:52:44 +00:00
parent 694da178c4
commit 6477e2e1bb
3 changed files with 2473 additions and 246 deletions

View File

@@ -16,6 +16,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with iTop. If not, see <http://www.gnu.org/licenses/>
require_once('tar.php');
interface BackupArchive
{
/**
@@ -71,8 +73,11 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
if ($dh = opendir($sDir))
{
// Add the directory
if (!empty($sZipDir)) $this->addEmptyDir($sZipDir);
if (!empty($sZipDir))
{
$this->addEmptyDir($sZipDir);
}
// Loop through all the files
while (($sFile = readdir($dh)) !== false)
{
@@ -94,6 +99,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
}
}
/**
* @param string $sFile
* @return bool <b>TRUE</b> if the file is present, <b>FALSE</b> otherwise.
@@ -143,45 +149,45 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
}
// Extract only the selected files
if ((count($aFiles) > 0) && ($this->extractTo($sDestinationDir, $aFiles) === true))
if ((count($aFiles) > 0) && ($this->extractTo($sDestinationDir, $aFiles) === true))
{
return true;
}
return false;
}
} // class ZipArchiveEx
class BackupException extends Exception
{
}
class DBBackup
{
// To be overriden depending on the expected usages
protected function LogInfo($sMsg)
{
}
protected function LogError($sMsg)
{
}
protected $sDBHost;
protected $iDBPort;
protected $sDBUser;
protected $sDBPwd;
protected $sDBName;
protected $sDBSubName;
/**
* Connects to the database to backup
* By default, connects to the current MetaModel (must be loaded)
*
* @param sDBHost string Database host server
* @param $sDBUser string User login
* @param $sDBPwd string User password
* @param $sDBName string Database name
* @param $sDBSubName string Prefix to the tables of itop in the database
*
* @param string sDBHost Database host server
* @param string $sDBUser User login
* @param string $sDBPwd User password
* @param string $sDBName Database name
* @param string $sDBSubName Prefix to the tables of itop in the database
*/
public function __construct($sDBHost = null, $sDBUser = null, $sDBPwd = null, $sDBName = null, $sDBSubName = null)
{
@@ -194,7 +200,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$sDBName = MetaModel::GetConfig()->GetDBName();
$sDBSubName = MetaModel::GetConfig()->GetDBSubName();
}
// Compute the port (if present in the host name)
$aConnectInfo = explode(':', $sDBHost);
$sDBHostName = $aConnectInfo[0];
@@ -206,7 +212,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$iDBPort = null;
}
$this->sDBHost = $sDBHostName;
$this->iDBPort = $iDBPort;
$this->sDBUser = $sDBUser;
@@ -214,21 +220,22 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$this->sDBName = $sDBName;
$this->sDBSubName = $sDBSubName;
}
protected $sMySQLBinDir = '';
/**
* Create a normalized backup name, depending on the current date/time and Database
* @param sNameSpec string Name and path, eventually containing itop placeholders + time formatting specs
*/
*/
public function SetMySQLBinDir($sMySQLBinDir)
{
$this->sMySQLBinDir = $sMySQLBinDir;
}
}
/**
* Create a normalized backup name, depending on the current date/time and Database
* @param sNameSpec string Name and path, eventually containing itop placeholders + time formatting specs
*/
* @param string sNameSpec Name and path, eventually containing itop placeholders + time formatting specs
*/
public function MakeName($sNameSpec = "__DB__-%Y-%m-%d")
{
$sFileName = $sNameSpec;
@@ -242,8 +249,8 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
/**
* @deprecated 2.4.0 Zip files are limited to 4 Gb, use CreateCompressedBackup to create tar.gz files
* @param $sZipFile
* @param null $sSourceConfigFile
* @param string $sZipFile
* @param string|null $sSourceConfigFile
*/
public function CreateZip($sZipFile, $sSourceConfigFile = null)
{
@@ -259,7 +266,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
'dest' => 'itop-dump.sql',
);
foreach ($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
foreach($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
{
$aContents[] = array(
'source' => $sSourceFile,
@@ -282,124 +289,59 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$this->LogInfo("Creating backup: '$sTargetFile.tar.gz'");
// Note: PharData::compress strips everything after the first dot found in the name of the tar, then it adds .tar.gz
// Hence, we have to create our own file in the target directory, and rename it when the process is complete
$sTarFile = dirname($sTargetFile) . '/' . str_replace('.', '_', basename($sTargetFile)) . '.tar';
$this->LogInfo("Tar file: '$sTarFile'");
$oArchive = new PharData($sTarFile);
$oArchive = new ArchiveTar($sTargetFile.'.tar.gz');
foreach ($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
{
if (is_dir($sSourceFile))
{
$this->LogInfo("Adding directory into tar file: '$sSourceFile', recorded as '$sArchiveFile'");
// Note: Phar::buildFromDirectory does not allow to specify a destination subdirectory
// Hence we have to add all files one by one
$sSourceDir = realpath($sSourceFile);
$sArchiveDir = trim($sArchiveFile, '/');
$sTmpFolder = APPROOT.'data/tmp-backup-'.rand(10000, getrandmax());
$aFiles = $this->PrepareFilesToBackup($sSourceConfigFile, $sTmpFolder);
$oDirectoryIterator = new RecursiveDirectoryIterator($sSourceDir, RecursiveDirectoryIterator::SKIP_DOTS);
$oAllFiles = new RecursiveIteratorIterator($oDirectoryIterator);
foreach ($oAllFiles as $oSomeFile)
{
if ($oSomeFile->isDir()) continue;
$oArchive->createModify($aFiles, '', $sTmpFolder);
// Replace the local path by the archive path - the resulting string starts with a '/'
$sRelativePathName = substr($oSomeFile->getRealPath(), strlen($sSourceDir));
// Under Windows realpath gives a mix of backslashes and slashes
$sRelativePathName = str_replace('\\', '/', $sRelativePathName);
$sArchiveFile = $sArchiveDir.$sRelativePathName;
$oArchive->addFile($oSomeFile->getPathName(), $sArchiveFile);
}
}
else
{
$this->LogInfo("Adding file into tar file: '$sSourceFile', recorded as '$sArchiveFile'");
$oArchive->addFile($sSourceFile, $sArchiveFile);
};
}
// Note: the file is created by tempnam and might not be writeable by another process (Windows/IIS)
// (delete it before spawning a process)
// Note: the file is created by tempnam and might not be writeable by another process (Windows/IIS)
// (delete it before spawning a process)
$sDataFile = tempnam(SetupUtils::GetTmpDir(), 'itop-');
$this->LogInfo("Data file: '$sDataFile'");
$this->DoBackup($sDataFile);
$oArchive->addFile($sDataFile, 'itop-dump.sql');
// todo: reduce disk space needed by the operation by piping the output of mysqldump directly into the tar
// tip1 : this syntax works fine (did not work with addFile)
//$oArchive->buildFromIterator(
// new ArrayIterator(
// array('production.delta.xml' => fopen(ROOTDIR.'production.delta.xml', 'rb'))
// )
//);
// tip2 : use the phar stream by redirecting the output of mysqldump into
// phar://var/www/itop/data/backups/manual/trunk_pro-2017-07-05_15_10.tar.gz/itop-dump.sql
//
// new ArrayIterator(
// array('production.delta.xml' => fopen(ROOTDIR.'production.delta.xml', 'rb'))
// )
//);
// Windows/IIS: the data file has been created by the spawned process...
// trying to delete it will issue a warning, itself stopping the setup abruptely
@unlink($sDataFile);
if (file_exists($sTarFile.'.gz'))
{
// Prevent the gzip compression from failing -> the whole operation is an overwrite
$this->LogInfo("Overwriting tar.gz: '$sTarFile'");
unlink($sTarFile.'.gz');
}
// zlib is a must!
$oArchive->compress(Phar::GZ);
// Cleanup
unset($oArchive);
unlink($sTarFile);
if ($sTargetFile != $sTarFile)
{
// Give the file the expected name
if (file_exists($sTargetFile.'.gz'))
{
// Remove it -> the whole operation is an overwrite
$this->LogInfo("Overwriting tar.gz: '$sTargetFile'");
unlink($sTargetFile.'.gz');
}
rename($sTarFile.'.gz', $sTargetFile.'.tar.gz');
}
SetupUtils::rrmdir($sTmpFolder);
}
/**
* List files to store into the archive, in addition to the SQL dump
* @return array of sArchiveName => sFilePath
* Copy files to store into the temporary folder, in addition to the SQL dump
* @param string $sSourceConfigFile
* @param string $sTmpFolder
* @return array list of files to archive
*/
protected function GetAdditionalFiles($sSourceConfigFile)
protected function PrepareFilesToBackup($sSourceConfigFile, $sTmpFolder)
{
$aRet = array();
if (is_dir($sTmpFolder))
{
SetupUtils::rrmdir($sTmpFolder);
}
@mkdir($sTmpFolder, 0777, true);
if (is_null($sSourceConfigFile))
{
$sSourceConfigFile = MetaModel::GetConfig()->GetLoadedFile();
}
if (!empty($sSourceConfigFile))
{
$aRet['config-itop.php'] = $sSourceConfigFile;
$sFile = $sTmpFolder.'/config-itop.php';
copy($sSourceConfigFile, $sFile);
$aRet[] = $sFile;
}
$sDeltaFile = APPROOT.'data/'.utils::GetCurrentEnvironment().'.delta.xml';
if (file_exists($sDeltaFile))
{
$aRet['delta.xml'] = $sDeltaFile;
$sFile = $sTmpFolder.'/delta.xml';
copy($sDeltaFile, $sFile);
$aRet[] = $sFile;
}
$sExtraDir = APPROOT.'data/'.utils::GetCurrentEnvironment().'-modules/';
if (is_dir($sExtraDir))
{
$sModules = utils::GetCurrentEnvironment().'-modules/';
$aRet[$sModules] = $sExtraDir;
$sModules = utils::GetCurrentEnvironment().'-modules';
$sFile = $sTmpFolder.'/'.$sModules;
SetupUtils::copydir($sExtraDir, $sFile);
$aRet[] = $sFile;
}
$sDataFile = $sTmpFolder.'/itop-dump.sql';
$this->DoBackup($sDataFile);
$aRet[] = $sDataFile;
return $aRet;
}
@@ -409,20 +351,20 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
// It suggests to rely on pctnl_* function instead of using escapeshellargs
return escapeshellarg($sValue);
}
/**
* Create a backup file
*/
* Create a backup file
*/
public function DoBackup($sBackupFileName)
{
$sHost = self::EscapeShellArg($this->sDBHost);
$sUser = self::EscapeShellArg($this->sDBUser);
$sPwd = self::EscapeShellArg($this->sDBPwd);
$sDBName = self::EscapeShellArg($this->sDBName);
// Just to check the connection to the DB (better than getting the retcode of mysqldump = 1)
$oMysqli = $this->DBConnect();
$this->DBConnect();
$sTables = '';
if ($this->sDBSubName != '')
{
@@ -441,9 +383,9 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
$sTables = implode(' ', $aEscapedTables);
}
$this->LogInfo("Starting backup of $this->sDBHost/$this->sDBName(suffix:'$this->sDBSubName')");
$sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true);
if (empty($sMySQLBinDir))
{
@@ -453,7 +395,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$sMySQLDump = '"'.$sMySQLBinDir.'/mysqldump"';
}
// Store the results in a temporary file
$sTmpFileName = self::EscapeShellArg($sBackupFileName);
if (is_null($this->iDBPort))
@@ -465,12 +407,12 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$sPortOption = '--port='.$this->iDBPort.' ';
}
// Delete the file created by tempnam() so that the spawned process can write into it (Windows/IIS)
unlink($sBackupFileName);
@unlink($sBackupFileName);
// Note: opt implicitely sets lock-tables... which cancels the benefit of single-transaction!
// skip-lock-tables compensates and allows for writes during a backup
$sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd --result-file=$sTmpFileName $sDBName $sTables 2>&1";
$sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx --result-file=$sTmpFileName $sDBName $sTables";
// Now run the command for real
$this->LogInfo("Executing command: $sCommandDisplay");
$aOutput = array();
@@ -487,15 +429,15 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
unlink($sBackupFileName);
}
$this->LogError("Failed to execute: $sCommandDisplay. The command returned:$iRetCode");
foreach($aOutput as $sLine)
{
$this->LogError("mysqldump said: $sLine");
}
if (count($aOutput) == 1)
if (count($aOutput) == 1)
{
$sMoreInfo = trim($aOutput[0]);
$sMoreInfo = trim($aOutput[0]);
}
else
{
@@ -504,13 +446,13 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
throw new BackupException("Failed to execute mysqldump: ".$sMoreInfo);
}
}
/**
* Helper to create a ZIP out of several files
*/
*/
protected function DoZip($aFiles, $sZipArchiveFile)
{
foreach ($aFiles as $aFile)
foreach($aFiles as $aFile)
{
$sFile = $aFile['source'];
if (!is_file($sFile) && !is_dir($sFile))
@@ -521,12 +463,12 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
// Make sure the target path exists
$sZipDir = dirname($sZipArchiveFile);
SetupUtils::builddir($sZipDir);
$oZip = new ZipArchiveEx();
$res = $oZip->open($sZipArchiveFile, ZipArchive::CREATE | ZipArchive::OVERWRITE);
if ($res === TRUE)
{
foreach ($aFiles as $aFile)
foreach($aFiles as $aFile)
{
if (is_dir($aFile['source']))
{
@@ -553,10 +495,10 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
throw new BackupException("Failed to create zip archive: $sZipArchiveFile.");
}
}
/**
* Helper to download the file directly from the browser
*/
* Helper to download the file directly from the browser
*/
public function DownloadBackup($sFile)
{
header('Content-Description: File Transfer');
@@ -568,7 +510,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
header('Content-Length: '.filesize($sFile));
readfile($sFile);
}
/**
* Helper to open a Database connection
*/
@@ -585,7 +527,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
if ($oMysqli->connect_errno)
{
$sHost = is_null($this->iDBPort) ? $this->sDBHost : $this->sDBHost.' on port '.$this->iDBPort;
throw new BackupException("Cannot connect to the MySQL server '$this->sDBHost' (".$oMysqli->connect_errno . ") ".$oMysqli->connect_error);
throw new BackupException("Cannot connect to the MySQL server '$sHost' (".$oMysqli->connect_errno.") ".$oMysqli->connect_error);
}
if (!$oMysqli->select_db($this->sDBName))
{
@@ -593,10 +535,10 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
return $oMysqli;
}
/**
* Helper to enumerate the tables of the database
*/
*/
protected function EnumerateTables()
{
$oMysqli = $this->DBConnect();
@@ -625,9 +567,9 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
class TarGzArchive implements BackupArchive
{
/*
* @var PharData
* @var ArchiveTar
*/
protected $oPharArchive;
protected $oArchive;
/*
* string[]
*/
@@ -635,7 +577,7 @@ class TarGzArchive implements BackupArchive
public function __construct($sFile)
{
$this->oPharArchive = new PharData($sFile);
$this->oArchive = new ArchiveTar($sFile);
}
/**
@@ -644,7 +586,21 @@ class TarGzArchive implements BackupArchive
*/
public function hasFile($sFile)
{
return $this->oPharArchive->offsetExists($sFile);
// remove leading and tailing /
$sFile = trim($sFile, "/ \t\n\r\0\x0B");
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
foreach($this->aFiles as $aArchFile)
{
if ($aArchFile['filename'] == $sFile)
{
return true;
}
}
return false;
}
/**
@@ -653,7 +609,21 @@ class TarGzArchive implements BackupArchive
*/
public function hasDir($sDirectory)
{
return $this->oPharArchive->offsetExists($sDirectory);
// remove leading and tailing /
$sDirectory = trim($sDirectory, "/ \t\n\r\0\x0B");
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
foreach($this->aFiles as $aArchFile)
{
if (($aArchFile['typeflag'] == 5) && ($aArchFile['filename'] == $sDirectory))
{
return true;
}
}
return false;
}
/**
@@ -663,127 +633,38 @@ class TarGzArchive implements BackupArchive
*/
public function extractFileTo($sDestinationDir, $sArchiveFile)
{
return $this->oPharArchive->extractTo($sDestinationDir, $sArchiveFile, true);
return $this->oArchive->extractList($sArchiveFile, $sDestinationDir);
}
/**
* Extract a whole directory from the archive.
* Usage: $oArchive->extractDirTo('/var/www/html/itop/data', '/production-modules/')
* @param string $sDestinationDir
* @param string $sArchiveDir Note: must start and end with a slash !!
* @param string $sArchiveDir
* @return bool <b>TRUE</b> on success or <b>FALSE</b> on failure.
*/
public function extractDirTo($sDestinationDir, $sArchiveDir)
{
$aFiles = array();
foreach ($this->getFiles($sArchiveDir) as $oFileInfo)
{
$aFiles[] = $oFileInfo->getRelativePath();
}
if ((count($aFiles) > 0) && ($this->oPharArchive->extractTo($sDestinationDir, $aFiles, true) === true))
{
return true;
}
return false;
return $this->oArchive->extractList($sArchiveDir, $sDestinationDir);
}
/**
* Returns the entry contents using its name
* @param string $name Name of the entry
* @param int $length [optional] The length to be read from the entry. If 0, then the entire entry is read.
* @param int $flags [optional] The flags to use to open the archive. the following values may be ORed to it. <b>ZipArchive::FL_UNCHANGED</b>
* @param int $length unused.
* @param int $flags unused.
* @return string the contents of the entry on success or <b>FALSE</b> on failure.
*/
public function getFromName($name, $length = 0, $flags = null)
{
$oFileInfo = $this->oPharArchive->offsetGet($name);
$sFile = $oFileInfo->getPathname();
$sRet = file_get_contents($sFile);
return $sRet;
return $this->oArchive->extractInString($name);
}
/**
* @param string|null $sArchivePath Path to search for
* @return null
*/
public function getFiles($sArchivePath = null)
protected function buildFileList()
{
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
if ($sArchivePath === null)
{
// Take them all
$aRet = $this->aFiles;
}
else
{
// Filter out files not in the given path
$aRet = array();
foreach ($this->aFiles as $oFileInfo)
{
if ($oFileInfo->isUnder($sArchivePath))
{
$aRet[] = $oFileInfo;
}
}
}
return $aRet;
}
/**
* @param PharData|null $oPharData
* @param string $sArchivePath Path relatively to the archive root
*/
protected function buildFileList($oPharData = null, $sArchivePath = '/')
{
if ($oPharData === null)
{
$oPharData = $this->oPharArchive;
}
foreach($oPharData as $oPharFileInfo)
{
if($oPharFileInfo->isDir())
{
$oSubDirectory = new PharData($oPharFileInfo->getPathname());
// Recurse
$this->buildFileList($oSubDirectory, $sArchivePath.'/'.$oPharFileInfo->getFileName());
}
else
{
$this->aFiles[] = new TarGzFileInfo($oPharFileInfo, $sArchivePath);
}
}
}
}
class TarGzFileInfo
{
public function __construct(PharFileInfo $oFileInfo, $sArchivePath)
{
$this->oPharFileInfo = $oFileInfo;
$this->sArchivePath = trim($sArchivePath, '/');
}
protected $sArchivePath;
protected $oPharFileInfo;
public function getPathname()
{
return $this->oPharFileInfo->getPathname();
}
public function getRelativePath()
{
return $this->sArchivePath.'/'.$this->oPharFileInfo->getFilename();
}
public function isUnder($sArchivePath)
{
$sTestedPath = trim($sArchivePath, '/');
return (strpos($this->sArchivePath, $sTestedPath) === 0);
$this->aFiles = $this->oArchive->listContent();
}
}

View File

@@ -2443,6 +2443,22 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
</pre>]]></text>
</license>
<license>
<product>Archive_Tar</product>
<author>Vincent Blavet</author>
<license_type>BSD</license_type>
<text><![CDATA[<pre>The 2-Clause BSD License
Copyright Vincent Blavet
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
</pre>]]></text>
</license>
</licenses>

2330
setup/tar.php Normal file

File diff suppressed because it is too large Load Diff