composer/iTop
6
0
Fork 0
mirror of https://github.com/Combodo/iTop.git synced 2026-05-02 06:58:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

N°1188 - Backup needs too much disk space

Browse Source 
SVN:trunk[5144]
This commit is contained in:
Eric Espié
2017-11-30 08:52:44 +00:00
parent 694da178c4
commit 6477e2e1bb
3 changed files with 2473 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

373
setup/backup.class.inc.php
View File

@@ -16,6 +16,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with iTop. If not, see <http://www.gnu.org/licenses/>
require_once('tar.php');
interface BackupArchive
{
/**
@@ -71,8 +73,11 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
if ($dh = opendir($sDir))
{
// Add the directory
if (!empty($sZipDir)) $this->addEmptyDir($sZipDir);
if (!empty($sZipDir))
{
$this->addEmptyDir($sZipDir);
}
// Loop through all the files
while (($sFile = readdir($dh)) !== false)
{
@@ -94,6 +99,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
}
}
/**
* @param string $sFile
* @return bool <b>TRUE</b> if the file is present, <b>FALSE</b> otherwise.
@@ -143,45 +149,45 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
}
// Extract only the selected files
if ((count($aFiles) > 0) && ($this->extractTo($sDestinationDir, $aFiles) === true))
if ((count($aFiles) > 0) && ($this->extractTo($sDestinationDir, $aFiles) === true))
{
return true;
}
return false;
}
} // class ZipArchiveEx
class BackupException extends Exception
{
}
class DBBackup
{
// To be overriden depending on the expected usages
protected function LogInfo($sMsg)
{
}
protected function LogError($sMsg)
{
}
protected $sDBHost;
protected $iDBPort;
protected $sDBUser;
protected $sDBPwd;
protected $sDBName;
protected $sDBSubName;
/**
* Connects to the database to backup
* By default, connects to the current MetaModel (must be loaded)
*
* @param sDBHost string Database host server
* @param $sDBUser string User login
* @param $sDBPwd string User password
* @param $sDBName string Database name
* @param $sDBSubName string Prefix to the tables of itop in the database
*
* @param string sDBHost Database host server
* @param string $sDBUser User login
* @param string $sDBPwd User password
* @param string $sDBName Database name
* @param string $sDBSubName Prefix to the tables of itop in the database
*/
public function __construct($sDBHost = null, $sDBUser = null, $sDBPwd = null, $sDBName = null, $sDBSubName = null)
{
@@ -194,7 +200,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$sDBName = MetaModel::GetConfig()->GetDBName();
$sDBSubName = MetaModel::GetConfig()->GetDBSubName();
}
// Compute the port (if present in the host name)
$aConnectInfo = explode(':', $sDBHost);
$sDBHostName = $aConnectInfo[0];
@@ -206,7 +212,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$iDBPort = null;
}
$this->sDBHost = $sDBHostName;
$this->iDBPort = $iDBPort;
$this->sDBUser = $sDBUser;
@@ -214,21 +220,22 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$this->sDBName = $sDBName;
$this->sDBSubName = $sDBSubName;
}
protected $sMySQLBinDir = '';
/**
* Create a normalized backup name, depending on the current date/time and Database
* @param sNameSpec string Name and path, eventually containing itop placeholders + time formatting specs
*/
*/
public function SetMySQLBinDir($sMySQLBinDir)
{
$this->sMySQLBinDir = $sMySQLBinDir;
}
}
/**
* Create a normalized backup name, depending on the current date/time and Database
* @param sNameSpec string Name and path, eventually containing itop placeholders + time formatting specs
*/
* @param string sNameSpec Name and path, eventually containing itop placeholders + time formatting specs
*/
public function MakeName($sNameSpec = "__DB__-%Y-%m-%d")
{
$sFileName = $sNameSpec;
@@ -242,8 +249,8 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
/**
* @deprecated 2.4.0 Zip files are limited to 4 Gb, use CreateCompressedBackup to create tar.gz files
* @param $sZipFile
* @param null $sSourceConfigFile
* @param string $sZipFile
* @param string|null $sSourceConfigFile
*/
public function CreateZip($sZipFile, $sSourceConfigFile = null)
{
@@ -259,7 +266,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
'dest' => 'itop-dump.sql',
);
foreach ($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
foreach($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
{
$aContents[] = array(
'source' => $sSourceFile,
@@ -282,124 +289,59 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$this->LogInfo("Creating backup: '$sTargetFile.tar.gz'");
// Note: PharData::compress strips everything after the first dot found in the name of the tar, then it adds .tar.gz
// Hence, we have to create our own file in the target directory, and rename it when the process is complete
$sTarFile = dirname($sTargetFile) . '/' . str_replace('.', '_', basename($sTargetFile)) . '.tar';
$this->LogInfo("Tar file: '$sTarFile'");
$oArchive = new PharData($sTarFile);
$oArchive = new ArchiveTar($sTargetFile.'.tar.gz');
foreach ($this->GetAdditionalFiles($sSourceConfigFile) as $sArchiveFile => $sSourceFile)
{
if (is_dir($sSourceFile))
{
$this->LogInfo("Adding directory into tar file: '$sSourceFile', recorded as '$sArchiveFile'");
// Note: Phar::buildFromDirectory does not allow to specify a destination subdirectory
// Hence we have to add all files one by one
$sSourceDir = realpath($sSourceFile);
$sArchiveDir = trim($sArchiveFile, '/');
$sTmpFolder = APPROOT.'data/tmp-backup-'.rand(10000, getrandmax());
$aFiles = $this->PrepareFilesToBackup($sSourceConfigFile, $sTmpFolder);
$oDirectoryIterator = new RecursiveDirectoryIterator($sSourceDir, RecursiveDirectoryIterator::SKIP_DOTS);
$oAllFiles = new RecursiveIteratorIterator($oDirectoryIterator);
foreach ($oAllFiles as $oSomeFile)
{
if ($oSomeFile->isDir()) continue;
$oArchive->createModify($aFiles, '', $sTmpFolder);
// Replace the local path by the archive path - the resulting string starts with a '/'
$sRelativePathName = substr($oSomeFile->getRealPath(), strlen($sSourceDir));
// Under Windows realpath gives a mix of backslashes and slashes
$sRelativePathName = str_replace('\\', '/', $sRelativePathName);
$sArchiveFile = $sArchiveDir.$sRelativePathName;
$oArchive->addFile($oSomeFile->getPathName(), $sArchiveFile);
}
}
else
{
$this->LogInfo("Adding file into tar file: '$sSourceFile', recorded as '$sArchiveFile'");
$oArchive->addFile($sSourceFile, $sArchiveFile);
};
}
// Note: the file is created by tempnam and might not be writeable by another process (Windows/IIS)
// (delete it before spawning a process)
// Note: the file is created by tempnam and might not be writeable by another process (Windows/IIS)
// (delete it before spawning a process)
$sDataFile = tempnam(SetupUtils::GetTmpDir(), 'itop-');
$this->LogInfo("Data file: '$sDataFile'");
$this->DoBackup($sDataFile);
$oArchive->addFile($sDataFile, 'itop-dump.sql');
// todo: reduce disk space needed by the operation by piping the output of mysqldump directly into the tar
// tip1 : this syntax works fine (did not work with addFile)
//$oArchive->buildFromIterator(
// new ArrayIterator(
// array('production.delta.xml' => fopen(ROOTDIR.'production.delta.xml', 'rb'))
// )
//);
// tip2 : use the phar stream by redirecting the output of mysqldump into
// phar://var/www/itop/data/backups/manual/trunk_pro-2017-07-05_15_10.tar.gz/itop-dump.sql
//
// new ArrayIterator(
// array('production.delta.xml' => fopen(ROOTDIR.'production.delta.xml', 'rb'))
// )
//);
// Windows/IIS: the data file has been created by the spawned process...
// trying to delete it will issue a warning, itself stopping the setup abruptely
@unlink($sDataFile);
if (file_exists($sTarFile.'.gz'))
{
// Prevent the gzip compression from failing -> the whole operation is an overwrite
$this->LogInfo("Overwriting tar.gz: '$sTarFile'");
unlink($sTarFile.'.gz');
}
// zlib is a must!
$oArchive->compress(Phar::GZ);
// Cleanup
unset($oArchive);
unlink($sTarFile);
if ($sTargetFile != $sTarFile)
{
// Give the file the expected name
if (file_exists($sTargetFile.'.gz'))
{
// Remove it -> the whole operation is an overwrite
$this->LogInfo("Overwriting tar.gz: '$sTargetFile'");
unlink($sTargetFile.'.gz');
}
rename($sTarFile.'.gz', $sTargetFile.'.tar.gz');
}
SetupUtils::rrmdir($sTmpFolder);
}
/**
* List files to store into the archive, in addition to the SQL dump
* @return array of sArchiveName => sFilePath
* Copy files to store into the temporary folder, in addition to the SQL dump
* @param string $sSourceConfigFile
* @param string $sTmpFolder
* @return array list of files to archive
*/
protected function GetAdditionalFiles($sSourceConfigFile)
protected function PrepareFilesToBackup($sSourceConfigFile, $sTmpFolder)
{
$aRet = array();
if (is_dir($sTmpFolder))
{
SetupUtils::rrmdir($sTmpFolder);
}
@mkdir($sTmpFolder, 0777, true);
if (is_null($sSourceConfigFile))
{
$sSourceConfigFile = MetaModel::GetConfig()->GetLoadedFile();
}
if (!empty($sSourceConfigFile))
{
$aRet['config-itop.php'] = $sSourceConfigFile;
$sFile = $sTmpFolder.'/config-itop.php';
copy($sSourceConfigFile, $sFile);
$aRet[] = $sFile;
}
$sDeltaFile = APPROOT.'data/'.utils::GetCurrentEnvironment().'.delta.xml';
if (file_exists($sDeltaFile))
{
$aRet['delta.xml'] = $sDeltaFile;
$sFile = $sTmpFolder.'/delta.xml';
copy($sDeltaFile, $sFile);
$aRet[] = $sFile;
}
$sExtraDir = APPROOT.'data/'.utils::GetCurrentEnvironment().'-modules/';
if (is_dir($sExtraDir))
{
$sModules = utils::GetCurrentEnvironment().'-modules/';
$aRet[$sModules] = $sExtraDir;
$sModules = utils::GetCurrentEnvironment().'-modules';
$sFile = $sTmpFolder.'/'.$sModules;
SetupUtils::copydir($sExtraDir, $sFile);
$aRet[] = $sFile;
}
$sDataFile = $sTmpFolder.'/itop-dump.sql';
$this->DoBackup($sDataFile);
$aRet[] = $sDataFile;
return $aRet;
}
@@ -409,20 +351,20 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
// It suggests to rely on pctnl_* function instead of using escapeshellargs
return escapeshellarg($sValue);
}
/**
* Create a backup file
*/
* Create a backup file
*/
public function DoBackup($sBackupFileName)
{
$sHost = self::EscapeShellArg($this->sDBHost);
$sUser = self::EscapeShellArg($this->sDBUser);
$sPwd = self::EscapeShellArg($this->sDBPwd);
$sDBName = self::EscapeShellArg($this->sDBName);
// Just to check the connection to the DB (better than getting the retcode of mysqldump = 1)
$oMysqli = $this->DBConnect();
$this->DBConnect();
$sTables = '';
if ($this->sDBSubName != '')
{
@@ -441,9 +383,9 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
$sTables = implode(' ', $aEscapedTables);
}
$this->LogInfo("Starting backup of $this->sDBHost/$this->sDBName(suffix:'$this->sDBSubName')");
$sMySQLBinDir = utils::ReadParam('mysql_bindir', $this->sMySQLBinDir, true);
if (empty($sMySQLBinDir))
{
@@ -453,7 +395,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
$sMySQLDump = '"'.$sMySQLBinDir.'/mysqldump"';
}
// Store the results in a temporary file
$sTmpFileName = self::EscapeShellArg($sBackupFileName);
if (is_null($this->iDBPort))
@@ -465,12 +407,12 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
$sPortOption = '--port='.$this->iDBPort.' ';
}
// Delete the file created by tempnam() so that the spawned process can write into it (Windows/IIS)
unlink($sBackupFileName);
@unlink($sBackupFileName);
// Note: opt implicitely sets lock-tables... which cancels the benefit of single-transaction!
// skip-lock-tables compensates and allows for writes during a backup
$sCommand = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=$sUser --password=$sPwd --result-file=$sTmpFileName $sDBName $sTables 2>&1";
$sCommandDisplay = "$sMySQLDump --opt --skip-lock-tables --default-character-set=utf8 --add-drop-database --single-transaction --host=$sHost $sPortOption --user=xxxxx --password=xxxxx --result-file=$sTmpFileName $sDBName $sTables";
// Now run the command for real
$this->LogInfo("Executing command: $sCommandDisplay");
$aOutput = array();
@@ -487,15 +429,15 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
{
unlink($sBackupFileName);
}
$this->LogError("Failed to execute: $sCommandDisplay. The command returned:$iRetCode");
foreach($aOutput as $sLine)
{
$this->LogError("mysqldump said: $sLine");
}
if (count($aOutput) == 1)
if (count($aOutput) == 1)
{
$sMoreInfo = trim($aOutput[0]);
$sMoreInfo = trim($aOutput[0]);
}
else
{
@@ -504,13 +446,13 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
throw new BackupException("Failed to execute mysqldump: ".$sMoreInfo);
}
}
/**
* Helper to create a ZIP out of several files
*/
*/
protected function DoZip($aFiles, $sZipArchiveFile)
{
foreach ($aFiles as $aFile)
foreach($aFiles as $aFile)
{
$sFile = $aFile['source'];
if (!is_file($sFile) && !is_dir($sFile))
@@ -521,12 +463,12 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
// Make sure the target path exists
$sZipDir = dirname($sZipArchiveFile);
SetupUtils::builddir($sZipDir);
$oZip = new ZipArchiveEx();
$res = $oZip->open($sZipArchiveFile, ZipArchive::CREATE | ZipArchive::OVERWRITE);
if ($res === TRUE)
{
foreach ($aFiles as $aFile)
foreach($aFiles as $aFile)
{
if (is_dir($aFile['source']))
{
@@ -553,10 +495,10 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
throw new BackupException("Failed to create zip archive: $sZipArchiveFile.");
}
}
/**
* Helper to download the file directly from the browser
*/
* Helper to download the file directly from the browser
*/
public function DownloadBackup($sFile)
{
header('Content-Description: File Transfer');
@@ -568,7 +510,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
header('Content-Length: '.filesize($sFile));
readfile($sFile);
}
/**
* Helper to open a Database connection
*/
@@ -585,7 +527,7 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
if ($oMysqli->connect_errno)
{
$sHost = is_null($this->iDBPort) ? $this->sDBHost : $this->sDBHost.' on port '.$this->iDBPort;
throw new BackupException("Cannot connect to the MySQL server '$this->sDBHost' (".$oMysqli->connect_errno . ") ".$oMysqli->connect_error);
throw new BackupException("Cannot connect to the MySQL server '$sHost' (".$oMysqli->connect_errno.") ".$oMysqli->connect_error);
}
if (!$oMysqli->select_db($this->sDBName))
{
@@ -593,10 +535,10 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
}
return $oMysqli;
}
/**
* Helper to enumerate the tables of the database
*/
*/
protected function EnumerateTables()
{
$oMysqli = $this->DBConnect();
@@ -625,9 +567,9 @@ if (class_exists('ZipArchive')) // The setup must be able to start even if the "
class TarGzArchive implements BackupArchive
{
/*
* @var PharData
* @var ArchiveTar
*/
protected $oPharArchive;
protected $oArchive;
/*
* string[]
*/
@@ -635,7 +577,7 @@ class TarGzArchive implements BackupArchive
public function __construct($sFile)
{
$this->oPharArchive = new PharData($sFile);
$this->oArchive = new ArchiveTar($sFile);
}
/**
@@ -644,7 +586,21 @@ class TarGzArchive implements BackupArchive
*/
public function hasFile($sFile)
{
return $this->oPharArchive->offsetExists($sFile);
// remove leading and tailing /
$sFile = trim($sFile, "/ \t\n\r\0\x0B");
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
foreach($this->aFiles as $aArchFile)
{
if ($aArchFile['filename'] == $sFile)
{
return true;
}
}
return false;
}
/**
@@ -653,7 +609,21 @@ class TarGzArchive implements BackupArchive
*/
public function hasDir($sDirectory)
{
return $this->oPharArchive->offsetExists($sDirectory);
// remove leading and tailing /
$sDirectory = trim($sDirectory, "/ \t\n\r\0\x0B");
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
foreach($this->aFiles as $aArchFile)
{
if (($aArchFile['typeflag'] == 5) && ($aArchFile['filename'] == $sDirectory))
{
return true;
}
}
return false;
}
/**
@@ -663,127 +633,38 @@ class TarGzArchive implements BackupArchive
*/
public function extractFileTo($sDestinationDir, $sArchiveFile)
{
return $this->oPharArchive->extractTo($sDestinationDir, $sArchiveFile, true);
return $this->oArchive->extractList($sArchiveFile, $sDestinationDir);
}
/**
* Extract a whole directory from the archive.
* Usage: $oArchive->extractDirTo('/var/www/html/itop/data', '/production-modules/')
* @param string $sDestinationDir
* @param string $sArchiveDir Note: must start and end with a slash !!
* @param string $sArchiveDir
* @return bool <b>TRUE</b> on success or <b>FALSE</b> on failure.
*/
public function extractDirTo($sDestinationDir, $sArchiveDir)
{
$aFiles = array();
foreach ($this->getFiles($sArchiveDir) as $oFileInfo)
{
$aFiles[] = $oFileInfo->getRelativePath();
}
if ((count($aFiles) > 0) && ($this->oPharArchive->extractTo($sDestinationDir, $aFiles, true) === true))
{
return true;
}
return false;
return $this->oArchive->extractList($sArchiveDir, $sDestinationDir);
}
/**
* Returns the entry contents using its name
* @param string $name Name of the entry
* @param int $length [optional] The length to be read from the entry. If 0, then the entire entry is read.
* @param int $flags [optional] The flags to use to open the archive. the following values may be ORed to it. <b>ZipArchive::FL_UNCHANGED</b>
* @param int $length unused.
* @param int $flags unused.
* @return string the contents of the entry on success or <b>FALSE</b> on failure.
*/
public function getFromName($name, $length = 0, $flags = null)
{
$oFileInfo = $this->oPharArchive->offsetGet($name);
$sFile = $oFileInfo->getPathname();
$sRet = file_get_contents($sFile);
return $sRet;
return $this->oArchive->extractInString($name);
}
/**
* @param string|null $sArchivePath Path to search for
* @return null
*/
public function getFiles($sArchivePath = null)
protected function buildFileList()
{
if ($this->aFiles === null)
{
// Initial load
$this->buildFileList();
}
if ($sArchivePath === null)
{
// Take them all
$aRet = $this->aFiles;
}
else
{
// Filter out files not in the given path
$aRet = array();
foreach ($this->aFiles as $oFileInfo)
{
if ($oFileInfo->isUnder($sArchivePath))
{
$aRet[] = $oFileInfo;
}
}
}
return $aRet;
}
/**
* @param PharData|null $oPharData
* @param string $sArchivePath Path relatively to the archive root
*/
protected function buildFileList($oPharData = null, $sArchivePath = '/')
{
if ($oPharData === null)
{
$oPharData = $this->oPharArchive;
}
foreach($oPharData as $oPharFileInfo)
{
if($oPharFileInfo->isDir())
{
$oSubDirectory = new PharData($oPharFileInfo->getPathname());
// Recurse
$this->buildFileList($oSubDirectory, $sArchivePath.'/'.$oPharFileInfo->getFileName());
}
else
{
$this->aFiles[] = new TarGzFileInfo($oPharFileInfo, $sArchivePath);
}
}
}
}
class TarGzFileInfo
{
public function __construct(PharFileInfo $oFileInfo, $sArchivePath)
{
$this->oPharFileInfo = $oFileInfo;
$this->sArchivePath = trim($sArchivePath, '/');
}
protected $sArchivePath;
protected $oPharFileInfo;
public function getPathname()
{
return $this->oPharFileInfo->getPathname();
}
public function getRelativePath()
{
return $this->sArchivePath.'/'.$this->oPharFileInfo->getFilename();
}
public function isUnder($sArchivePath)
{
$sTestedPath = trim($sArchivePath, '/');
return (strpos($this->sArchivePath, $sTestedPath) === 0);
$this->aFiles = $this->oArchive->listContent();
}
}