composer/iTop
6
0
Fork 0
mirror of https://github.com/Combodo/iTop.git synced 2026-02-13 07:24:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

- Make sure that the CSV Parser has enough time to run on big amount of data.

Browse Source 
- Speedup the display of the CSV Import interactive wizard by parsing only the needed lines of the CSV data (in the first steps of the wizard).

SVN:trunk[4223]
This commit is contained in:
Denis Flaven
2016-06-17 08:41:20 +00:00
parent 12857ceba1
commit 9b774d3f72
4 changed files with 28 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/csvparser.class.inc.php
View File

@@ -54,12 +54,14 @@ class CSVParser
private $m_sCSVData;
private $m_sSep;
private $m_sTextQualifier;
private $m_iTimeLimitPerRow;
public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"')
public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"', $iTimeLimitPerRow = null)
{
$this->m_sCSVData = str_replace("\r\n", "\n", $sTxt);
$this->m_sSep = $sSep;
$this->m_sTextQualifier = $sTextQualifier;
$this->m_iTimeLimitPerRow = $iTimeLimitPerRow;
}
protected $m_sCurrCell = '';
@@ -129,6 +131,12 @@ class CSVParser
// blank line, skip silently
}
$this->m_aCurrRow = array();
// More time for the next row
if ($this->m_iTimeLimitPerRow !== null)
{
set_time_limit($this->m_iTimeLimitPerRow);
}
}
protected function __AddCellTrimmed($c = null, $aFieldMap = null)
{
@@ -181,6 +189,13 @@ class CSVParser
$iDataLength = strlen($this->m_sCSVData);
$iState = stSTARTING;
$iTimeLimit = null;
if ($this->m_iTimeLimitPerRow !== null)
{
// Give some time for the first row
$iTimeLimit = ini_get('max_execution_time');
set_time_limit($this->m_iTimeLimitPerRow);
}
for($i = 0; $i <= $iDataLength ; $i++)
{
if ($i == $iDataLength)
@@ -237,6 +252,11 @@ class CSVParser
$iLineCount = count($this->m_aDataSet);
if (($iMax > 0) && ($iLineCount >= $iMax)) break;
}
if ($iTimeLimit !== null)
{
// Restore the previous time limit
set_time_limit($iTimeLimit);
}
return $this->m_aDataSet;
}

10
pages/ajax.csvimport.php
View File

@@ -244,8 +244,9 @@ try
$bFirstLineAsHeader = utils::ReadParam('header_line', true);
$sEncoding = utils::ReadParam('encoding', 'UTF-8');
$sData = stripslashes(utils::ReadParam('csvdata', true, false, 'raw_data'));
$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
$aData = $oCSVParser->ToArray($iLinesToSkip);
$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
$iMaxIndex= 10; // Display maximum 10 lines for the preview
$aData = $oCSVParser->ToArray($iLinesToSkip, null, $iMaxIndex);
$iTarget = count($aData);
if ($iTarget == 0)
{
@@ -258,7 +259,6 @@ try
$oPage->p("<h3>".Dict::S('UI:Title:DataPreview')."</h3>\n");
$oPage->p("<div style=\"overflow-y:auto\" class=\"white\">\n");
$oPage->add("<table cellspacing=\"0\" style=\"overflow-y:auto\">");
$iMaxIndex= 10; // Display maximum 10 lines for the preview
$index = 1;
foreach($aData as $aRow)
{
@@ -318,8 +318,8 @@ try
$aInitFieldMapping = empty($sInitFieldMapping) ? array() : json_decode($sInitFieldMapping, true);
$aInitSearchField = empty($sInitSearchField) ? array() : json_decode($sInitSearchField, true);
$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
$aData = $oCSVParser->ToArray($iLinesToSkip);
$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
$aData = $oCSVParser->ToArray($iLinesToSkip, null, 3 /* Max: 1 header line + 2 lines of sample data */);
$iTarget = count($aData);
if ($iTarget == 0)
{

2
pages/csvimport.php
View File

@@ -232,7 +232,7 @@ try
}
// Parse the data set
$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier);
$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
$aData = $oCSVParser->ToArray($iSkippedLines);
$iRealSkippedLines = $iSkippedLines;
if ($bHeaderLine)

2
synchro/synchro_import.php
View File

@@ -429,7 +429,7 @@ try
{
$sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
}
$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier);
$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
$aInputColumns = $oCSVParser->ListFields();
$iColCount = count($aInputColumns);