From 9b774d3f726159e5125fa222e0cd6751571d52c2 Mon Sep 17 00:00:00 2001
From: Denis Flaven <denis.flaven@combodo.com>
Date: Fri, 17 Jun 2016 08:41:20 +0000
Subject: [PATCH] - Make sure that the CSV Parser has enough time to run on big
 amount of data. - Speedup the display of the CSV Import interactive wizard by
 parsing only the needed lines of the CSV data (in the first steps of the
 wizard).

SVN:trunk[4223]
---
 core/csvparser.class.inc.php | 22 +++++++++++++++++++++-
 pages/ajax.csvimport.php     | 10 +++++-----
 pages/csvimport.php          |  2 +-
 synchro/synchro_import.php   |  2 +-
 4 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/core/csvparser.class.inc.php b/core/csvparser.class.inc.php
index 89a508947..3912a37e7 100644
--- a/core/csvparser.class.inc.php
+++ b/core/csvparser.class.inc.php
@@ -54,12 +54,14 @@ class CSVParser
 	private $m_sCSVData;
 	private $m_sSep;
 	private $m_sTextQualifier;
+	private $m_iTimeLimitPerRow;
 
-	public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"')
+	public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"', $iTimeLimitPerRow = null)
 	{
 		$this->m_sCSVData = str_replace("\r\n", "\n", $sTxt);
 		$this->m_sSep = $sSep;
 		$this->m_sTextQualifier = $sTextQualifier;
+		$this->m_iTimeLimitPerRow = $iTimeLimitPerRow;
 	}
 
 	protected $m_sCurrCell = '';
@@ -129,6 +131,12 @@ class CSVParser
 			// blank line, skip silently
 		}
 		$this->m_aCurrRow = array();
+		
+		// More time for the next row
+		if ($this->m_iTimeLimitPerRow !== null)
+		{
+			set_time_limit($this->m_iTimeLimitPerRow);
+		}
 	}
 	protected function __AddCellTrimmed($c = null, $aFieldMap = null)
 	{
@@ -181,6 +189,13 @@ class CSVParser
 		$iDataLength = strlen($this->m_sCSVData);
 
 		$iState = stSTARTING;
+		$iTimeLimit = null;
+		if ($this->m_iTimeLimitPerRow !== null)
+		{
+			// Give some time for the first row
+			$iTimeLimit = ini_get('max_execution_time');
+			set_time_limit($this->m_iTimeLimitPerRow);
+		}
 		for($i = 0; $i <= $iDataLength ; $i++)
 		{
 			if ($i == $iDataLength)
@@ -237,6 +252,11 @@ class CSVParser
 			$iLineCount = count($this->m_aDataSet);
 			if (($iMax > 0) && ($iLineCount >= $iMax)) break;
 		}
+		if ($iTimeLimit !== null)
+		{
+			// Restore the previous time limit
+			set_time_limit($iTimeLimit);
+		}
 		return $this->m_aDataSet;
 	}
 
diff --git a/pages/ajax.csvimport.php b/pages/ajax.csvimport.php
index 234c537e5..f3333604e 100644
--- a/pages/ajax.csvimport.php
+++ b/pages/ajax.csvimport.php
@@ -244,8 +244,9 @@ try
 		$bFirstLineAsHeader = utils::ReadParam('header_line', true);
 		$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 		$sData = stripslashes(utils::ReadParam('csvdata', true, false, 'raw_data'));
-		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
-		$aData = $oCSVParser->ToArray($iLinesToSkip);
+		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
+		$iMaxIndex= 10; // Display maximum 10 lines for the preview
+		$aData = $oCSVParser->ToArray($iLinesToSkip, null, $iMaxIndex);
 		$iTarget = count($aData);
 		if ($iTarget == 0)
 		{
@@ -258,7 +259,6 @@ try
 			$oPage->p("<h3>".Dict::S('UI:Title:DataPreview')."</h3>\n");
 			$oPage->p("<div style=\"overflow-y:auto\" class=\"white\">\n");
 			$oPage->add("<table cellspacing=\"0\" style=\"overflow-y:auto\">");
-			$iMaxIndex= 10; // Display maximum 10 lines for the preview
 			$index = 1;
 			foreach($aData as $aRow)
 			{
@@ -318,8 +318,8 @@ try
 		$aInitFieldMapping = empty($sInitFieldMapping) ? array() : json_decode($sInitFieldMapping, true);
 		$aInitSearchField = empty($sInitSearchField) ? array() : json_decode($sInitSearchField, true);
 
-		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
-		$aData = $oCSVParser->ToArray($iLinesToSkip);
+		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
+		$aData = $oCSVParser->ToArray($iLinesToSkip, null, 3 /* Max: 1 header line + 2 lines of sample data */);
 		$iTarget = count($aData);
 		if ($iTarget == 0)
 		{
diff --git a/pages/csvimport.php b/pages/csvimport.php
index 0ff47f197..7ea68f8eb 100644
--- a/pages/csvimport.php
+++ b/pages/csvimport.php
@@ -232,7 +232,7 @@ try
 		}
 				
 		// Parse the data set
-		$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier);
+		$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
 		$aData = $oCSVParser->ToArray($iSkippedLines);
 		$iRealSkippedLines = $iSkippedLines;
 		if ($bHeaderLine)
diff --git a/synchro/synchro_import.php b/synchro/synchro_import.php
index 11adf586b..75886c71b 100644
--- a/synchro/synchro_import.php
+++ b/synchro/synchro_import.php
@@ -429,7 +429,7 @@ try
 	{
 		$sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
 	}
-	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); 
+	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop')); 
 
 	$aInputColumns = $oCSVParser->ListFields();
 	$iColCount = count($aInputColumns);