QUAL Start to rewrite the export of accountancy. Misconception generates

a lot of trouble.

htdocs/accountancy/bookkeeping/export.php

@@ -505,6 +505,7 @@ $sql .= " t.label_operation,";
 $sql .= " t.debit,";
 $sql .= " t.credit,";
 $sql .= " t.lettering_code,";
+$sql .= " t.date_lettering,";
 $sql .= " t.montant as amount,";
 $sql .= " t.sens,";
 $sql .= " t.fk_user_author,";
@@ -513,6 +514,7 @@ $sql .= " t.code_journal,";
 $sql .= " t.journal_label,";
 $sql .= " t.piece_num,";
 $sql .= " t.date_creation,";
+$sql .= " t.date_lim_reglement,";
 $sql .= " t.tms as date_modification,";
 $sql .= " t.date_export,";
 $sql .= " t.date_validated as date_validation,";
@@ -575,19 +577,47 @@ if (count($sqlwhere) > 0) {
 // Export into a file with format defined into setup (FEC, CSV, ...)
 // Must be after definition of $sql
 if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements', 'export')) {
-	// TODO Replace the fetchAll to get all ->line followed by call to ->export(). fetchAll() currently consumes too much memory on large export.
-	// Replace this with the query($sql) and loop on each line to export them.
-	$result = $object->fetchAll($sortorder, $sortfield, 0, 0, $filter, 'AND', (!getDolGlobalString('ACCOUNTING_REEXPORT') ? 0 : 1));
+	// Export files then exit
+	$accountancyexport = new AccountancyExport($db);
+
+	$error = 0;
+	$nbtotalofrecords = 0;
+
+	// Open transaction to read lines to export, export them and update field date_export or date_validated
+	$db->begin();
+
+	/* The fast and low memory method to get and count full list converts the sql into a sql count */
+	$sqlforcount = preg_replace('/^'.preg_quote($sqlfields, '/').'/', 'SELECT COUNT(*) as nbtotalofrecords', $sql);
+	$sqlforcount = preg_replace('/GROUP BY .*$/', '', $sqlforcount);
+	$resql = $db->query($sqlforcount);
+	if ($resql) {
+		$objforcount = $db->fetch_object($resql);
+		$nbtotalofrecords = $objforcount->nbtotalofrecords;
+	} else {
+		dol_print_error($db);
+	}
+
+	$db->free($resql);
+
+	//$sqlforexport = $sql;
+	//$sqlforexport .= $db->order($sortfield, $sortorder);
+
+
+	// TODO Call the fetchAll for a $limit and $offset
+	// Replace the fetchAll to get all ->line followed by call to ->export(). fetchAll() currently consumes too much memory on large export.
+	// Replace this with the query($sqlforexport) on a limited block and loop on each line to export them.
+	$limit = 0;
+	$offset = 0;
+	$result = $object->fetchAll($sortorder, $sortfield, $limit, $offset, $filter, 'AND', (!getDolGlobalString('ACCOUNTING_REEXPORT') ? 0 : 1));
 
 	if ($result < 0) {
+		$error++;
 		setEventMessages($object->error, $object->errors, 'errors');
 	} else {
-		// Export files then exit
-		$accountancyexport = new AccountancyExport($db);
-
 		$formatexport = GETPOST('formatexport', 'int');
 		$notexportlettering = GETPOST('notexportlettering', 'alpha');
 
+
 		if (!empty($notexportlettering)) {
 			if (is_array($object->lines)) {
 				foreach ($object->lines as $k => $movement) {
@@ -602,9 +632,9 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements
 		$withAttachment = !empty(trim(GETPOST('notifiedexportfull', 'alphanohtml'))) ? 1 : 0;
 
 		// Output data on screen or download
-		$result = $accountancyexport->export($object->lines, $formatexport, $withAttachment);
+		//$result = $accountancyexport->export($object->lines, $formatexport, $withAttachment);
+		$result = $accountancyexport->export($object->lines, $formatexport, $withAttachment, 1, 1, 1);
 
-		$error = 0;
 		if ($result < 0) {
 			$error++;
 		} else {
@@ -612,11 +642,9 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements
 				if (is_array($object->lines)) {
 					dol_syslog("/accountancy/bookkeeping/list.php Function export_file Specify movements as exported", LOG_DEBUG);
 
-					// Specify as export : update field date_export or date_validated
-					$db->begin();
-
 					// TODO Merge update for each line into one global using rowid IN (list of movement ids)
 					foreach ($object->lines as $movement) {
+						// Upate the line to set date_export and/or date_validated (if not already set !)
 						$now = dol_now();
 
 						$setfields = '';
@@ -640,23 +668,45 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements
 						}
 					}
 
-					if (!$error) {
-						$db->commit();
-					} else {
-						$error++;
+					if ($error) {
 						$accountancyexport->errors[] = $langs->trans('NotAllExportedMovementsCouldBeRecordedAsExportedOrValidated');
-						$db->rollback();
 					}
 				}
 			}
 		}
-
-		if ($error) {
-			setEventMessages('', $accountancyexport->errors, 'errors');
-			header('Location: '.$_SERVER['PHP_SELF']);
-		}
-		exit(); // download or show errors
 	}
+
+	if (!$error) {
+		$db->commit();
+
+		$downloadFilePath = $accountancyexport->generatedfiledata['downloadFilePath'];
+		$downloadFileMimeType = $accountancyexport->generatedfiledata['downloadFileMimeType'];
+		$downloadFileFullName = $accountancyexport->generatedfiledata['downloadFileFullName'];
+
+		// No error, we can output the file
+		top_httphead($downloadFileMimeType);
+
+		header('Content-Description: File Transfer');
+		// Add MIME Content-Disposition from RFC 2183 (inline=automatically displayed, attachment=need user action to open)
+		$attachment = 1;
+		if ($attachment) {
+			header('Content-Disposition: attachment; filename="'.$downloadFileFullName.'"');
+		} else {
+			header('Content-Disposition: inline; filename="'.$downloadFileFullName.'"');
+		}
+		// Ajout directives pour resoudre bug IE
+		header('Cache-Control: Public, must-revalidate');
+		header('Pragma: public');
+		header('Content-Length: ' . dol_filesize($downloadFilePath));
+
+		readfileLowMemory($downloadFilePath);
+	} else {
+		$db->rollback();
+
+		setEventMessages('', $accountancyexport->errors, 'errors');
+		header('Location: '.$_SERVER['PHP_SELF']);
+	}
+	exit(); // download or show errors
 }
 
 

htdocs/accountancy/class/accountancyexport.class.php

@@ -81,17 +81,21 @@ class AccountancyExport
 	public $errors = array();
 
 	/**
-	 *
-	 * @var string Separator
+	 * @var string 	Separator
 	 */
 	public $separator = '';
 
 	/**
-	 *
-	 * @var string End of line
+	 * @var string 	End of line
 	 */
 	public $end_line = '';
 
+	/**
+	 * @var array 	Generated file
+	 */
+	public $generatedfiledata = array();
+
+
 	/**
 	 * Constructor
 	 *
@@ -195,7 +199,7 @@ class AccountancyExport
 	 */
 	public function getTypeConfig()
 	{
-		global $conf, $langs;
+		global $langs;
 
 		$exporttypes = array(
 			'param' => array(
@@ -319,24 +323,25 @@ class AccountancyExport
 	 * @param	int		$formatexportset			Id of export format
 	 * @param	int		$withAttachment				[=0] Not add files
 	 * 												or 1 to have attached in an archive (ex : Quadratus) - Force output mode to write in a file (output mode = 1)
-	 * @param	int		$downloadMode				[=0] Direct download
+	 * @param	int		$downloadMode				[=0] Direct download. Deprecated. Always use value 1.
 	 * 												or 1 to download after writing files - Forced by default when use withAttachment = 1
 	 * 												or -1 not to download files
-	 * @param	int		$outputMode					[=0] Print on screen
-	 * 												or 1 to write in file and uses a temp directory - Forced by default when use withAttachment = 1
+	 * @param	int		$outputMode					[=0] Print on screen. Deprecated. Always use value 1.
+	 * 												or 1 to write in file and uses the temp directory - Forced by default when use withAttachment = 1
 	 * 												or 2 to write in file a default export directory (accounting/export/)
-	 * @return 	int		Return integer <0 if KO, >0 OK
+	 * @param	int		$noouput					0=old mode. Deprecated. Always use value 1.
+	 * 												or 1=Do not output the file on stdout with this method. This must always be done by the main page, never by a method.
+	 * @return 	int									Return integer <0 if KO, >0 OK. The property ->generatedfile is also filled.
 	 */
-	public function export(&$TData, $formatexportset, $withAttachment = 0, $downloadMode = 0, $outputMode = 0)
+	public function export(&$TData, $formatexportset, $withAttachment = 0, $downloadMode = 0, $outputMode = 0, $noouput = 0)
 	{
-		global $conf, $langs;
-		global $search_date_end; // Used into /accountancy/tpl/export_journal.tpl.php
+		global $db, $conf, $langs;	// The tpl file use $db
+		global $search_date_end; 	// Used into /accountancy/tpl/export_journal.tpl.php
 
 		// Define name of file to save
 		$filename = 'general_ledger-'.$this->getFormatCode($formatexportset);
 		$type_export = 'general_ledger';
 
-		global $db; // The tpl file use $db
 		$completefilename = '';
 		$exportFile = null;
 		$exportFileName = '';
@@ -369,17 +374,19 @@ class AccountancyExport
 			// begin to print header for direct download
 			top_httphead($mimetype, 1);
 		}
+
 		include DOL_DOCUMENT_ROOT.'/accountancy/tpl/export_journal.tpl.php';
+
 		if ($outputMode == 1 || $outputMode == 2) {
 			if ($outputMode == 1) {
-				// uses temp directory by default to write files
+				// uses the temp directory by default to write files
 				if (!empty($conf->accounting->multidir_temp[$conf->entity])) {
 					$outputDir = $conf->accounting->multidir_temp[$conf->entity];
 				} else {
 					$outputDir = $conf->accounting->dir_temp;
 				}
 			} else {
-				// uses default export directory "accounting/export"
+				// uses the default export directory "accounting/export"
 				if (!empty($conf->accounting->multidir_output[$conf->entity])) {
 					$outputDir = $conf->accounting->multidir_output[$conf->entity];
 				} else {
@@ -503,7 +510,8 @@ class AccountancyExport
 				break;
 		}
 
-		// create and download export file or archive
+
+		// Create and download export file or archive
 		if ($outputMode == 1 || $outputMode == 2) {
 			$error = 0;
 
@@ -558,14 +566,19 @@ class AccountancyExport
 					}
 
 					// download export file or archive
-					if (!empty($downloadFileMimeType) && !empty($downloadFileFullName) && !empty($downloadFilePath)) {
+					if (!empty($downloadFileMimeType) && !empty($downloadFileFullName) && !empty($downloadFilePath) && empty($noouput)) {
+						// deprecated. We must not use this anymore, but have $noouput = 1 because HTTP header must be sent
+						// into main page not into a method.
 						header('Content-Type: ' . $downloadFileMimeType);
 						header('Content-Disposition: attachment; filename=' . $downloadFileFullName);
 						header('Cache-Control: Public, must-revalidate');
 						header('Pragma: public');
 						header('Content-Length: ' . dol_filesize($downloadFilePath));
+
 						readfileLowMemory($downloadFilePath);
 					}
+
+					$this->generatedfiledata = array('downloadFilePath' => $downloadFilePath, 'downloadFileMimeType' => $downloadFileMimeType, 'downloadFileFullName' => $downloadFileFullName);
 				}
 			}
 
@@ -1343,6 +1356,7 @@ class AccountancyExport
 
 		foreach ($objectLines as $line) {
 			if ($line->debit == 0 && $line->credit == 0) {
+				//var_dump($line->id);
 				//unset($array[$line]);
 			} else {
 				$date_creation = dol_print_date($line->date_creation, '%Y%m%d');
@@ -1492,6 +1506,7 @@ class AccountancyExport
 				$tab[] = $attachmentFileName;
 
 				$output = implode($separator, $tab).$end_line;
+
 				if ($exportFile) {
 					fwrite($exportFile, $output);
 				} else {

htdocs/accountancy/class/api_accountancy.class.php

@@ -219,6 +219,7 @@ class Accountancy extends DolibarrApi
 			$filter['t.doc_date<='] = $doc_date_end;
 		}
 
+		// @FIXME Critical bugged. Never use fetchAll without limit !
 		$result = $bookkeeping->fetchAll($sortorder, $sortfield, 0, 0, $filter, 'AND', $alreadyexport);
 
 		if ($result < 0) {

htdocs/document.php

@@ -295,6 +295,7 @@ if ($reshook < 0) {
 
 // Permissions are ok and file found, so we return it
 top_httphead($type);
+
 header('Content-Description: File Transfer');
 if ($encoding) {
 	header('Content-Encoding: '.$encoding);