From 949f5fc4f91e753d0aad5f97196e59565de1fdfd Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Mon, 22 Jan 2024 19:17:22 +0100 Subject: [PATCH] QUAL Start to rewrite the export of accountancy. Misconception generates a lot of trouble. --- htdocs/accountancy/bookkeeping/export.php | 94 ++++++++++++++----- .../class/accountancyexport.class.php | 49 ++++++---- .../class/api_accountancy.class.php | 1 + htdocs/document.php | 1 + 4 files changed, 106 insertions(+), 39 deletions(-) diff --git a/htdocs/accountancy/bookkeeping/export.php b/htdocs/accountancy/bookkeeping/export.php index f92c3aee5f0..f35cad6c18f 100644 --- a/htdocs/accountancy/bookkeeping/export.php +++ b/htdocs/accountancy/bookkeeping/export.php @@ -505,6 +505,7 @@ $sql .= " t.label_operation,"; $sql .= " t.debit,"; $sql .= " t.credit,"; $sql .= " t.lettering_code,"; +$sql .= " t.date_lettering,"; $sql .= " t.montant as amount,"; $sql .= " t.sens,"; $sql .= " t.fk_user_author,"; @@ -513,6 +514,7 @@ $sql .= " t.code_journal,"; $sql .= " t.journal_label,"; $sql .= " t.piece_num,"; $sql .= " t.date_creation,"; +$sql .= " t.date_lim_reglement,"; $sql .= " t.tms as date_modification,"; $sql .= " t.date_export,"; $sql .= " t.date_validated as date_validation,"; @@ -575,19 +577,47 @@ if (count($sqlwhere) > 0) { // Export into a file with format defined into setup (FEC, CSV, ...) // Must be after definition of $sql if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements', 'export')) { - // TODO Replace the fetchAll to get all ->line followed by call to ->export(). fetchAll() currently consumes too much memory on large export. - // Replace this with the query($sql) and loop on each line to export them. - $result = $object->fetchAll($sortorder, $sortfield, 0, 0, $filter, 'AND', (!getDolGlobalString('ACCOUNTING_REEXPORT') ? 0 : 1)); + // Export files then exit + $accountancyexport = new AccountancyExport($db); + + $error = 0; + $nbtotalofrecords = 0; + + // Open transaction to read lines to export, export them and update field date_export or date_validated + $db->begin(); + + /* The fast and low memory method to get and count full list converts the sql into a sql count */ + $sqlforcount = preg_replace('/^'.preg_quote($sqlfields, '/').'/', 'SELECT COUNT(*) as nbtotalofrecords', $sql); + $sqlforcount = preg_replace('/GROUP BY .*$/', '', $sqlforcount); + $resql = $db->query($sqlforcount); + if ($resql) { + $objforcount = $db->fetch_object($resql); + $nbtotalofrecords = $objforcount->nbtotalofrecords; + } else { + dol_print_error($db); + } + + $db->free($resql); + + //$sqlforexport = $sql; + //$sqlforexport .= $db->order($sortfield, $sortorder); + + + // TODO Call the fetchAll for a $limit and $offset + // Replace the fetchAll to get all ->line followed by call to ->export(). fetchAll() currently consumes too much memory on large export. + // Replace this with the query($sqlforexport) on a limited block and loop on each line to export them. + $limit = 0; + $offset = 0; + $result = $object->fetchAll($sortorder, $sortfield, $limit, $offset, $filter, 'AND', (!getDolGlobalString('ACCOUNTING_REEXPORT') ? 0 : 1)); if ($result < 0) { + $error++; setEventMessages($object->error, $object->errors, 'errors'); } else { - // Export files then exit - $accountancyexport = new AccountancyExport($db); - $formatexport = GETPOST('formatexport', 'int'); $notexportlettering = GETPOST('notexportlettering', 'alpha'); + if (!empty($notexportlettering)) { if (is_array($object->lines)) { foreach ($object->lines as $k => $movement) { @@ -602,9 +632,9 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements $withAttachment = !empty(trim(GETPOST('notifiedexportfull', 'alphanohtml'))) ? 1 : 0; // Output data on screen or download - $result = $accountancyexport->export($object->lines, $formatexport, $withAttachment); + //$result = $accountancyexport->export($object->lines, $formatexport, $withAttachment); + $result = $accountancyexport->export($object->lines, $formatexport, $withAttachment, 1, 1, 1); - $error = 0; if ($result < 0) { $error++; } else { @@ -612,11 +642,9 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements if (is_array($object->lines)) { dol_syslog("/accountancy/bookkeeping/list.php Function export_file Specify movements as exported", LOG_DEBUG); - // Specify as export : update field date_export or date_validated - $db->begin(); - // TODO Merge update for each line into one global using rowid IN (list of movement ids) foreach ($object->lines as $movement) { + // Upate the line to set date_export and/or date_validated (if not already set !) $now = dol_now(); $setfields = ''; @@ -640,23 +668,45 @@ if ($action == 'export_fileconfirm' && $user->hasRight('accounting', 'mouvements } } - if (!$error) { - $db->commit(); - } else { - $error++; + if ($error) { $accountancyexport->errors[] = $langs->trans('NotAllExportedMovementsCouldBeRecordedAsExportedOrValidated'); - $db->rollback(); } } } } - - if ($error) { - setEventMessages('', $accountancyexport->errors, 'errors'); - header('Location: '.$_SERVER['PHP_SELF']); - } - exit(); // download or show errors } + + if (!$error) { + $db->commit(); + + $downloadFilePath = $accountancyexport->generatedfiledata['downloadFilePath']; + $downloadFileMimeType = $accountancyexport->generatedfiledata['downloadFileMimeType']; + $downloadFileFullName = $accountancyexport->generatedfiledata['downloadFileFullName']; + + // No error, we can output the file + top_httphead($downloadFileMimeType); + + header('Content-Description: File Transfer'); + // Add MIME Content-Disposition from RFC 2183 (inline=automatically displayed, attachment=need user action to open) + $attachment = 1; + if ($attachment) { + header('Content-Disposition: attachment; filename="'.$downloadFileFullName.'"'); + } else { + header('Content-Disposition: inline; filename="'.$downloadFileFullName.'"'); + } + // Ajout directives pour resoudre bug IE + header('Cache-Control: Public, must-revalidate'); + header('Pragma: public'); + header('Content-Length: ' . dol_filesize($downloadFilePath)); + + readfileLowMemory($downloadFilePath); + } else { + $db->rollback(); + + setEventMessages('', $accountancyexport->errors, 'errors'); + header('Location: '.$_SERVER['PHP_SELF']); + } + exit(); // download or show errors } diff --git a/htdocs/accountancy/class/accountancyexport.class.php b/htdocs/accountancy/class/accountancyexport.class.php index fea26deae26..a240dca3387 100644 --- a/htdocs/accountancy/class/accountancyexport.class.php +++ b/htdocs/accountancy/class/accountancyexport.class.php @@ -81,17 +81,21 @@ class AccountancyExport public $errors = array(); /** - * - * @var string Separator + * @var string Separator */ public $separator = ''; /** - * - * @var string End of line + * @var string End of line */ public $end_line = ''; + /** + * @var array Generated file + */ + public $generatedfiledata = array(); + + /** * Constructor * @@ -195,7 +199,7 @@ class AccountancyExport */ public function getTypeConfig() { - global $conf, $langs; + global $langs; $exporttypes = array( 'param' => array( @@ -319,24 +323,25 @@ class AccountancyExport * @param int $formatexportset Id of export format * @param int $withAttachment [=0] Not add files * or 1 to have attached in an archive (ex : Quadratus) - Force output mode to write in a file (output mode = 1) - * @param int $downloadMode [=0] Direct download + * @param int $downloadMode [=0] Direct download. Deprecated. Always use value 1. * or 1 to download after writing files - Forced by default when use withAttachment = 1 * or -1 not to download files - * @param int $outputMode [=0] Print on screen - * or 1 to write in file and uses a temp directory - Forced by default when use withAttachment = 1 + * @param int $outputMode [=0] Print on screen. Deprecated. Always use value 1. + * or 1 to write in file and uses the temp directory - Forced by default when use withAttachment = 1 * or 2 to write in file a default export directory (accounting/export/) - * @return int Return integer <0 if KO, >0 OK + * @param int $noouput 0=old mode. Deprecated. Always use value 1. + * or 1=Do not output the file on stdout with this method. This must always be done by the main page, never by a method. + * @return int Return integer <0 if KO, >0 OK. The property ->generatedfile is also filled. */ - public function export(&$TData, $formatexportset, $withAttachment = 0, $downloadMode = 0, $outputMode = 0) + public function export(&$TData, $formatexportset, $withAttachment = 0, $downloadMode = 0, $outputMode = 0, $noouput = 0) { - global $conf, $langs; - global $search_date_end; // Used into /accountancy/tpl/export_journal.tpl.php + global $db, $conf, $langs; // The tpl file use $db + global $search_date_end; // Used into /accountancy/tpl/export_journal.tpl.php // Define name of file to save $filename = 'general_ledger-'.$this->getFormatCode($formatexportset); $type_export = 'general_ledger'; - global $db; // The tpl file use $db $completefilename = ''; $exportFile = null; $exportFileName = ''; @@ -369,17 +374,19 @@ class AccountancyExport // begin to print header for direct download top_httphead($mimetype, 1); } + include DOL_DOCUMENT_ROOT.'/accountancy/tpl/export_journal.tpl.php'; + if ($outputMode == 1 || $outputMode == 2) { if ($outputMode == 1) { - // uses temp directory by default to write files + // uses the temp directory by default to write files if (!empty($conf->accounting->multidir_temp[$conf->entity])) { $outputDir = $conf->accounting->multidir_temp[$conf->entity]; } else { $outputDir = $conf->accounting->dir_temp; } } else { - // uses default export directory "accounting/export" + // uses the default export directory "accounting/export" if (!empty($conf->accounting->multidir_output[$conf->entity])) { $outputDir = $conf->accounting->multidir_output[$conf->entity]; } else { @@ -503,7 +510,8 @@ class AccountancyExport break; } - // create and download export file or archive + + // Create and download export file or archive if ($outputMode == 1 || $outputMode == 2) { $error = 0; @@ -558,14 +566,19 @@ class AccountancyExport } // download export file or archive - if (!empty($downloadFileMimeType) && !empty($downloadFileFullName) && !empty($downloadFilePath)) { + if (!empty($downloadFileMimeType) && !empty($downloadFileFullName) && !empty($downloadFilePath) && empty($noouput)) { + // deprecated. We must not use this anymore, but have $noouput = 1 because HTTP header must be sent + // into main page not into a method. header('Content-Type: ' . $downloadFileMimeType); header('Content-Disposition: attachment; filename=' . $downloadFileFullName); header('Cache-Control: Public, must-revalidate'); header('Pragma: public'); header('Content-Length: ' . dol_filesize($downloadFilePath)); + readfileLowMemory($downloadFilePath); } + + $this->generatedfiledata = array('downloadFilePath' => $downloadFilePath, 'downloadFileMimeType' => $downloadFileMimeType, 'downloadFileFullName' => $downloadFileFullName); } } @@ -1343,6 +1356,7 @@ class AccountancyExport foreach ($objectLines as $line) { if ($line->debit == 0 && $line->credit == 0) { + //var_dump($line->id); //unset($array[$line]); } else { $date_creation = dol_print_date($line->date_creation, '%Y%m%d'); @@ -1492,6 +1506,7 @@ class AccountancyExport $tab[] = $attachmentFileName; $output = implode($separator, $tab).$end_line; + if ($exportFile) { fwrite($exportFile, $output); } else { diff --git a/htdocs/accountancy/class/api_accountancy.class.php b/htdocs/accountancy/class/api_accountancy.class.php index 815c7b69f05..8e6649b6e4b 100644 --- a/htdocs/accountancy/class/api_accountancy.class.php +++ b/htdocs/accountancy/class/api_accountancy.class.php @@ -219,6 +219,7 @@ class Accountancy extends DolibarrApi $filter['t.doc_date<='] = $doc_date_end; } + // @FIXME Critical bugged. Never use fetchAll without limit ! $result = $bookkeeping->fetchAll($sortorder, $sortfield, 0, 0, $filter, 'AND', $alreadyexport); if ($result < 0) { diff --git a/htdocs/document.php b/htdocs/document.php index 9efa37487b8..17589a636d9 100644 --- a/htdocs/document.php +++ b/htdocs/document.php @@ -295,6 +295,7 @@ if ($reshook < 0) { // Permissions are ok and file found, so we return it top_httphead($type); + header('Content-Description: File Transfer'); if ($encoding) { header('Content-Encoding: '.$encoding);