Work on import module

htdocs/imports/import.php

@@ -506,6 +506,8 @@ if ($step == 3 && $datatoimport)
 	print '<td>';
     $text=$objmodelimport->getDriverDesc($format);
     print $html->textwithpicto($objmodelimport->getDriverLabel($format),$text);
+    print '</td><td align="right" nowrap="nowrap" width="100"><a href="'.DOL_URL_ROOT.'/imports/emptyexample.php?format='.$key.'&datatoimport='.$datatoimport.'" target="_blank">'.$langs->trans("DownloadEmptyExample").'</a>';
+
 	print '</td></tr>';
 
 	print '</table>';
@@ -559,7 +561,14 @@ if ($step == 3 && $datatoimport)
 			$var=!$var;
 			print '<tr '.$bc[$var].'>';
 			print '<td width="16">'.img_mime($file).'</td>';
-			print '<td>'.$file.'</td>';
+			print '<td>';
+			$param='format='.$format.'&datatoimport='.$datatoimport;
+			$modulepart='import';
+			//$relativepath=$filetoimport;
+    		print '<a href="'.DOL_URL_ROOT.'/document.php?modulepart='.$modulepart.'&amp;file='.urlencode($relativepath).'&amp;step=3&amp;format='.$format.'&amp;datatoimport='.$datatoimport.'" target="_blank">';
+    		print $file;
+    		print '</a>';
+			print '</td>';
 			// Affiche taille fichier
 			print '<td align="right">'.dol_print_size(dol_filesize($newdir.'/'.$newfile)).'</td>';
 			// Affiche date fichier
@@ -610,7 +619,7 @@ if ($step == 4 && $datatoimport)
 		$i=1;
 		foreach($arrayrecord as $key => $val)
 		{
-			$fieldssource[$i]['example1']=dol_trunc($val,24);
+			$fieldssource[$i]['example1']=dol_trunc($val['val'],24);
 			$i++;
 		}
 		$obj->import_close_file();
@@ -717,14 +726,21 @@ if ($step == 4 && $datatoimport)
 
 	// File to import
 	print '<tr><td width="25%">'.$langs->trans("FileToImport").'</td>';
-	print '<td>'.$filetoimport.'</td></tr>';
+	print '<td>';
+	$param='format='.$format.'&datatoimport='.$datatoimport;
+	$modulepart='import';
+	//$relativepath=$filetoimport;
+    print '<a href="'.DOL_URL_ROOT.'/document.php?modulepart='.$modulepart.'&amp;file='.urlencode($relativepath).'&amp;step=4&amp;format='.$format.'&amp;datatoimport='.$datatoimport.'" target="_blank">';
+    print $filetoimport;
+    print '</a>';
+	print '</td></tr>';
 
 	print '</table>';
 	print '<br>'."\n";
 
 
-    // List of import models
+    // List of source fields
-    print '<!-- List of import models -->'."\n";
+    print '<!-- List of source fields -->'."\n";
     print '<form action="'.$_SERVER["PHP_SELF"].'" method="post">';
     print '<input type="hidden" name="token" value="'.$_SESSION['newtoken'].'">';
     print '<input type="hidden" name="action" value="select_model">';
@@ -1047,7 +1063,7 @@ if ($step == 5 && $datatoimport)
 		$i=1;
 		foreach($arrayrecord as $key => $val)
 		{
-			$fieldssource[$i]['example1']=dol_trunc($val,24);
+			$fieldssource[$i]['example1']=dol_trunc($val['val'],24);
 			$i++;
 		}
 		$obj->import_close_file();
@@ -1092,7 +1108,7 @@ if ($step == 5 && $datatoimport)
 	print '</td></tr>';
 
 	// Lot de donnees a importer
-	print '<tr><td width="25%">'.$langs->trans("DatasetToImport").'</td>';
+	print '<tr><td>'.$langs->trans("DatasetToImport").'</td>';
 	print '<td>';
 	print img_object($objimport->array_import_module[0]->getName(),$objimport->array_import_icon[0]).' ';
 	print $objimport->array_import_label[0];
@@ -1104,7 +1120,7 @@ if ($step == 5 && $datatoimport)
 	//print '<tr><td colspan="2"><b>'.$langs->trans("InformationOnSourceFile").'</b></td></tr>';
 
 	// Source file format
-	print '<tr><td width="40%">'.$langs->trans("SourceFileFormat").'</td>';
+	print '<tr><td width="25%">'.$langs->trans("SourceFileFormat").'</td>';
 	print '<td>';
     $text=$objmodelimport->getDriverDesc($format);
     print $html->textwithpicto($objmodelimport->getDriverLabel($format),$text);
@@ -1112,7 +1128,13 @@ if ($step == 5 && $datatoimport)
 
 	// File to import
 	print '<tr><td>'.$langs->trans("FileToImport").'</td>';
-	print '<td>'.$filetoimport.'</td></tr>';
+	print '<td>';
+	$modulepart='import';
+	//$relativepath=$filetoimport;
+    print '<a href="'.DOL_URL_ROOT.'/document.php?modulepart='.$modulepart.'&step=4'.$param.'" target="_blank">';
+    print $filetoimport;
+    print '</a>';
+    print '</td></tr>';
 
 	// Nb of fields
 	print '<tr><td>';
@@ -1135,10 +1157,11 @@ if ($step == 5 && $datatoimport)
 	//print '<tr><td colspan="2"><b>'.$langs->trans("InformationOnTargetTables").'</b></td></tr>';
 
 	// Tables imported
-	print '<tr><td width="40%">';
+	print '<tr><td width="25%">';
 	print $langs->trans("TablesTarget");
 	print '</td><td>';
 	$listtables=array();
+	$sort_array_match_file_to_database=$array_match_file_to_database;
 	foreach($array_match_file_to_database as $code=>$label)
 	{
 		//var_dump($fieldssource);
@@ -1150,6 +1173,7 @@ if ($step == 5 && $datatoimport)
 	if (sizeof($listtables))
 	{
 		$newval='';
+		//ksort($listtables);
 		foreach ($listtables as $val)
 		{
 			if ($newval) print ', ';
@@ -1177,8 +1201,10 @@ if ($step == 5 && $datatoimport)
 	$listfields=array();
 	$i=0;
 	//print 'fieldsource='.$fieldssource;
-	//var_dump($array_match_file_to_database);
+	$sort_array_match_file_to_database=$array_match_file_to_database;
-	foreach($array_match_file_to_database as $code=>$label)
+	ksort($sort_array_match_file_to_database);
+	//var_dump($sort_array_match_file_to_database);
+	foreach($sort_array_match_file_to_database as $code=>$label)
 	{
 		$i++;
 		//var_dump($fieldssource);
@@ -1202,7 +1228,7 @@ if ($step == 5 && $datatoimport)
 	print '<center>';
 	if ($user->rights->import->run)
 	{
-		print '<a class="butAction" href="'.DOL_URL_ROOT.'/imports/import.php?leftmenu=import&step=6&'.$param.'">'.$langs->trans("RunSimulateImportFile").'</a>';
+		print '<a class="butAction" href="'.DOL_URL_ROOT.'/imports/import.php?leftmenu=import&step=6'.$param.'">'.$langs->trans("RunSimulateImportFile").'</a>';
 	}
 	else
 	{
@@ -1243,7 +1269,7 @@ if ($step == 6 && $datatoimport)
 		$i=1;
 		foreach($arrayrecord as $key => $val)
 		{
-			$fieldssource[$i]['example1']=dol_trunc($val,24);
+			$fieldssource[$i]['example1']=dol_trunc($val['val'],24);
 			$i++;
 		}
 		$obj->import_close_file();
@@ -1312,7 +1338,13 @@ if ($step == 6 && $datatoimport)
 
 	// File to import
 	print '<tr><td>'.$langs->trans("FileToImport").'</td>';
-	print '<td>'.$filetoimport.'</td></tr>';
+	print '<td>';
+	$modulepart='import';
+	//$relativepath=$filetoimport;
+    print '<a href="'.DOL_URL_ROOT.'/document.php?modulepart='.$modulepart.'&step=4'.$param.'" target="_blank">';
+    print $filetoimport;
+    print '</a>';
+	print '</td></tr>';
 
 	// Nb of fields
 	print '<tr><td>';
@@ -1376,7 +1408,10 @@ if ($step == 6 && $datatoimport)
 	print $langs->trans("FieldsTarget").'</td><td>';
 	$listfields=array();
 	$i=0;
-	foreach($array_match_file_to_database as $code=>$label)
+	$sort_array_match_file_to_database=$array_match_file_to_database;
+	ksort($sort_array_match_file_to_database);
+	//var_dump($sort_array_match_file_to_database);
+	foreach($sort_array_match_file_to_database as $code=>$label)
 	{
 		$i++;
 		//var_dump($fieldssource);
@@ -1393,33 +1428,59 @@ if ($step == 6 && $datatoimport)
 	print '<br>';
 
 	// Launch import
-	$arrayofresult=array();
+	$arrayoferrors=array();
+	$arrayofwarnings=array();
+	$maxnboferrors=empty($conf->global->IMPORT_MAX_NB_OF_ERRORS)?100:$conf->global->IMPORT_MAX_NB_OF_ERRORS;
+	$maxnbofwarnings=empty($conf->global->IMPORT_MAX_NB_OF_WARNINGS)?100:$conf->global->IMPORT_MAX_NB_OF_WARNINGS;
+	$nboferrors=0;
+	$nbofwarnings=0;
+	
 	$db->begin();
 
+	//var_dump($array_match_file_to_database);
+	
 	// Open input file
-	$result=$obj->import_open_file($conf->import->dir_temp.'/'.$filetoimport,$langs);
+	$pathfile=$conf->import->dir_temp.'/'.$filetoimport;
+	$result=$obj->import_open_file($pathfile,$langs);
 	if ($result > 0)
 	{
-		$i=1;
+		$sourcelinenb=0;
 		// Loop on each input file record
 		while ($arrayrecord=$obj->import_read_record())
 		{
-			$i++;
+			$sourcelinenb++;
-			$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport);
+			$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,sizeof($fieldssource));
-			$arrayofresult[$result]++;
+			if (sizeof($obj->errors))
+			{
+				$arrayoferrors[$sourcelinenb]=$obj->errors;	
+			}
+			if (sizeof($obj->warnings))
+			{
+				$arrayofwarnings[$sourcelinenb]=$obj->warnings;	
+			}
 		}
 		// Close file
 		$obj->import_close_file();
 	}
+	else
+	{
+		print $langs->trans("ErrorFailedToOpenFile",$pathfile);
+	}
 
 	$db->rollback();	// We force rollback because this was just a simulation.
 
+	var_dump($arrayoferrors);
+	// Print result $arrayoferrors
+	foreach ($arrayoferrors as $key => $val)
+	{
+		//print $langs->trans("Result".$key)." ".$arrayofresult[$key].'<br>';
+	}
 
-	// Print $arrayofresult
+	$importid=dol_print_date(dol_now('tzserver'),'%Y%m%d%H%M%S');
-	// TODO
-
 
+	print '<br>';
 	print $langs->trans("NowClickToRunTheImport",$langs->transnoentitiesnoconv("RunImportFile")).'<br>';
+	print $langs->trans("DataLoadedWithId",$importid).'<br>';
 
 	print '</div>';
 
@@ -1460,7 +1521,7 @@ function show_elem($fieldssource,$i,$pos,$key,$var,$nostyle='')
 	print '<div style="padding: 0px 0px 0px 0px;" id="boxto_'.$pos.'">'."\n";
 
 	print '<table summary="boxtable'.$pos.'" width="100%" class="nobordernopadding">'."\n";
-	if ($pos && $pos > sizeof($fieldssource))	// NoFields
+	if ($pos && $pos > sizeof($fieldssource))	// No fields
 	{
 		print '<tr '.($nostyle?'':$bc[$var]).' height="20">';
 		print '<td class="nocellnopadding" width="16" style="font-weight: normal">';
@@ -1471,7 +1532,7 @@ function show_elem($fieldssource,$i,$pos,$key,$var,$nostyle='')
 		print '</td>';
 		print '</tr>';
 	}
-	elseif ($key == 'none')
+	elseif ($key == 'none')	// Empty line
 	{
 		print '<tr '.($nostyle?'':$bc[$var]).' height="20">';
 		print '<td class="nocellnopadding" width="16" style="font-weight: normal">';
@@ -1492,7 +1553,7 @@ function show_elem($fieldssource,$i,$pos,$key,$var,$nostyle='')
 		print '</td>';
 		print '<td style="font-weight: normal">';
 		print $langs->trans("Field").' '.$pos;
-		if (! empty($fieldssource[$pos]['example1'])) print ' (<i>'.$fieldssource[$pos]['example1'].'</i>)';
+		if (! empty($fieldssource[$pos]['example1'])) print ' (<i>'.htmlentities($fieldssource[$pos]['example1']).'</i>)';
 		print '</td>';
 		print '</tr>';
 	}

htdocs/includes/modules/import/import_csv.modules.php

@@ -35,6 +35,9 @@ require_once(DOL_DOCUMENT_ROOT ."/includes/modules/import/modules_import.php");
 class ImportCsv extends ModeleImports
 {
     var $id;
+    var $error;
+    var $errors=array();
+    
     var $label;
     var $extension;
     var $version;
@@ -192,6 +195,7 @@ class ImportCsv extends ModeleImports
 	/**
 	 * 	\brief		Return array of next record in input file.
 	 * 	\return		Array		Array of field values. Data are UTF8 encoded.
+	 * 							[0]=>(['val']=>val,['type']=>-1=null,0=blank,1=string)
 	 */
     function import_read_record()
     {
@@ -216,13 +220,29 @@ class ImportCsv extends ModeleImports
 			{
 		    	if (! empty($conf->global->IMPORT_CSV_FORCE_CHARSET))	// Forced charset
 		    	{
-		    		if (strtolower($conf->global->IMPORT_CSV_FORCE_CHARSET) == 'utf8') $newarrayres[$key]=$val;
+		    		if (strtolower($conf->global->IMPORT_CSV_FORCE_CHARSET) == 'utf8') 
-		    		else $newarrayres[$key]=utf8_encode($val);
+		    		{
+		    			$newarrayres[$key]['val']=$val;
+		    			$newarrayres[$key]['type']=1;
+		    		}
+		    		else
+		    		{
+		    			$newarrayres[$key]['val']=utf8_encode($val);
+		    			$newarrayres[$key]['type']=1;
+		    		}
 		    	}
 		    	else	// Autodetect format (UTF8 or ISO)
 		    	{
-					if (utf8_check($val)) $newarrayres[$key]=$val;
+					if (utf8_check($val)) 
-					else $newarrayres[$key]=utf8_encode($val);
+					{
+						$newarrayres[$key]['val']=$val;
+						$newarrayres[$key]['type']=1;
+					}
+					else 
+					{
+						$newarrayres[$key]['val']=utf8_encode($val);
+						$newarrayres[$key]['type']=1;
+					}
 		    	}
 			}
 
@@ -247,52 +267,120 @@ class ImportCsv extends ModeleImports
      * @param 	arrayrecord						Array of field values
      * @param	array_match_file_to_database
      * @param 	objimport
+     * @param	maxfields						Max number of fiels to use
      * @return	int								<0 if KO, >0 if OK
      */
-    function import_insert($arrayrecord,$array_match_file_to_database,$objimport)
+    function import_insert($arrayrecord,$array_match_file_to_database,$objimport,$maxfields)
     {
     	$error=0;
+    	$warning=0;
+    	$this->errors=array();
+    	$this->warnings=array();
     	
-		// For each table to insert, me make a separate insert
+    	dol_syslog("import_csv.modules maxfields=".$maxfields);
-		foreach($objimport->array_import_tables[0] as $alias=>$tablename)
+    	//print "X".$maxfields;
+    	
+		var_dump($array_match_file_to_database);
+		var_dump($arrayrecord);
+    	$sort_array_match_file_to_database=$array_match_file_to_database;
+    	ksort($sort_array_match_file_to_database);
+		//var_dump($sort_array_match_file_to_database);
+
+		if (sizeof($arrayrecord) == 0 ||
+			(sizeof($arrayrecord) == 1 && empty($arrayrecord[0]['val'])))
 		{
-			// Build sql request
+			print 'W';
-			$sql='';
+			$this->warnings[$warning]['lib']='Empty line';	
-			$listfields='';
+			$this->warnings[$warning]['type']='EMPTY';
-			$listvalues='';
+			$warning++;
-			foreach($array_match_file_to_database as $key => $val)
+		}
+		else
+		{
+			// For each table to insert, me make a separate insert
+			foreach($objimport->array_import_tables[0] as $alias=>$tablename)
 			{
-				if ($listfields) { $listfields.=', '; $listvalues.=', '; }
+				// Build sql request
-				$listfields.=$val;
+				$sql='';
-				$listvalues.='ee';
+				$listfields='';
-			}
+				$listvalues='';
-			if ($listfields)
+				$i=0;
-			{
+				$errorforthistable=0;
-				$sql='INSERT INTO '.$tablename.'('.$listfields.') VALUES('.$listvalues.')';
+				foreach($sort_array_match_file_to_database as $key => $val)
-			}
-
-			//print '> '.join(',',$arrayrecord);
-			print 'sql='.$sql;
-			print '<br>'."\n";
-
-			// Run insert request
-			if ($sql)
-			{
-				$resql=$this->db->query($sql);
-				if ($resql)
 				{
-					print '.';
+					if ($key <= $maxfields)
+					{
+						if ($listfields) { $listfields.=', '; $listvalues.=', '; }
+						$listfields.=$val;
+						$newval='';
+						if ($arrayrecord[($key-1)]['type'] < 0)
+						{
+							$listvalues.="null";
+						}
+						else if ($arrayrecord[($key-1)]['type'] == 0)
+						{
+							$listvalues.="''";
+						}
+						else if ($arrayrecord[($key-1)]['type'] > 0)
+						{
+							$newval=$arrayrecord[($key-1)]['val'];
+							$listvalues.="'".$arrayrecord[($key-1)]['val']."'";
+						}
+						
+						// Make some tests
+	
+						// Required field is ok
+						if (eregi('\*',$objimport->array_import_fields[0][$val]) && empty($newval))
+						{
+							$this->errors[$error]['lib']='ErrorMissingMandatoryValue field nb '.$key.' target='.$val;	
+							$this->errors[$error]['type']='NOTNULL';
+							$errorforthistable++;
+							$error++;
+						}
+						// Test format
+	
+						// Other tests
+						// ...
+						
+					}
+					$i++;
 				}
-				else
+				if (! $errorforthistable)
 				{
-					print 'E';
+					if ($listfields)
-					$this->error=$this->db->lasterror();
+					{
-					$error++;
+						$sql='INSERT INTO '.$tablename.'('.$listfields.') VALUES('.$listvalues.')';
+		    			dol_syslog("import_csv.modules sql=".$sql);
+						
+						//print '> '.join(',',$arrayrecord);
+						//print 'sql='.$sql;
+						//print '<br>'."\n";
+			
+						// Run insert request
+						if ($sql)
+						{
+							$resql=$this->db->query($sql);
+							if ($resql)
+							{
+								print '.';
+							}
+							else
+							{
+								print 'E';
+								$this->errors[$error]['lib']='ErrorSQL '.$this->db->lasterror();	
+								$this->errors[$error]['type']='SQL';
+								$error++;
+							}
+						}
+					}
+					else
+					{
+						dol_print_error('Ne doit pas arriver AAA');
+					}
 				}
 			}
 		}
 		
-		return $error?-$error:1;
+		return 1;
 	}
 
 }

htdocs/langs/en_US/exports.lang

@@ -9,7 +9,7 @@ ImportableDatas=Importable dataset
 SelectExportDataSet=Choose dataset you want to export...
 SelectImportDataSet=Choose dataset you want to import...
 SelectExportFields=Choose fields you want to export, or select a predefined export profil
-SelectImportFields=Choose source file fields you want to import and their target field in database by moving them up and down with anchor %s,<br>or select a predefined import profil:
+SelectImportFields=Choose source file fields you want to import and their target field in database by moving them up and down with anchor %s, or select a predefined import profil:
 NotImportedFields=Fields of source file not imported
 SaveExportModel=Save this export profile if you plan to reuse it later...
 SaveImportModel=Save this import profile if you plan to reuse it later...
@@ -82,3 +82,7 @@ SomeMandatoryFieldHaveNoSource=Some mandatory fields have no source from data fi
 InformationOnSourceFile=Informations on source file
 InformationOnTargetTables=Informations on target fields
 SelectAtLeastOneField=Switch at least one source field in the column of fields to export
+SelectFormat=Choose this import file format
+RunImportFile=Launch import file
+NowClickToRunTheImport=Check result of import simulation. If everything is ok, launch the definitive import.
+DataLoadedWithId=All data will be loaded with the following import id: <b>%s<b>

htdocs/langs/fr_FR/exports.lang

@@ -82,3 +82,7 @@ SomeMandatoryFieldHaveNoSource=Certains champs obligatoires n'ont pas de champ s
 InformationOnSourceFile=Informations sur le fichier source
 InformationOnTargetTables=Informations sur les champs cibles
 SelectAtLeastOneField=Basculer au moins un champ source dans la colonne des champs à exporter
+SelectFormat=Choisir ce format de fichier import
+RunImportFile=Lancer l'import en base
+NowClickToRunTheImport=Vérifiez le résultat de la simulation. Si tout est ok, lancer l'import définitif en base.
+DataLoadedWithId=Toutes les données seront importées avec l'id d'import suivant: <b>%s</b>

htdocs/lib/functions.lib.php

@@ -2641,8 +2641,9 @@ function dol_nl2br($stringtoencode,$nl2brmode=0)
 }
 
 /**
- *	\brief		This function is called to encode a string into a HTML string.
+ *	\brief		This function is called to encode a string into a HTML string but differs from htmlentities because
- * 				All entities except &,<,> are converted. This permits to encode special chars to entities with no double encoded for already encoded HTML strings.
+ * 				all entities but &,<,> are converted. This permits to encode special chars to entities with no double
+ *              encoding for already encoded HTML strings.
  * 				This function also remove last CR/BR.
  *	\param		stringtoencode		String to encode
  *	\param		nl2brmode			0=Adding br before \n, 1=Replacing \n by br (for use with FPDF writeHTMLCell function for example)