Merge branch 'node10' of gitea.invaders.stream:onja/bodacc into node10

src/assets/js/main.js

@@ -36,22 +36,6 @@ const sendRequest = (url, data) => {
       type: 'POST',
       data: data,
       success: function(data, textStatus, jqXHR) {
-        // Créez un lien de téléchargement et définissez ses attributs
-        const blob = new Blob([data], { type: 'text/csv' });
-        const url = window.URL.createObjectURL(blob);
-        const a = document.createElement('a');
-        let date = new Date();
-        // format date to YYYY-MM-DD HH:MM
-        date = date.toISOString().slice(0, 16).replace('T', ' ');
-        a.href = url;
-        a.download = `export-${date}.csv`; // Nom du fichier
-        document.body.appendChild(a);
-
-        // Cliquez sur le lien pour déclencher le téléchargement
-        a.click();
-
-        // Supprimez le lien du DOM
-        window.URL.revokeObjectURL(url);
         resolve({ message: 'Fichier généré' });
       },
       error: function(jqXHR, textStatus, errorThrown) {
@@ -78,6 +62,11 @@ const initSubmitForm = () => {
     $submitBtn.prop('disabled', true);
     $spinner.removeClass('d-none');
 
+    // reset result
+    const $form__result = $('#form__result');
+    $form__result.addClass('d-none');
+    $form__result.find('a').attr('href', '#').html('');
+
     if ( !$urlInput.val() ) {
       toastr.error('Veuillez saisir une URL');
       $submitBtn.prop('disabled', false);
@@ -102,6 +91,7 @@ const initSubmitForm = () => {
       data.columns.push($this.val());
     });
 
+    $urlInput.val('');    
 
     sendRequest($form.attr('action'), data)
     .then((response) => {

src/models/file.js

@@ -18,15 +18,6 @@ const { basedir } = require('../config/constants');
 const dest = path.join(basedir, 'public/csv');
 
 
-// Create a generateFilePath function witch returns a path with a filename and datetime
-function generateFilePath(filename) {
-  return {
-    filepath: path.join(dest, `${filename}-${Date.now()}.csv`),
-    generatedpath: path.join(dest, `${filename}-generated-${Date.now()}.csv`),
-  };
-}
-
-
 // Create a class File that extends EventEmitter
 class File {
 
@@ -38,7 +29,29 @@ class File {
     this.url = url;
   }
 
+  formatDateToCustomFormat(date) {  
+    // Obtenir les composantes de la date et de l'heure
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
+    const day = String(date.getDate()).padStart(2, '0');
+    const hours = String(date.getHours()).padStart(2, '0');
+    const minutes = String(date.getMinutes()).padStart(2, '0');
+    const seconds = String(date.getSeconds()).padStart(2, '0');
+  
+    // Créer la chaîne au format personnalisé
+    const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
+  
+    return formattedDate;
+  }
   
+  // Create a generateFilePath function witch returns a path with a filename and datetime
+  generateFilePath(filename) {
+    const date = new Date();
+    return {
+      filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
+      generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
+    };
+  }
 
   /**
    * Download a file from a url
@@ -65,7 +78,7 @@ class File {
   async download() {
     const url = URL.parse(this.url);
     this.filename = slugify(url.hostname, { lower: true });
-    const { filepath, generatedpath } = generateFilePath(this.filename);
+    const { filepath, generatedpath } = this.generateFilePath(this.filename);
     this.filepath = filepath;
     this.generatedpath = generatedpath;
 
@@ -139,13 +152,14 @@ class File {
 
   // create a parse method which read the file and return a stream
   parse(columns) {
-    const stream = new PassThrough();
+    return new Promise((resolve, reject) => {
       const fileStream = fs.createWriteStream(this.generatedpath);
 
       // check if columns is valid
       if (!columns || !columns.length) {
         // return Promise.reject(new Error('Invalid columns'));
         emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
+        reject(new Error('Invalid columns'));
         return false;
       }
 
@@ -181,14 +195,12 @@ class File {
         // Emit a parse.start event with the url and filepath
         emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
 
-      stream.write(result.join(';') + "\n");
         fileStream.write(result.join(';') + "\n");
       })
       .on('data', (row) => {
         // Emit a parse.data event with the url, filepath and data
         let result = this.processRow(row, columnsIndex, columnsFiltered);
         emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
-      stream.write(result.join(';') + "\n");
         fileStream.write(result.join(';') + "\n");
         count++;
       })
@@ -197,16 +209,16 @@ class File {
         emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
 
         fileStream.close();
+        reject(err);
         fs.unlink(this.generatedpath, () => {});
       })
       .on('end', () => {
         // Emit a parse.end event with the url and filepath
-      stream.end();
         fileStream.close();
+        resolve(this.generatedpath);
         emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
       });
-
-    return stream;
+    });
   }
 
   /**

src/routes/index.js

@@ -105,7 +105,7 @@ router.get('/', async function(req, res, next) {
   return res;
 });
 
-router.post('/', async function(req, res, next) {  
+router.post('/', function(req, res, next) {  
   // const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B';
 
   // get url from form
@@ -120,22 +120,16 @@ router.post('/', async function(req, res, next) {
     return res.status(500).send('Invalid columns');
   }
 
-  let stream = null;
-  try {
-    stream = await fileService.parseFromUrl(url, columns);
-  } catch (err) {
+  fileService.parseFromUrl(url, columns)
+  .then((filepath) => {
+    res.send({
+      success: true
+    })
+  })
+  .catch(err => {
     console.error('routes [/] error', err.message);
-  }
-
-  if ( !stream ) {
     return res.status(500).send('Invalid stream');
-  }
-
-  res.setHeader('Content-Disposition', 'attachment; filename="mon_fichier.csv"');
-  res.setHeader('Content-Type', 'text/csv; charset=utf-8');
-  stream.pipe(res);
-
-  // res.render('index', { title: 'Express' });
+  });
 });
 
 module.exports = router;

src/services/file.js

@@ -29,14 +29,15 @@ class FileService {
     const file = new File(url);
     const filepath = await file.download();
 
-    const stream = file.parse(columns);
-    if ( !stream ) {
-      emitter.emit('parseFromUrl.error', { url, columns, error: 'Invalid stream' });
-      return Promise.reject(new Error('Invalid stream'));
-    }
-
+    return file.parse(columns)
+    .then((filepath) => {
       emitter.emit('parseFromUrl.end', { url, columns, filepath });
-    return Promise.resolve(stream);
+      return filepath;
+    })
+    .catch((err) => {
+      emitter.emit('parseFromUrl.error', { url, columns, error: err.message });
+      return err;
+    });
   }
 
   /**

src/subscribers/consoleSubscriber.js

@@ -78,6 +78,16 @@ emitter.on('parse.error', ({ filepath, columns, error }) => {
 // emitter.on('parse.data', ({ filepath, columns, data, index }) => {
 //   log('parse.data', `Parsed ${filepath} with columns at index ${index}`);
 // });
+let processed = 0;
+let limit = 10000;
+emitter.on('parse.data', ({ filepath, columns, data, index }) => {
+  processed++;
+  
+  if ( processed == limit ) {
+    log('parse.data', `[${index.toLocaleString()} lignes] traités`);
+    processed = 0;
+  }
+});
 
 
 // Create a new listener for the deleteOldFiles.start event