fix: all task

package.json

@@ -10,25 +10,27 @@
     "watch": "webpack --watch --mode development"
   },
   "dependencies": {
+    "@popperjs/core": "^2.11.8",
+    "axios": "^0.19.2",
     "bootstrap": "^5.3.2",
-    "cookie-parser": "~1.4.4",
+    "cookie-parser": "^1.4.6",
     "csv-parser": "^3.0.0",
-    "debug": "~2.6.9",
+    "debug": "^4.3.4",
-    "ejs": "~2.6.1",
+    "ejs": "^3.1.9",
     "events": "^3.3.0",
-    "express": "~4.16.1",
+    "express": "^4.18.2",
-    "express-handlebars": "^7.1.2",
+    "express-handlebars": "^6.0.7",
     "handlebars": "^4.7.8",
-    "http-errors": "~1.6.3",
+    "http-errors": "^2.0.0",
     "jquery": "^3.7.1",
-    "morgan": "~1.9.1",
+    "morgan": "^1.10.0",
-    "popper.js": "^1.16.1",
     "slugify": "^1.6.6",
     "socket.io": "^4.7.2",
     "socket.io-client": "^4.7.2",
     "toastr": "^2.1.4"
   },
   "devDependencies": {
+    "@babel/core": "^7.23.2",
     "@babel/preset-env": "^7.22.20",
     "babel-core": "^6.26.3",
     "babel-loader": "^9.1.3",

public/js/main.min.js.LICENSE.txt

@@ -0,0 +1,16 @@
+/*!
+  * Bootstrap v5.3.2 (https://getbootstrap.com/)
+  * Copyright 2011-2023 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors)
+  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)
+  */
+
+/*!
+ * jQuery JavaScript Library v3.7.1
+ * https://jquery.com/
+ *
+ * Copyright OpenJS Foundation and other contributors
+ * Released under the MIT license
+ * https://jquery.org/license
+ *
+ * Date: 2023-08-28T13:37Z
+ */

src/app.js

@@ -8,6 +8,7 @@ const { basedir } = require('./config/constants');
 
 var indexRouter = require('./routes/index');
 const consoleSubscriber = require('./subscribers/consoleSubscriber');
+const mailerSubscriber = require('./subscribers/emailSubscriber');
 
 var app = express();
 
@@ -26,6 +27,8 @@ app.use(express.urlencoded({ extended: true }));
 app.use(cookieParser());
 app.use(express.static(path.join(basedir, 'public')));
 
+
+mailerSubscriber(app);
 app.use('/', indexRouter);
 
 // catch 404 and forward to error handler
@@ -41,7 +44,7 @@ app.use(function(err, req, res, next) {
 
   // render the error page
   res.status(err.status || 500);
-  res.render('error');
+  res.send('error');
 });
 
 module.exports = app;

src/assets/js/main.js

@@ -36,23 +36,7 @@ const sendRequest = (url, data) => {
       type: 'POST',
       data: data,
       success: function(data, textStatus, jqXHR) {
-        // Créez un lien de téléchargement et définissez ses attributs
+        resolve(data);
-        const blob = new Blob([data], { type: 'text/csv' });
-        const url = window.URL.createObjectURL(blob);
-        const a = document.createElement('a');
-        let date = new Date();
-        // format date to YYYY-MM-DD HH:MM
-        date = date.toISOString().slice(0, 16).replace('T', ' ');
-        a.href = url;
-        a.download = `export-${date}.csv`; // Nom du fichier
-        document.body.appendChild(a);
-
-        // Cliquez sur le lien pour déclencher le téléchargement
-        a.click();
-
-        // Supprimez le lien du DOM
-        window.URL.revokeObjectURL(url);
-        resolve({ message: 'Fichier généré' });
       },
       error: function(jqXHR, textStatus, errorThrown) {
         reject( new Error('Erreur lors de la requête Ajax : ' + jqXHR.responseText) );
@@ -78,6 +62,11 @@ const initSubmitForm = () => {
     $submitBtn.prop('disabled', true);
     $spinner.removeClass('d-none');
 
+    // reset result
+    const $form__result = $('#form__result');
+    $form__result.addClass('d-none');
+    $form__result.find('a').attr('href', '#').html('');
+
     if ( !$urlInput.val() ) {
       toastr.error('Veuillez saisir une URL');
       $submitBtn.prop('disabled', false);
@@ -101,13 +90,19 @@ const initSubmitForm = () => {
       const $this = $(this);
       data.columns.push($this.val());
     });
-    
+
+    $urlInput.val('');    
 
     sendRequest($form.attr('action'), data)
     .then((response) => {
-      toastr.success(response.message);
+      // toastr.success(response.message);
       $submitBtn.prop('disabled', false);
       $spinner.addClass('d-none');
+      if ( response.generated ) {
+        $form__result.removeClass('d-none');
+        $form__result.find('a').attr('href', `/csv/${response.generated}`)
+        $form__result.find('a').html(response.generated);
+      }
     })
     .catch((error) => {
       toastr.error(error.message);
@@ -137,14 +132,27 @@ const initSocket = () => {
     'download.progress',
     'parse.start',
     'parse.end',
-    'parse.error'
+    'parse.error',
+    'parse.data'
   ];
   for(let event of events) {
     socket.on(event, function(data) {
-      if (event.includes('error')) {
+      let eventName = data.event || event;
+      if (eventName.includes('error')) {
         toastr.error(data.message);
         return;
       }
+
+      if (eventName === 'parse.end') {
+        toastr.success(data.message, '', {
+          closeButton: true,
+          hideDuration: 0,
+          timeOut: 0,
+          extendedTimeOut: 0,
+        });
+        return;
+      }
+
       toastr.info(data.message);
     });
   }

src/assets/sass/main.scss

@@ -53,4 +53,16 @@
   &__input {
     display: none;
   }
-}
+}
+
+/* CSS pour personnaliser le helper */
+// .ui-draggable-dragging {
+//   opacity: 0.7; /* Opacité initiale */
+//   transition: opacity 0.3s ease-in-out; /* Ajouter une transition d'opacité */
+//   border: 2px dashed #ccc; /* Ajouter une bordure en pointillés */
+//   box-shadow: 0 0 5px rgba(0, 0, 0, 0.3); /* Ajouter une ombre légère */
+//   background-color: #f0f0f0; /* Changer la couleur de fond */
+//   z-index: 1000; /* Assurer que l'élément est au-dessus des autres éléments */
+  
+// }
+ 

src/config/params.json

@@ -0,0 +1,17 @@
+{
+  "sendgrid": {
+    "to": [
+      {
+        "email": "onja@blastream.com"
+      },
+      {
+        "email": "onja.asmad@gmail.com"
+      }
+    ],
+    "from": "contact@blastream.com",
+    "fromName": "Blastream",
+    "uri": "https://api.sendgrid.com/v3/mail/send",
+    "application_key": "YABUthwqTla79YQbR7gToQ",
+    "application_secret": "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"
+  }
+}

src/models/file.js

@@ -5,7 +5,7 @@ const fs = require('fs');
 const util = require('util');
 const http = require('http');
 const https = require('https');
-const exec = util.promisify(require('child_process').exec);
+const exec = util.promisify(require('child_process').exec); 
 
 const csvParser = require('csv-parser');
 const { PassThrough } = require('stream');
@@ -18,15 +18,6 @@ const { basedir } = require('../config/constants');
 const dest = path.join(basedir, 'public/csv');
 
 
-// Create a generateFilePath function witch returns a path with a filename and datetime
-function generateFilePath(filename) {
-  return {
-    filepath: path.join(dest, `${filename}-${Date.now()}.csv`),
-    generatedpath: path.join('csv', `${filename}-generated-${Date.now()}.csv`),
-  };
-}
-
-
 // Create a class File that extends EventEmitter
 class File {
 
@@ -37,8 +28,47 @@ class File {
   constructor(url) {
     this.url = url;
   }
+
+  formatDateToCustomFormat(date) {  
+    // Obtenir les composantes de la date et de l'heure
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
+    const day = String(date.getDate()).padStart(2, '0');
+    const hours = String(date.getHours()).padStart(2, '0');
+    const minutes = String(date.getMinutes()).padStart(2, '0');
+    const seconds = String(date.getSeconds()).padStart(2, '0');
+  
+    // Créer la chaîne au format personnalisé
+    const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
   
+    return formattedDate;
+  }
+  
+  // Create a generateFilePath function witch returns a path with a filename and datetime
+  generateFilePath(filename) {
+    const date = new Date();
+    return {
+      filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
+      generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
+    };
+  }
 
+  /**
+   * Download a file from a url
+   * @returns {Promise<string>} filepath
+   */
+  async fakeDownload() {
+    const url = URL.parse(this.url);
+    this.filename = slugify(url.hostname, { lower: true });
+    const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
+    const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
+    this.filepath = filepath;
+    this.generatedpath = generatedpath;
+    return new Promise((resolve, reject) => {
+      resolve(filepath);
+    });
+  }
+  
 
   /**
    * Download a file from a url
@@ -47,11 +77,11 @@ class File {
   async download() {
     const url = URL.parse(this.url);
     this.filename = slugify(url.hostname, { lower: true });
-    const { filepath, generatedpath } = generateFilePath(this.filename);
+    const { filepath, generatedpath } = this.generateFilePath(this.filename);  
     this.filepath = filepath;
     this.generatedpath = generatedpath;
 
-    const file = fs.createWriteStream(this.filepath);
+    const file = fs.createWriteStream(this.filepath); 
     const request = url.protocol === 'https:' ? https : http;
 
     // Emit a download.start event with the url and filepath
@@ -120,13 +150,15 @@ class File {
 
 
   // create a parse method which read the file and return a stream
-  parse(columns) {
+parse(columns) {
-    const stream = new PassThrough();
+  return new Promise((resolve, reject) => {
+    const fileStream = fs.createWriteStream(this.generatedpath, { encoding: 'utf8' }); // Specify UTF-8 encoding for write stream
 
     // check if columns is valid
     if (!columns || !columns.length) {
       // return Promise.reject(new Error('Invalid columns'));
       emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
+      reject(new Error('Invalid columns'));
       return false;
     }
 
@@ -135,54 +167,64 @@ class File {
     for (let column of columns) {
       columnsIndex[column] = {
         exist: false,
-        main: (String(column)).split('.')[0],
+        main: String(column).split('.')[0],
         value: column,
-        rest: (String(column)).split('.').slice(1).join('.'),
+        rest: String(column).split('.').slice(1).join('.'),
-        last: (String(column)).split('.').pop(),
+        last: String(column).split('.').pop(),
       };
     }
 
     const columnsFiltered = [];
 
     let count = 1;
-    fs.createReadStream(this.filepath)
+    fs.createReadStream(this.filepath, { encoding: 'utf8' }) // Specify UTF-8 encoding for read stream
-    .pipe(csvParser({ separator: ';' }))
+      .pipe(csvParser({ separator: ';' }))
-    .on('headers', (headers) => {
+      .on('headers', (headers) => {
-      headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
+        headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
-      
+
-      const result = [];
+        const result = []; 
-      for (let key in columnsIndex) {
+        for (let key in columnsIndex) {
-        columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
+          columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
-        if ( columnsIndex[key].exist ) {
+          if (columnsIndex[key].exist) {
-          columnsFiltered.push(columnsIndex[key].value);
+            columnsFiltered.push(columnsIndex[key].value);
-          result.push(columnsIndex[key].last);
+            if(columnsIndex[key].main ==='listeprecedentexploitant'){
+              result.push(`Prec Exp ${columnsIndex[key].last}`)
+            }else if(columnsIndex[key].main ==='listeprecedentproprietaire'){
+              result.push(`Prec Prop ${columnsIndex[key].last}`)
+            }else{
+              result.push(columnsIndex[key].last);
+            }
+          }
         }
-      }
-      
-      // Emit a parse.start event with the url and filepath
-      emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
-
-      stream.write(result.join(';') + "\n");
-    })
-    .on('data', (row) => {
-      // Emit a parse.data event with the url, filepath and data
-      let result = this.processRow(row, columnsIndex, columnsFiltered);
-      emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
-      stream.write(result.join(';') + "\n");
-      count++;
-    })
-    .on('error', (err) => {
-      // Emit a parse.error event with the error
-      emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
-    })
-    .on('end', () => {
-      // Emit a parse.end event with the url and filepath
-      emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1 });
-      stream.end();
-    });
 
-    return stream;
+        // Emit a parse.start event with the url and filepath
-  }
+        emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result });
+        fileStream.write(result.join(';') + "\n");
+      })
+      .on('data', (row) => {
+        // Emit a parse.data event with the url, filepath and data
+        let result = this.processRow(row, columnsIndex, columnsFiltered);
+        emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
+        fileStream.write(result.join(';') + "\n");
+        count++;
+      })
+      .on('error', (err) => {
+        // Emit a parse.error event with the error
+        emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
+
+        fileStream.close();
+        reject(err);
+        fs.unlink(this.generatedpath, () => { });
+      })
+      .on('end', () => {
+        // Emit a parse.end event with the url and filepath
+        fileStream.close();
+        emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
+        resolve(path.basename(this.generatedpath));
+      });
+  });
+}
+
 
   /**
    * Generate a new file with the columns
@@ -223,8 +265,8 @@ class File {
 
     for (let key in row) {
       try {
-        if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) {
+        if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) { 
-          row[key] = JSON.parse(row[key]);
+           row[key] = JSON.parse(row[key]); 
         }
         
         row[key.trim()] = row[key]
@@ -240,7 +282,7 @@ class File {
       }
 
       const column = columnsIndex[key];
-      const item = row[column.main] || '';
+      const item = row[column.main] || ''; 
 
       if ( column.primary === column.value ) {
         result.push(item);
@@ -251,9 +293,8 @@ class File {
           result.push(item);
         }
       }
-    }
+    }  
-
+    return result.map((res)=>res.replace(/;/g, ','));
-    return result;
   }
 }
 

src/routes/index.js

@@ -1,141 +1,148 @@
-var express = require('express');
+var express = require("express");
 var router = express.Router();
 
-const FileService = require('../services/file');
+const FileService = require("../services/file");
-const TemplateService = require('../services/template');
+const TemplateService = require("../services/template");
 
 const templateService = new TemplateService();
 const fileService = new FileService();
+let initialSelectedColumns = [
+  "region_code",
+  "region_nom_officiel",
+  "numerodepartement",
+  "departement_nom_officiel",
+  "cp",
+  "listepersonnes.personne.typePersonne",
+  "listepersonnes.personne.formeJuridique",
+  "listepersonnes.personne.denomination",
+  "listepersonnes.personne.numeroImmatriculation.codeRCS",
+  "listepersonnes.personne.numeroImmatriculation.numeroIdentification",
+  "listepersonnes.personne.nom",
+  "listepersonnes.personne.nomCommercial",
+  "listepersonnes.personne.prenom",
+  "listeetablissements.etablissement.activite",
+  "listeetablissements.etablissement.adresse.complGeographique",
+  "listeetablissements.etablissement.adresse.ville",
+  "acte.dateCommencementActivite",
+  "acte.vente.publiciteLegale.date",
+  "acte.descriptif",
+  "acte.vente.categorieVente",
+];
 
+const data = {
+  form: {
+    url: {
+      value:
+        "https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&refine=numerodepartement%3A%2275%22&refine=typeavis_lib%3A%22Avis%20d%E2%80%99annulation%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B",
+    },
+    columns: {
+      options: [
+        "id",
+        "publicationavis",
+        "publicationavis_facette",
+        "parution",
+        "dateparution",
+        "numeroannonce",
+        "typeavis",
+        "typeavis_lib",
+        "familleavis",
+        "familleavis_lib",
+        "tribunal",
+        "commercant",
+        "ville",
+        "registre",
+        "pdf_parution_subfolder",
+        "ispdf_unitaire",
+        "listepersonnes.personne.administration",
+        "listepersonnes.personne.numeroImmatriculation.nomGreffeImmat",
+        "listepersonnes.personne.capital.devise",
+        "listepersonnes.personne.capital.montantCapital",
+        "listepersonnes.personne.adresseSiegeSocial.ville",
+        "listepersonnes.personne.adresseSiegeSocial.codePostal",
+        "listepersonnes.personne.adresseSiegeSocial.pays",
+        "listepersonnes.personne.adresseSiegeSocial.typeVoie",
+        "listepersonnes.personne.adresseSiegeSocial.numeroVoie",
+        "listepersonnes.personne.adresseSiegeSocial.nomVoie",
+        "listepersonnes.personne.adresseSiegeSocial.complGeographique",
+        "listeetablissements.etablissement.qualiteEtablissement",
+        "listeetablissements.etablissement.adresse.codePostal",
+        "listeetablissements.etablissement.adresse.pays",
+        "listeetablissements.etablissement.adresse.typeVoie",
+        "listeetablissements.etablissement.adresse.numeroVoie",
+        "listeetablissements.etablissement.adresse.nomVoie",
+        "jugement",
+        "acte.dateImmatriculation",
+        "acte.vente.publiciteLegale.titre",
+        "acte.vente.opposition",
+        "acte.vente.dateEffet",
+        "modificationsgenerales",
+        "radiationaurcs",
+        "depot",
+        "listeprecedentexploitant.personne.typePersonne",
+        "listeprecedentexploitant.personne.numeroImmatriculation.codeRCS",
+        "listeprecedentexploitant.personne.numeroImmatriculation.numeroIdentification",
+        "listeprecedentexploitant.personne.numeroImmatriculation.nomGreffeImmat",
+        "listeprecedentexploitant.personne.denomination",
+        "listeprecedentproprietaire.personne.typePersonne",
+        "listeprecedentproprietaire.personne.numeroImmatriculation.codeRCS",
+        "listeprecedentproprietaire.personne.numeroImmatriculation.numeroIdentification",
+        "listeprecedentproprietaire.personne.numeroImmatriculation.nomGreffeImmat",
+        "listeprecedentproprietaire.personne.denomination",
+        "divers",
+        "parutionavisprecedent.nomPublication",
+        "parutionavisprecedent.dateParution",
+        "parutionavisprecedent.numeroParution",
+        "parutionavisprecedent.numeroAnnonce",
+      ],
+      selected: initialSelectedColumns,
+    },
+  },
+};
 /* GET home page. */
-router.get('/', async function(req, res, next) {
+router.get("/", async function (req, res, next) {
   fileService.checkLastOperationDate();
 
-  res.setHeader('Content-Type', 'text/html; charset=utf-8');
+  res.setHeader("Content-Type", "text/html; charset=utf-8");
 
-  const data = {
+  res.send(templateService.renderPage("index", data));
-    form: {
-      url: {
-        value: 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&refine=numerodepartement%3A%2275%22&refine=typeavis_lib%3A%22Avis%20d%E2%80%99annulation%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B'
-      },
-      columns: {
-        options: [
-          'id',
-          'publicationavis',
-          'publicationavis_facette',
-          'parution',
-          'dateparution',
-          'numeroannonce',
-          'typeavis',
-          'typeavis_lib',
-          'familleavis',
-          'familleavis_lib',
-          'tribunal',
-          'commercant',
-          'ville',
-          'registre',
-          'pdf_parution_subfolder',
-          'ispdf_unitaire',
-          'listepersonnes.personne.administration',
-          'listepersonnes.personne.numeroImmatriculation.nomGreffeImmat',
-          'listepersonnes.personne.capital.devise',
-          'listepersonnes.personne.capital.montantCapital',
-          'listepersonnes.personne.adresseSiegeSocial.ville',
-          'listepersonnes.personne.adresseSiegeSocial.codePostal',
-          'listepersonnes.personne.adresseSiegeSocial.pays',
-          'listepersonnes.personne.adresseSiegeSocial.typeVoie',
-          'listepersonnes.personne.adresseSiegeSocial.numeroVoie',
-          'listepersonnes.personne.adresseSiegeSocial.nomVoie',
-          'listepersonnes.personne.adresseSiegeSocial.complGeographique',
-          'listeetablissements.etablissement.qualiteEtablissement',
-          'listeetablissements.etablissement.adresse.codePostal',
-          'listeetablissements.etablissement.adresse.pays',
-          'listeetablissements.etablissement.adresse.typeVoie',
-          'listeetablissements.etablissement.adresse.numeroVoie',
-          'listeetablissements.etablissement.adresse.nomVoie',
-          'jugement',
-          'acte.vente.publiciteLegale.titre',
-          'acte.vente.opposition',
-          'modificationsgenerales',
-          'radiationaurcs',
-          'depot',
-          'listeprecedentexploitant.personne.typePersonne',
-          'listeprecedentexploitant.personne.numeroImmatriculation.codeRCS',
-          'listeprecedentexploitant.personne.numeroImmatriculation.numeroIdentification',
-          'listeprecedentexploitant.personne.numeroImmatriculation.nomGreffeImmat',
-          'listeprecedentexploitant.personne.denomination',
-          'listeprecedentproprietaire.personne.typePersonne',
-          'listeprecedentproprietaire.personne.numeroImmatriculation.codeRCS',
-          'listeprecedentproprietaire.personne.numeroImmatriculation.numeroIdentification',
-          'listeprecedentproprietaire.personne.numeroImmatriculation.nomGreffeImmat',
-          'listeprecedentproprietaire.personne.denomination',
-          'divers',
-          'parutionavisprecedent.nomPublication',
-          'parutionavisprecedent.dateParution',
-          'parutionavisprecedent.numeroParution',
-          'parutionavisprecedent.numeroAnnonce',
-        ],
-        selected: [
-          'region_code',
-          'region_nom_officiel',
-          'numerodepartement',
-          'departement_nom_officiel',
-          'cp',
-          'listepersonnes.personne.typePersonne',
-          'listepersonnes.personne.formeJuridique',
-          'listepersonnes.personne.denomination',
-          'listepersonnes.personne.numeroImmatriculation.codeRCS',
-          'listepersonnes.personne.numeroImmatriculation.numeroIdentification',
-          'listepersonnes.personne.nom',
-          'listepersonnes.personne.nomCommercial',
-          'listepersonnes.personne.prenom',
-          'listeetablissements.etablissement.activite',
-          'listeetablissements.etablissement.adresse.complGeographique',
-          'listeetablissements.etablissement.adresse.ville',
-          'acte.dateCommencementActivite',
-          'acte.vente.publiciteLegale.date',
-          'acte.descriptif',
-          'acte.vente.categorieVente',
-        ]
-      }
-    }
-  };
-
-  res.send(templateService.renderPage('index', data));
   return res;
 });
 
-router.post('/', async function(req, res, next) {  
+// Logic to handle the drag and drop update
+router.post("/update-columns-order", async function (req, res, next) {
+  let newColumnsOrder = req.body.columnsOrder;
+  data.form.columns.selected = newColumnsOrder;
+  // Handle saving the new column order to a database or file if necessary
+  res.send("Columns order updated successfully");
+});
+
+router.post("/", function (req, res, next) {
   // const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B';
 
   // get url from form
   // get columns from form
-  const url = req.body.url || '';
+  const url = req.body.url || "";
   const columns = req.body.columns || [];
 
-  if ( !url ) {
+  if (!url) {
-    return res.status(500).send('Invalid url');
+    return res.status(500).send("Invalid url");
-  }
-  if ( !columns ) {
-    return res.status(500).send('Invalid columns');
-  }
-
-  let stream = null;
-  try {
-    stream = await fileService.parseFromUrl(url, columns);
-  } catch (err) {
-    console.error('routes [/] error', err.message);
   }
-
+  if (!columns) {
-  if ( !stream ) {
+    return res.status(500).send("Invalid columns");
-    return res.status(500).send('Invalid stream');
   }
 
-  res.setHeader('Content-Disposition', 'attachment; filename="mon_fichier.csv"');
+  fileService
-  res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+    .parseFromUrl(url, columns)
-  stream.pipe(res);
+    .then((filename) => {
-
+      res.send({
-  // res.render('index', { title: 'Express' });
+        generated: filename,
+        message: "Fichier généré avec success",
+      });
+    })
+    .catch((err) => {
+      console.error("routes [/] error", err.message);
+      return res.status(500).send("Invalid stream");
+    });
 });
 
 module.exports = router;

src/services/file.js

@@ -1,9 +1,9 @@
 const csv = require('csv-parser');
 const fs = require('fs');
 const path = require('path');
-const File = require('../models/File');
+const File = require('../models/file.js');
-const emitter = require('./eventEmitter');
+const emitter = require('./eventEmitter.js');
-const { basedir } = require('../config/constants');
+const { basedir } = require('../config/constants.js');
 const { emit } = require('process');
 
 // Create a class FileService that extends EventEmitter
@@ -24,19 +24,21 @@ class FileService {
       emitter.emit('parseFromUrl.error', { url, columns, error: 'Invalid columns' });
       return Promise.reject(new Error('Invalid columns'));
     }
-
+    const directoryPath = path.join(basedir, 'public/csv');
+    if(!fs.existsSync(directoryPath)) fs.mkdirSync(directoryPath);
     // Create a new File instance
     const file = new File(url);
     const filepath = await file.download();
 
-    const stream = file.parse(columns);
+    return file.parse(columns)
-    if ( !stream ) {
+    .then((filename) => {
-      emitter.emit('parseFromUrl.error', { url, columns, error: 'Invalid stream' });
+      emitter.emit('parseFromUrl.end', { url, columns, filename });
-      return Promise.reject(new Error('Invalid stream'));
+      return filename;
-    }
+    })
-
+    .catch((err) => {
-    emitter.emit('parseFromUrl.end', { url, columns, filepath });
+      emitter.emit('parseFromUrl.error', { url, columns, error: err.message });
-    return Promise.resolve(stream);
+      return err;
+    });
   }
 
   /**
@@ -46,7 +48,7 @@ class FileService {
   deleteOldFiles() {    
     const directoryPath = path.join(basedir, 'public/csv');
     const files = fs.readdirSync(directoryPath);
-    const oneDay = 24 * 60 * 60 * 1000; // Un jour en millisecondes
+    const oneDay = 48 * 60 * 60 * 1000; // Un jour en millisecondes
     const currentDate = new Date();
     
     emitter.emit('deleteOldFiles.start', { files, directoryPath });
@@ -75,15 +77,16 @@ class FileService {
     emitter.emit('checkLastOperationDate.start', { dateFilePath });
 
     try {
-      const oneDay = 24 * 60 * 60 * 1000; // Un jour en millisecondes
+      const oneDay = 5 * 24 * 60 * 60 * 1000; // 5 jour en millisecondes
+      const fileDate = 6 * 24 * 60 * 60 * 1000; // 6 jour en millisecondes
       const currentDate = new Date();
 
 
       // Check if file exists and create it if not with a default date of 48 hours ago
       if (!fs.existsSync(dateFilePath)) {
         emitter.emit('checkLastOperationDate.created', { dateFilePath });
-        const twodaysdate = new Date(currentDate - (2*oneDay));
+        const defaultFiledate = new Date(currentDate - (fileDate)); // 6 jours avant la date actuelle
-        fs.writeFileSync(dateFilePath, twodaysdate.toISOString(), 'utf-8');
+        fs.writeFileSync(dateFilePath, defaultFiledate.toISOString(), 'utf-8');
       }
 
       const lastOperationDate = fs.readFileSync(dateFilePath, 'utf-8');

src/services/template.js

@@ -74,6 +74,26 @@ class TemplateService {
 
     return layoutTemplate;
   }
+
+
+  renderEmail(name, data = {}, params = {}) {
+    // Get the page template
+    const pageTemplate = this.render(`emails/${name}`, data, params);
+
+    if ( !pageTemplate ) {
+      return '';
+    }
+    
+
+    // Get the layout template
+    const layoutTemplate = this.render(`layout/email`, {...data, ...{ body: pageTemplate }}, params);
+
+    if ( !layoutTemplate ) {
+      return '';
+    }
+
+    return layoutTemplate;
+  }
   
 }
 

src/subscribers/consoleSubscriber.js

@@ -3,7 +3,7 @@ const emitter = require('../services/eventEmitter');
 
 // Create a log function that logs the event name and the data
 const log = (event, data) => {
-  console.log(`----- Event: ${event} ----- `);
+  console.log(`----- [${(new Date()).toLocaleString()}] Event: ${event} ----- `);
   console.log(data);
   console.log('-----');
 };
@@ -44,9 +44,9 @@ emitter.on('parseFromUrl.start', ({ url, columns }) => {
 });
 
 // Create a new listener for the parseFromUrl.end event
-emitter.on('parseFromUrl.end', ({ url, columns, filepath }) => {
+emitter.on('parseFromUrl.end', ({ url, columns, filename }) => {
-  // log('parseFromUrl.end', `Parsed ${url} with columns ${columns.join(', ')} to ${filepath}`);
+  // log('parseFromUrl.end', `Parsed ${url} with columns ${columns.join(', ')} to ${filename}`);
-  log('parseFromUrl.end', `Parsed with columns to ${filepath}`);
+  log('parseFromUrl.end', `Parsed with columns to ${filename}`);
 });
 
 // Create a new listener for the parseFromUrl.error event
@@ -78,6 +78,16 @@ emitter.on('parse.error', ({ filepath, columns, error }) => {
 // emitter.on('parse.data', ({ filepath, columns, data, index }) => {
 //   log('parse.data', `Parsed ${filepath} with columns at index ${index}`);
 // });
+let processed = 0;
+let limit = 10000;
+emitter.on('parse.data', ({ filepath, columns, data, index }) => {
+  processed++;
+  
+  if ( processed == limit ) {
+    log('parse.data', `[${index.toLocaleString()} lignes] traités`);
+    processed = 0;
+  }
+});
 
 
 // Create a new listener for the deleteOldFiles.start event
@@ -121,3 +131,27 @@ emitter.on('checkLastOperationDate.updated', ({ dateFilePath, date }) => {
 emitter.on('checkLastOperationDate.skipped', ({ dateFilePath, date }) => {
   log('checkLastOperationDate.skipped', `Skipped last operation date - Date: ${date}`);
 });
+
+
+
+
+
+
+emitter.on('mailer.parse.end.success', ({ response }) => {
+  log('mailer.parse.end.success', response);
+});
+emitter.on('mailer.parse.end.error', ({ error }) => {
+  log('mailer.parse.end.error', error);
+});
+emitter.on('mailer.parse.error.success', ({ response }) => {
+  log('mailer.parse.error.success', response);
+});
+emitter.on('mailer.parse.error.error', ({ error }) => {
+  log('mailer.parse.error.error', error);
+});
+emitter.on('mailer.parseFromUrl.error.success', ({ response }) => {
+  log('mailer.parseFromUrl.error.success', response);
+});
+emitter.on('mailer.parseFromUrl.error.error', ({ error }) => {
+  log('mailer.parseFromUrl.error.error', error);
+});

src/subscribers/emailSubscriber.js

@@ -0,0 +1,129 @@
+const emitter = require("../services/eventEmitter");
+const axios = require("axios");
+
+const params = require("../config/params");
+const TemplateService = require("../services/template");
+const templateService = new TemplateService();
+
+const errorResponse = (error) => {
+  if (error.response) {
+    return error.response.status + " - " + error.response.statusText;
+  } else if (error.request) {
+    // The request was made but no response was received
+    // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
+    // http.ClientRequest in node.js
+    return error.request;
+  } else {
+    return error.message;
+  }
+};
+
+module.exports = (app) => {
+  // Get app url
+  let host = "";
+  app.use((req, res, next) => {
+    console.log("--- APP_URL ---", req.protocol + "://" + req.get("host"));
+    host = req.protocol + "://" + req.get("host");
+    next();
+  });
+
+  // Check if sendgrid params exists
+  if (!params.sendgrid) {
+    console.log("--- No sendgrid params ---");
+    return;
+  }
+  console.log("--- sendgrid params exists ---");
+
+  const sendgridParams = params.sendgrid;
+  const defaultParams = {
+    url: sendgridParams.uri,
+    headers: {
+      Authorization: "Bearer " + sendgridParams.application_secret,
+      "Content-Type": "application/json",
+    },
+  };
+
+  const sendEmail = async (to, subject, message) => {
+    const data = {
+      personalizations: [
+        {
+          to: to,
+          subject: subject,
+        },
+      ],
+      content: [
+        {
+          type: "text/html",
+          value: message,
+        },
+      ],
+      from: {
+        email: sendgridParams.from,
+        name: sendgridParams.fromName,
+      },
+    };
+return null
+  //  const response = await axios.post(defaultParams.url, data, {
+  //    headers: defaultParams.headers,
+  //  });
+  //  return response;
+  };
+
+  // Create a new listener for the parse.end event
+  emitter.on("parse.end", async ({ url, columns, count, generated }) => {
+    console.log("HOST", host);
+    try {
+      const message = templateService.renderEmail("parse-success", {
+        host,
+        generated,
+        count,
+        columns,
+        url,
+      });
+      const response = await sendEmail(
+        sendgridParams.to,
+        "BODACC - Traitement terminé",
+        message
+      );
+      emitter.emit("mailer.parse.end.success", { response: response.data });
+    } catch (error) {
+      emitter.emit("mailer.parse.end.error", { error: errorResponse(error) });
+    }
+  });
+
+  // Create a new listener for the parse.error event
+  emitter.on("parse.error", async ({ filepath, columns, error }) => {
+    try {
+      const reponse = await sendEmail(
+        sendgridParams.to,
+        "BODACC - Erreur traitement",
+        "<p>Erreur traitement</p><br /><p>Voici le message d'erreur: " +
+          error +
+          "</p>"
+      );
+      emitter.emit("mailer.parse.error.success", { response: response });
+    } catch (error) {
+      emitter.emit("mailer.parse.error.error", { error: errorResponse(error) });
+    }
+  });
+
+  // Create a new listener for the parseFromUrl.error event
+  emitter.on("parseFromUrl.error", async ({ url, columns, error }) => {
+    try {
+      const response = await sendEmail(
+        sendgridParams.to,
+        "BODACC - Erreur génération fichier",
+        "<p>Erreur génération fichier</p><br /><p>Voici le message d'erreur: " +
+          error +
+          "</p>"
+      );
+      emitter.emit("mailer.parseFromUrl.error.success", {
+        response: response.data,
+      });
+    } catch (error) {
+      emitter.emit("mailer.parseFromUrl.error.error", {
+        error: errorResponse(error),
+      });
+    }
+  });
+};

src/subscribers/ioSubscriber.js

@@ -3,23 +3,23 @@ const emitter = require('../services/eventEmitter');
 const configure = (socket) => {
   // Create a new listener for the download.start event
   emitter.on('download.start', ({ url, filepath }) => {
-    socket.emit('download.start', { message: 'Début de téléchargement' });
+    socket.emit('download.start', { event: 'download.start', message: 'Début de téléchargement' });
   });
 
   // Create a new listener for the download.end event
   emitter.on('download.end', ({ url, filepath }) => {
-    socket.emit('download.end', { message: 'Téléchargement terminé' });
+    socket.emit('download.end', { event: 'download.end', message: 'Téléchargement terminé' });
   });
 
   // Create a new listener for the download.error event
   emitter.on('download.error', ({ url, filepath, error, type }) => {
-    socket.emit('download.error', { error, type, message: 'Erreur lors du téléchargement' });
+    socket.emit('download.error', { event: 'download.error', error, type, message: 'Erreur lors du téléchargement' });
   });
 
   // Create a new listener for the download.progress event
   emitter.on('download.progress', ({ url, filepath, downloaded }) => {
     downloaded = (Math.floor(downloaded / 1024 / 1024 * 100) / 100).toLocaleString();
-    socket.emit('download.progress', { downloaded, message: `[${downloaded} Mo] téléchargés` });
+    socket.emit('download.progress', { event: 'download.progress', downloaded, message: `[${downloaded} Mo] téléchargés` });
   });
 
 
@@ -27,17 +27,28 @@ const configure = (socket) => {
   
   // Create a new listener for the parse start event
   emitter.on('parse.start', ({ filepath, columns, headers, result }) => {
-    socket.emit('parse.start', { message: 'Début de traitement' });
+    socket.emit('parse.start', { event: 'parse.start', message: 'Début de traitement' });
   });
 
   // Create a new listener for the parse.end event
-  emitter.on('parse.end', ({ filepath, columns, count }) => {
+  emitter.on('parse.end', ({ filepath, columns, count, generated }) => {
-    socket.emit('parse.end', { message: 'Traitement terminé' });
+    socket.emit('parse.end', { event: 'parse.end', message: 'Traitement terminé: <br /><br />Lien de téléchargement <a href="/csv/' + generated + '">' + generated + '</a>', generated });
   });
 
   // Create a new listener for the parse.error event
   emitter.on('parse.error', ({ filepath, columns, error }) => {
-    socket.emit('parse.error', { message: 'Erreur lors du traitement' });
+    socket.emit('parse.error', { event: 'parse.error', message: 'Erreur lors du traitement' });
+  });
+  
+  let processed = 0;
+  let limit = 10000;
+  emitter.on('parse.data', ({ filepath, columns, data, index }) => {
+    processed++;
+    
+    if ( processed == limit ) {
+      socket.emit('parse.data', { message: `[${index.toLocaleString()} lignes] traités` });
+      processed = 0;
+    }
   });
 }
 

src/views/emails/parse-success.hbs

@@ -0,0 +1,8 @@
+<p>Bonjour, </p>
+<br />
+<p>Le traitement de votre fichier depuis l'url <strong>{{url}}</strong> est terminé</p>
+<p><strong>{{count}} lignes</strong> ont été générés</p>
+
+<br />
+<p>Voici le lien pour télécharger le fichier généré: <strong>{{host}}/csv/{{generated}}</strong></p>
+<p>Copier le lien ci-dessus et coller le dans votre navigateur</p>

src/views/layout/email.hbs

@@ -0,0 +1,13 @@
+<div class="email-layout">
+  <div class="email-layout__inner">
+    <div class="email-layout__header">
+
+    </div>
+    <div class="email-layout__body">
+      {{{body}}}
+    </div>
+    <div class="email-layout__footer">
+      
+    </div>
+  </div>
+</div>

src/views/pages/index.hbs

@@ -1,11 +1,24 @@
-<div id="index">
+<!DOCTYPE html>
-  <div class="container">
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Génération CSV</title>
+  <!-- Include jQuery -->
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
+  <!-- Include jQuery UI -->
+  <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
+  <!-- Include custom CSS -->
+  <link rel="stylesheet" href="styles.css">
+</head>
 
+<body>
+  <div class="container">
     <div class="px-4 py-5 my-5 text-center">
       <h1 class="display-5 fw-bold text-body-emphasis">Génération CSV</h1>
       <p class="lead mb-4">Génération de fichier CSV depuis une URL pointant vers un fichier de bodacc</p>
     </div>
-
     <div class="container-fluid">
       <div class="row">
         <div class="col-12">
@@ -18,7 +31,7 @@
               <label for="name" class="form-label">URL</label>
               <textarea class="form-control" id="form__url" name="url" rows="5">{{form.url.value}}</textarea>
               <div class="form-text">
-                Inserer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
+                Insérer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
               </div>
             </div>
             <div class="row my-5">
@@ -27,27 +40,29 @@
                   <label for="name" class="column-choices__title">Colonnes disponibles</label>
                   <ul id="form__choices" class="column-choices__list">
                     {{#each form.columns.options}}
-                      <li class="form__choice column-choice" data-value="{{this}}">
+                    <li class="form__choice column-choice" data-value="{{this}}">
-                        <label class="column-choice__inner">
+                      <label class="column-choice__inner">
-                          <input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input" />
+                        <input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input" />
-                          <span class="column-choice__text">{{this}}</span>
+                        <span class="column-choice__text">{{this}}</span>
-                        </label>
+                      </label>
-                      </li>
+                    </li>
                     {{/each}}
                   </ul>
                 </div>
               </div>
               <div class="col-12 col-md-6">
                 <div class="form-group form__column column-choices">
-                  <label for="name" class="column-choices__title column-choices__title--green">Colonnes exportées</label>
+                  <label for="name" class="column-choices__title column-choices__title--green">Colonnes
+                    exportées</label>
                   <ul id="form__selected" class="column-choices__list">
                     {{#each form.columns.selected}}
-                      <li class="form__choice column-choice" data-value="{{this}}">
+                    <li class="form__choice column-choice" data-value="{{this}}">
-                        <label class="column-choice__inner">
+                      <label class="column-choice__inner">
-                          <input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input" checked="checked" />
+                        <input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input"
-                          <span class="column-choice__text">{{this}}</span>
+                          checked="checked" />
-                        </label>
+                        <span class="column-choice__text">{{this}}</span>
-                      </li>
+                      </label>
+                    </li>
                     {{/each}}
                   </ul>
                 </div>
@@ -59,6 +74,10 @@
                 <span class="visually-hidden">Loading...</span>
               </div>
             </div>
+            <div class="d-none my-3" id="form__result">
+              <strong>Lien vers le fichier généré : </strong>
+              <a href="" target="_blank" download></a>
+            </div>
             <div class="d-grid gap-2 col-12 col-md-4 mx-auto my-3">
               <button class="btn btn-primary" id="form__submit" type="submit">Télécharger</button>
             </div>
@@ -66,6 +85,15 @@
         </div>
       </div>
     </div>
-    <!-- /.container-fluid -->
   </div>
-</div>
+  <!-- Script pour le fonctionnement de drag and drop et AJAX -->
+  <script>
+    $(document).ready(function () {
+        $("#form__selected").sortable({
+          revert: false, 
+          helper: "clone"
+        });
+      });
+  </script>
+</body>
+</html>