Merge pull request 'fix-feed' (#4) from fix-feed into master

Reviewed-on: #4

bin/www

@@ -30,6 +30,7 @@ const io = configureSocket(server);
 server.listen(port);
 server.on('error', onError);
 server.on('listening', onListening);
+server.timeout = 15 * 60 * 1000;
 
 /**
  * Normalize a port into a number, string, or false.

src/models/file.js

@@ -70,6 +70,47 @@ class File {
   }
   
 
+  formatDateToCustomFormat(date) {  
+    // Obtenir les composantes de la date et de l'heure
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
+    const day = String(date.getDate()).padStart(2, '0');
+    const hours = String(date.getHours()).padStart(2, '0');
+    const minutes = String(date.getMinutes()).padStart(2, '0');
+    const seconds = String(date.getSeconds()).padStart(2, '0');
+  
+    // Créer la chaîne au format personnalisé
+    const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
+  
+    return formattedDate;
+  }
+  
+  // Create a generateFilePath function witch returns a path with a filename and datetime
+  generateFilePath(filename) {
+    const date = new Date();
+    return {
+      filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
+      generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
+    };
+  }
+
+  /**
+   * Download a file from a url
+   * @returns {Promise<string>} filepath
+   */
+  async fakeDownload() {
+    const url = URL.parse(this.url);
+    this.filename = slugify(url.hostname, { lower: true });
+    const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
+    const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
+    this.filepath = filepath;
+    this.generatedpath = generatedpath;
+    return new Promise((resolve, reject) => {
+      resolve(filepath);
+    });
+  }
+  
+
   /**
    * Download a file from a url
    * @returns {Promise<string>} filepath
@@ -150,9 +191,9 @@ class File {
 
 
   // create a parse method which read the file and return a stream
-parse(columns) {
+  parse(columns) {
     return new Promise((resolve, reject) => {
-    const fileStream = fs.createWriteStream(this.generatedpath, { encoding: 'utf8' }); // Specify UTF-8 encoding for write stream
+      const fileStream = fs.createWriteStream(this.generatedpath);
 
       // check if columns is valid
       if (!columns || !columns.length) {
@@ -167,17 +208,17 @@ parse(columns) {
       for (let column of columns) {
         columnsIndex[column] = {
           exist: false,
-        main: String(column).split('.')[0],
+          main: (String(column)).split('.')[0],
           value: column,
-        rest: String(column).split('.').slice(1).join('.'),
-        last: String(column).split('.').pop(),
+          rest: (String(column)).split('.').slice(1).join('.'),
+          last: (String(column)).split('.').pop(),
         };
       }
 
       const columnsFiltered = [];
 
       let count = 1;
-    fs.createReadStream(this.filepath, { encoding: 'utf8' }) // Specify UTF-8 encoding for read stream
+      fs.createReadStream(this.filepath)
       .pipe(csvParser({ separator: ';' }))
       .on('headers', (headers) => {
         headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
@@ -185,7 +226,7 @@ parse(columns) {
         const result = [];
         for (let key in columnsIndex) {
           columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
-          if (columnsIndex[key].exist) {
+          if ( columnsIndex[key].exist ) {
             columnsFiltered.push(columnsIndex[key].value);
             if(columnsIndex[key].main ==='listeprecedentexploitant'){
               result.push(`Prec Exp ${columnsIndex[key].last}`)
@@ -198,7 +239,8 @@ parse(columns) {
         }
         
         // Emit a parse.start event with the url and filepath
-        emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result });
+        emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
+
         fileStream.write(result.join(';') + "\n");
       })
       .on('data', (row) => {
@@ -214,7 +256,7 @@ parse(columns) {
 
         fileStream.close();
         reject(err);
-        fs.unlink(this.generatedpath, () => { });
+        fs.unlink(this.generatedpath, () => {});
       })
       .on('end', () => {
         // Emit a parse.end event with the url and filepath
@@ -223,8 +265,7 @@ parse(columns) {
         resolve(path.basename(this.generatedpath));
       });
     });
-}
-
+  }
 
   /**
    * Generate a new file with the columns

src/routes/index.js

@@ -101,22 +101,102 @@ const data = {
 /* GET home page. */
 router.get("/", async function (req, res, next) {
   fileService.checkLastOperationDate();
+  let initialSelectedColumns = [
+    "region_code",
+    "region_nom_officiel",
+    "numerodepartement",
+    "departement_nom_officiel",
+    "cp",
+    "listepersonnes.personne.typePersonne",
+    "listepersonnes.personne.formeJuridique",
+    "listepersonnes.personne.denomination",
+    "listepersonnes.personne.numeroImmatriculation.codeRCS",
+    "listepersonnes.personne.numeroImmatriculation.numeroIdentification",
+    "listepersonnes.personne.nom",
+    "listepersonnes.personne.nomCommercial",
+    "listepersonnes.personne.prenom",
+    "listeetablissements.etablissement.activite",
+    "listeetablissements.etablissement.adresse.complGeographique",
+    "listeetablissements.etablissement.adresse.ville",
+    "acte.dateCommencementActivite",
+    "acte.vente.publiciteLegale.date",
+    "acte.descriptif",
+    "acte.vente.categorieVente",
+  ];
+  res.setHeader('Content-Type', 'text/html; charset=utf-8');
 
-  res.setHeader("Content-Type", "text/html; charset=utf-8");
+  const data = {
+    form: {
+      url: {
+        value: 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&refine=numerodepartement%3A%2275%22&refine=typeavis_lib%3A%22Avis%20d%E2%80%99annulation%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B'
+      },
+      columns: {
+        options: [
+          'id',
+          'publicationavis',
+          'publicationavis_facette',
+          'parution',
+          'dateparution',
+          'numeroannonce',
+          'typeavis',
+          'typeavis_lib',
+          'familleavis',
+          'familleavis_lib',
+          'tribunal',
+          'commercant',
+          'ville',
+          'registre',
+          'pdf_parution_subfolder',
+          'ispdf_unitaire',
+          'listepersonnes.personne.administration',
+          'listepersonnes.personne.numeroImmatriculation.nomGreffeImmat',
+          'listepersonnes.personne.capital.devise',
+          'listepersonnes.personne.capital.montantCapital',
+          'listepersonnes.personne.adresseSiegeSocial.ville',
+          'listepersonnes.personne.adresseSiegeSocial.codePostal',
+          'listepersonnes.personne.adresseSiegeSocial.pays',
+          'listepersonnes.personne.adresseSiegeSocial.typeVoie',
+          'listepersonnes.personne.adresseSiegeSocial.numeroVoie',
+          'listepersonnes.personne.adresseSiegeSocial.nomVoie',
+          'listepersonnes.personne.adresseSiegeSocial.complGeographique',
+          'listeetablissements.etablissement.qualiteEtablissement',
+          'listeetablissements.etablissement.adresse.codePostal',
+          'listeetablissements.etablissement.adresse.pays',
+          'listeetablissements.etablissement.adresse.typeVoie',
+          'listeetablissements.etablissement.adresse.numeroVoie',
+          'listeetablissements.etablissement.adresse.nomVoie',
+          'jugement',
+          'acte.vente.publiciteLegale.titre',
+          'acte.vente.opposition',
+          'modificationsgenerales',
+          'radiationaurcs',
+          'depot',
+          'listeprecedentexploitant.personne.typePersonne',
+          'listeprecedentexploitant.personne.numeroImmatriculation.codeRCS',
+          'listeprecedentexploitant.personne.numeroImmatriculation.numeroIdentification',
+          'listeprecedentexploitant.personne.numeroImmatriculation.nomGreffeImmat',
+          'listeprecedentexploitant.personne.denomination',
+          'listeprecedentproprietaire.personne.typePersonne',
+          'listeprecedentproprietaire.personne.numeroImmatriculation.codeRCS',
+          'listeprecedentproprietaire.personne.numeroImmatriculation.numeroIdentification',
+          'listeprecedentproprietaire.personne.numeroImmatriculation.nomGreffeImmat',
+          'listeprecedentproprietaire.personne.denomination',
+          'divers',
+          'parutionavisprecedent.nomPublication',
+          'parutionavisprecedent.dateParution',
+          'parutionavisprecedent.numeroParution',
+          'parutionavisprecedent.numeroAnnonce',
+        ],
+        selected: initialSelectedColumns,
+      }
+    }
+  };
 
-  res.send(templateService.renderPage("index", data));
+  res.send(templateService.renderPage('index', data));
   return res;
 });
 
-// Logic to handle the drag and drop update
-router.post("/update-columns-order", async function (req, res, next) {
-  let newColumnsOrder = req.body.columnsOrder;
-  data.form.columns.selected = newColumnsOrder;
-  // Handle saving the new column order to a database or file if necessary
-  res.send("Columns order updated successfully");
-});
-
-router.post("/", function (req, res, next) {
+router.post('/', function(req, res, next) {  
   // const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B';
 
   // get url from form
@@ -131,18 +211,25 @@ router.post("/", function (req, res, next) {
     return res.status(500).send("Invalid columns");
   }
 
-  fileService
-    .parseFromUrl(url, columns)
+  fileService.parseFromUrl(url, columns)
   .then((filename) => {
     res.send({
       generated: filename,
-        message: "Fichier généré avec success",
-      });
+      message: 'Fichier généré avec success'
+    })
   })
-    .catch((err) => {
-      console.error("routes [/] error", err.message);
-      return res.status(500).send("Invalid stream");
+  .catch(err => {
+    console.error('routes [/] error', err.message);
+    return res.status(500).send('Invalid stream');
   });
 });
 
+// Logic to handle the drag and drop update
+router.post("/update-columns-order", async function (req, res, next) {
+  let newColumnsOrder = req.body.columnsOrder;
+  data.form.columns.selected = newColumnsOrder;
+  // Handle saving the new column order to a database or file if necessary
+  res.send("Columns order updated successfully");
+});
+
 module.exports = router;

src/services/file.js

@@ -2,8 +2,8 @@ const csv = require('csv-parser');
 const fs = require('fs');
 const path = require('path');
 const File = require('../models/file.js');
-const emitter = require('./eventEmitter.js');
-const { basedir } = require('../config/constants.js');
+const emitter = require('./eventEmitter');
+const { basedir } = require('../config/constants');
 const { emit } = require('process');
 
 // Create a class FileService that extends EventEmitter

src/subscribers/emailSubscriber.js

@@ -1,13 +1,14 @@
-const emitter = require("../services/eventEmitter");
-const axios = require("axios");
+const emitter = require('../services/eventEmitter');
+const axios = require('axios');
 
-const params = require("../config/params");
-const TemplateService = require("../services/template");
+const params = require('../config/params');
+const TemplateService = require('../services/template');
 const templateService = new TemplateService();
 
+  
 const errorResponse = (error) => {
   if (error.response) {
-    return error.response.status + " - " + error.response.statusText;
+    return error.response.status + ' - ' + error.response.statusText;
   } else if (error.request) {
     // The request was made but no response was received
     // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
@@ -19,27 +20,28 @@ const errorResponse = (error) => {
 };
 
 module.exports = (app) => {
+
   // Get app url
-  let host = "";
+  let host = '';
   app.use((req, res, next) => {
-    console.log("--- APP_URL ---", req.protocol + "://" + req.get("host"));
-    host = req.protocol + "://" + req.get("host");
+    console.log('--- APP_URL ---', req.protocol + '://' + req.get('host'));
+    host = req.protocol + '://' + req.get('host');
     next();
   });
 
   // Check if sendgrid params exists
-  if (!params.sendgrid) {
-    console.log("--- No sendgrid params ---");
+  if ( !params.sendgrid ) {
+    console.log('--- No sendgrid params ---');
     return;
   }
-  console.log("--- sendgrid params exists ---");
+  console.log('--- sendgrid params exists ---');
   
   const sendgridParams = params.sendgrid;
   const defaultParams = {
     url: sendgridParams.uri,
     headers: {
-      Authorization: "Bearer " + sendgridParams.application_secret,
-      "Content-Type": "application/json",
+      Authorization: 'Bearer ' + sendgridParams.application_secret,
+      'Content-Type': 'application/json',
     },
   };
   
@@ -48,81 +50,61 @@ module.exports = (app) => {
       personalizations: [
         {
           to: to,
-          subject: subject,
-        },
+          subject: subject
+        }
       ],
       content: [
         {
           type: "text/html",
-          value: message,
-        },
+          value: message
+        }
       ],
       from: {
         email: sendgridParams.from,
         name: sendgridParams.fromName,
-      },
+      }
     };
-    const response = await axios.post(defaultParams.url, data, {
-      headers: defaultParams.headers,
-    });
+  
+    return null
+    const response = await axios.post(defaultParams.url, data, { headers: defaultParams.headers });
     return response;
-  };
+  }
 
   // Create a new listener for the parse.end event
-  emitter.on("parse.end", async ({ url, columns, count, generated }) => {
-    console.log("HOST", host);
+  emitter.on('parse.end', async ({ url, columns, count, generated }) => {
+    console.log('HOST', host);
     try {
-      const message = templateService.renderEmail("parse-success", {
+      const message = templateService.renderEmail('parse-success', { 
         host, 
         generated,
         count,
         columns,
         url,
       });
-      const response = await sendEmail(
-        sendgridParams.to,
-        "BODACC - Traitement terminé",
-        message
-      );
-      emitter.emit("mailer.parse.end.success", { response: response.data });
+      const response = await sendEmail(sendgridParams.to, 'BODACC - Traitement terminé', message);
+      emitter.emit('mailer.parse.end.success', { response: response.data });
     } catch (error) {
-      emitter.emit("mailer.parse.end.error", { error: errorResponse(error) });
+      emitter.emit('mailer.parse.end.error', { error: errorResponse(error) });
     }
   });
   
   // Create a new listener for the parse.error event
-  emitter.on("parse.error", async ({ filepath, columns, error }) => {
+  emitter.on('parse.error', async ({ filepath, columns, error }) => {
     try {
-      const reponse = await sendEmail(
-        sendgridParams.to,
-        "BODACC - Erreur traitement",
-        "<p>Erreur traitement</p><br /><p>Voici le message d'erreur: " +
-          error +
-          "</p>"
-      );
-      emitter.emit("mailer.parse.error.success", { response: response });
+      const reponse = await sendEmail(sendgridParams.to, 'BODACC - Erreur traitement', '<p>Erreur traitement</p><br /><p>Voici le message d\'erreur: ' + error + '</p>')
+      emitter.emit('mailer.parse.error.success', { response: response });
     } catch (error) {
-      emitter.emit("mailer.parse.error.error", { error: errorResponse(error) });
+      emitter.emit('mailer.parse.error.error', { error: errorResponse(error) });
     }
   });
   
   // Create a new listener for the parseFromUrl.error event
-  emitter.on("parseFromUrl.error", async ({ url, columns, error }) => {
+  emitter.on('parseFromUrl.error', async ({ url, columns, error }) => {
     try {
-      const response = await sendEmail(
-        sendgridParams.to,
-        "BODACC - Erreur génération fichier",
-        "<p>Erreur génération fichier</p><br /><p>Voici le message d'erreur: " +
-          error +
-          "</p>"
-      );
-      emitter.emit("mailer.parseFromUrl.error.success", {
-        response: response.data,
-      });
+      const response = await sendEmail(sendgridParams.to, 'BODACC - Erreur génération fichier', '<p>Erreur génération fichier</p><br /><p>Voici le message d\'erreur: ' + error + '</p>')
+      emitter.emit('mailer.parseFromUrl.error.success', { response: response.data });
     } catch (error) {
-      emitter.emit("mailer.parseFromUrl.error.error", {
-        error: errorResponse(error),
-      });
+      emitter.emit('mailer.parseFromUrl.error.error', { error: errorResponse(error) });
     }
   });
 };

src/views/pages/index.hbs

@@ -1,24 +1,23 @@
 <!DOCTYPE html>
 <html lang="en">
-
 <head>
   <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Génération CSV</title>
-  <!-- Include jQuery -->
-  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
-  <!-- Include jQuery UI -->
-  <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
-  <!-- Include custom CSS -->
-  <link rel="stylesheet" href="styles.css">
+<title>Génération CSV</title>
+<!-- Include jQuery -->
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
+<!-- Include jQuery UI -->
+<script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
+<!-- Include custom CSS -->
 </head>
-
 <body>
+  <div id="index">
     <div class="container">
+  
       <div class="px-4 py-5 my-5 text-center">
         <h1 class="display-5 fw-bold text-body-emphasis">Génération CSV</h1>
         <p class="lead mb-4">Génération de fichier CSV depuis une URL pointant vers un fichier de bodacc</p>
       </div>
+  
       <div class="container-fluid">
         <div class="row">
           <div class="col-12">
@@ -31,7 +30,7 @@
                 <label for="name" class="form-label">URL</label>
                 <textarea class="form-control" id="form__url" name="url" rows="5">{{form.url.value}}</textarea>
                 <div class="form-text">
-                Insérer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
+                  Inserer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
                 </div>
               </div>
               <div class="row my-5">
@@ -85,6 +84,8 @@
           </div>
         </div>
       </div>
+      <!-- /.container-fluid -->
+    </div>
   </div>
     <!-- Script pour le fonctionnement de drag and drop et AJAX -->
     <script>