onja
/
bodacc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge pull request 'fix-feed' (#4) from fix-feed into master 

Reviewed-on: #4
pull/6/head
diary 9 months ago
parent
88fc767e7a 301f8810c6
commit
955c8df46a
7 changed files with 1193 additions and 8814 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      bin/www
  2. 9501
      package-lock.json
  3. 111
      src/models/file.js
  4. 131
      src/routes/index.js
  5. 4
      src/services/file.js
  6. 104
      src/subscribers/emailSubscriber.js
  7. 155
      src/views/pages/index.hbs

1
bin/www
View File

@ -30,6 +30,7 @@ const io = configureSocket(server);
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
server.timeout = 15 * 60 * 1000;
/**
* Normalize a port into a number, string, or false.

9501
package-lock.json
View File

File diff suppressed because it is too large

111
src/models/file.js
View File

@ -70,6 +70,47 @@ class File {
}
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate;
}
// Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
};
}
/**
* Download a file from a url
* @returns {Promise<string>} filepath
*/
async fakeDownload() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
this.filepath = filepath;
this.generatedpath = generatedpath;
return new Promise((resolve, reject) => {
resolve(filepath);
});
}
/**
* Download a file from a url
* @returns {Promise<string>} filepath
@ -77,7 +118,7 @@ class File {
async download() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const { filepath, generatedpath } = this.generateFilePath(this.filename);
const { filepath, generatedpath } = this.generateFilePath(this.filename);
this.filepath = filepath;
this.generatedpath = generatedpath;
@ -150,42 +191,42 @@ class File {
// create a parse method which read the file and return a stream
parse(columns) {
return new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(this.generatedpath, { encoding: 'utf8' }); // Specify UTF-8 encoding for write stream
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
reject(new Error('Invalid columns'));
return false;
}
parse(columns) {
return new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(this.generatedpath);
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
reject(new Error('Invalid columns'));
return false;
}
// Create a variable to hold csv columns indexes
const columnsIndex = {};
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: String(column).split('.')[0],
value: column,
rest: String(column).split('.').slice(1).join('.'),
last: String(column).split('.').pop(),
};
}
// Create a variable to hold csv columns indexes
const columnsIndex = {};
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: (String(column)).split('.')[0],
value: column,
rest: (String(column)).split('.').slice(1).join('.'),
last: (String(column)).split('.').pop(),
};
}
const columnsFiltered = [];
const columnsFiltered = [];
let count = 1;
fs.createReadStream(this.filepath, { encoding: 'utf8' }) // Specify UTF-8 encoding for read stream
let count = 1;
fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' }))
.on('headers', (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
const result = [];
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if (columnsIndex[key].exist) {
if ( columnsIndex[key].exist ) {
columnsFiltered.push(columnsIndex[key].value);
if(columnsIndex[key].main ==='listeprecedentexploitant'){
result.push(`Prec Exp ${columnsIndex[key].last}`)
@ -196,9 +237,10 @@ parse(columns) {
}
}
}
// Emit a parse.start event with the url and filepath
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result });
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
fileStream.write(result.join(';') + "\n");
})
.on('data', (row) => {
@ -214,7 +256,7 @@ parse(columns) {
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => { });
fs.unlink(this.generatedpath, () => {});
})
.on('end', () => {
// Emit a parse.end event with the url and filepath
@ -222,9 +264,8 @@ parse(columns) {
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
resolve(path.basename(this.generatedpath));
});
});
}
});
}
/**
* Generate a new file with the columns

131
src/routes/index.js
View File

@ -101,22 +101,102 @@ const data = {
/* GET home page. */
router.get("/", async function (req, res, next) {
fileService.checkLastOperationDate();
let initialSelectedColumns = [
"region_code",
"region_nom_officiel",
"numerodepartement",
"departement_nom_officiel",
"cp",
"listepersonnes.personne.typePersonne",
"listepersonnes.personne.formeJuridique",
"listepersonnes.personne.denomination",
"listepersonnes.personne.numeroImmatriculation.codeRCS",
"listepersonnes.personne.numeroImmatriculation.numeroIdentification",
"listepersonnes.personne.nom",
"listepersonnes.personne.nomCommercial",
"listepersonnes.personne.prenom",
"listeetablissements.etablissement.activite",
"listeetablissements.etablissement.adresse.complGeographique",
"listeetablissements.etablissement.adresse.ville",
"acte.dateCommencementActivite",
"acte.vente.publiciteLegale.date",
"acte.descriptif",
"acte.vente.categorieVente",
];
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.setHeader("Content-Type", "text/html; charset=utf-8");
const data = {
form: {
url: {
value: 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&refine=numerodepartement%3A%2275%22&refine=typeavis_lib%3A%22Avis%20d%E2%80%99annulation%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B'
},
columns: {
options: [
'id',
'publicationavis',
'publicationavis_facette',
'parution',
'dateparution',
'numeroannonce',
'typeavis',
'typeavis_lib',
'familleavis',
'familleavis_lib',
'tribunal',
'commercant',
'ville',
'registre',
'pdf_parution_subfolder',
'ispdf_unitaire',
'listepersonnes.personne.administration',
'listepersonnes.personne.numeroImmatriculation.nomGreffeImmat',
'listepersonnes.personne.capital.devise',
'listepersonnes.personne.capital.montantCapital',
'listepersonnes.personne.adresseSiegeSocial.ville',
'listepersonnes.personne.adresseSiegeSocial.codePostal',
'listepersonnes.personne.adresseSiegeSocial.pays',
'listepersonnes.personne.adresseSiegeSocial.typeVoie',
'listepersonnes.personne.adresseSiegeSocial.numeroVoie',
'listepersonnes.personne.adresseSiegeSocial.nomVoie',
'listepersonnes.personne.adresseSiegeSocial.complGeographique',
'listeetablissements.etablissement.qualiteEtablissement',
'listeetablissements.etablissement.adresse.codePostal',
'listeetablissements.etablissement.adresse.pays',
'listeetablissements.etablissement.adresse.typeVoie',
'listeetablissements.etablissement.adresse.numeroVoie',
'listeetablissements.etablissement.adresse.nomVoie',
'jugement',
'acte.vente.publiciteLegale.titre',
'acte.vente.opposition',
'modificationsgenerales',
'radiationaurcs',
'depot',
'listeprecedentexploitant.personne.typePersonne',
'listeprecedentexploitant.personne.numeroImmatriculation.codeRCS',
'listeprecedentexploitant.personne.numeroImmatriculation.numeroIdentification',
'listeprecedentexploitant.personne.numeroImmatriculation.nomGreffeImmat',
'listeprecedentexploitant.personne.denomination',
'listeprecedentproprietaire.personne.typePersonne',
'listeprecedentproprietaire.personne.numeroImmatriculation.codeRCS',
'listeprecedentproprietaire.personne.numeroImmatriculation.numeroIdentification',
'listeprecedentproprietaire.personne.numeroImmatriculation.nomGreffeImmat',
'listeprecedentproprietaire.personne.denomination',
'divers',
'parutionavisprecedent.nomPublication',
'parutionavisprecedent.dateParution',
'parutionavisprecedent.numeroParution',
'parutionavisprecedent.numeroAnnonce',
],
selected: initialSelectedColumns,
}
}
};
res.send(templateService.renderPage("index", data));
res.send(templateService.renderPage('index', data));
return res;
});
// Logic to handle the drag and drop update
router.post("/update-columns-order", async function (req, res, next) {
let newColumnsOrder = req.body.columnsOrder;
data.form.columns.selected = newColumnsOrder;
// Handle saving the new column order to a database or file if necessary
res.send("Columns order updated successfully");
});
router.post("/", function (req, res, next) {
router.post('/', function(req, res, next) {
// const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B';
// get url from form
@ -131,18 +211,25 @@ router.post("/", function (req, res, next) {
return res.status(500).send("Invalid columns");
}
fileService
.parseFromUrl(url, columns)
.then((filename) => {
res.send({
generated: filename,
message: "Fichier généré avec success",
});
fileService.parseFromUrl(url, columns)
.then((filename) => {
res.send({
generated: filename,
message: 'Fichier généré avec success'
})
.catch((err) => {
console.error("routes [/] error", err.message);
return res.status(500).send("Invalid stream");
});
})
.catch(err => {
console.error('routes [/] error', err.message);
return res.status(500).send('Invalid stream');
});
});
// Logic to handle the drag and drop update
router.post("/update-columns-order", async function (req, res, next) {
let newColumnsOrder = req.body.columnsOrder;
data.form.columns.selected = newColumnsOrder;
// Handle saving the new column order to a database or file if necessary
res.send("Columns order updated successfully");
});
module.exports = router;

4
src/services/file.js
View File

@ -2,8 +2,8 @@ const csv = require('csv-parser');
const fs = require('fs');
const path = require('path');
const File = require('../models/file.js');
const emitter = require('./eventEmitter.js');
const { basedir } = require('../config/constants.js');
const emitter = require('./eventEmitter');
const { basedir } = require('../config/constants');
const { emit } = require('process');
// Create a class FileService that extends EventEmitter

104
src/subscribers/emailSubscriber.js
View File

@ -1,13 +1,14 @@
const emitter = require("../services/eventEmitter");
const axios = require("axios");
const emitter = require('../services/eventEmitter');
const axios = require('axios');
const params = require("../config/params");
const TemplateService = require("../services/template");
const params = require('../config/params');
const TemplateService = require('../services/template');
const templateService = new TemplateService();
const errorResponse = (error) => {
if (error.response) {
return error.response.status + " - " + error.response.statusText;
return error.response.status + ' - ' + error.response.statusText;
} else if (error.request) {
// The request was made but no response was received
// `error.request` is an instance of XMLHttpRequest in the browser and an instance of
@ -19,110 +20,91 @@ const errorResponse = (error) => {
};
module.exports = (app) => {
// Get app url
let host = "";
let host = '';
app.use((req, res, next) => {
console.log("--- APP_URL ---", req.protocol + "://" + req.get("host"));
host = req.protocol + "://" + req.get("host");
console.log('--- APP_URL ---', req.protocol + '://' + req.get('host'));
host = req.protocol + '://' + req.get('host');
next();
});
// Check if sendgrid params exists
if (!params.sendgrid) {
console.log("--- No sendgrid params ---");
if ( !params.sendgrid ) {
console.log('--- No sendgrid params ---');
return;
}
console.log("--- sendgrid params exists ---");
console.log('--- sendgrid params exists ---');
const sendgridParams = params.sendgrid;
const defaultParams = {
url: sendgridParams.uri,
headers: {
Authorization: "Bearer " + sendgridParams.application_secret,
"Content-Type": "application/json",
Authorization: 'Bearer ' + sendgridParams.application_secret,
'Content-Type': 'application/json',
},
};
const sendEmail = async (to, subject, message) => {
const data = {
personalizations: [
{
to: to,
subject: subject,
},
subject: subject
}
],
content: [
{
type: "text/html",
value: message,
},
value: message
}
],
from: {
email: sendgridParams.from,
name: sendgridParams.fromName,
},
}
};
const response = await axios.post(defaultParams.url, data, {
headers: defaultParams.headers,
});
return null
const response = await axios.post(defaultParams.url, data, { headers: defaultParams.headers });
return response;
};
}
// Create a new listener for the parse.end event
emitter.on("parse.end", async ({ url, columns, count, generated }) => {
console.log("HOST", host);
emitter.on('parse.end', async ({ url, columns, count, generated }) => {
console.log('HOST', host);
try {
const message = templateService.renderEmail("parse-success", {
host,
const message = templateService.renderEmail('parse-success', {
host,
generated,
count,
columns,
url,
});
const response = await sendEmail(
sendgridParams.to,
"BODACC - Traitement terminé",
message
);
emitter.emit("mailer.parse.end.success", { response: response.data });
const response = await sendEmail(sendgridParams.to, 'BODACC - Traitement terminé', message);
emitter.emit('mailer.parse.end.success', { response: response.data });
} catch (error) {
emitter.emit("mailer.parse.end.error", { error: errorResponse(error) });
emitter.emit('mailer.parse.end.error', { error: errorResponse(error) });
}
});
// Create a new listener for the parse.error event
emitter.on("parse.error", async ({ filepath, columns, error }) => {
emitter.on('parse.error', async ({ filepath, columns, error }) => {
try {
const reponse = await sendEmail(
sendgridParams.to,
"BODACC - Erreur traitement",
"<p>Erreur traitement</p><br /><p>Voici le message d'erreur: " +
error +
"</p>"
);
emitter.emit("mailer.parse.error.success", { response: response });
const reponse = await sendEmail(sendgridParams.to, 'BODACC - Erreur traitement', '<p>Erreur traitement</p><br /><p>Voici le message d\'erreur: ' + error + '</p>')
emitter.emit('mailer.parse.error.success', { response: response });
} catch (error) {
emitter.emit("mailer.parse.error.error", { error: errorResponse(error) });
emitter.emit('mailer.parse.error.error', { error: errorResponse(error) });
}
});
// Create a new listener for the parseFromUrl.error event
emitter.on("parseFromUrl.error", async ({ url, columns, error }) => {
emitter.on('parseFromUrl.error', async ({ url, columns, error }) => {
try {
const response = await sendEmail(
sendgridParams.to,
"BODACC - Erreur génération fichier",
"<p>Erreur génération fichier</p><br /><p>Voici le message d'erreur: " +
error +
"</p>"
);
emitter.emit("mailer.parseFromUrl.error.success", {
response: response.data,
});
const response = await sendEmail(sendgridParams.to, 'BODACC - Erreur génération fichier', '<p>Erreur génération fichier</p><br /><p>Voici le message d\'erreur: ' + error + '</p>')
emitter.emit('mailer.parseFromUrl.error.success', { response: response.data });
} catch (error) {
emitter.emit("mailer.parseFromUrl.error.error", {
error: errorResponse(error),
});
emitter.emit('mailer.parseFromUrl.error.error', { error: errorResponse(error) });
}
});
};

155
src/views/pages/index.hbs
View File

@ -1,99 +1,100 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Génération CSV</title>
<!-- Include jQuery -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<!-- Include jQuery UI -->
<script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
<!-- Include custom CSS -->
<link rel="stylesheet" href="styles.css">
<title>Génération CSV</title>
<!-- Include jQuery -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<!-- Include jQuery UI -->
<script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
<!-- Include custom CSS -->
</head>
<body>
<div class="container">
<div class="px-4 py-5 my-5 text-center">
<h1 class="display-5 fw-bold text-body-emphasis">Génération CSV</h1>
<p class="lead mb-4">Génération de fichier CSV depuis une URL pointant vers un fichier de bodacc</p>
</div>
<div class="container-fluid">
<div class="row">
<div class="col-12">
<form method="post" action="/" id="form-download">
<div class="form-group mb-3">
<img src="/img/help.jpg" alt="help" class="img-fluid img-thumbnail" />
</div>
<hr />
<div class="form-group mt-3">
<label for="name" class="form-label">URL</label>
<textarea class="form-control" id="form__url" name="url" rows="5">{{form.url.value}}</textarea>
<div class="form-text">
Insérer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
<div id="index">
<div class="container">
<div class="px-4 py-5 my-5 text-center">
<h1 class="display-5 fw-bold text-body-emphasis">Génération CSV</h1>
<p class="lead mb-4">Génération de fichier CSV depuis une URL pointant vers un fichier de bodacc</p>
</div>
<div class="container-fluid">
<div class="row">
<div class="col-12">
<form method="post" action="/" id="form-download">
<div class="form-group mb-3">
<img src="/img/help.jpg" alt="help" class="img-fluid img-thumbnail" />
</div>
<hr />
<div class="form-group mt-3">
<label for="name" class="form-label">URL</label>
<textarea class="form-control" id="form__url" name="url" rows="5">{{form.url.value}}</textarea>
<div class="form-text">
Inserer dans le champs ci-dessus l'url du fichier CSV bodacc à télécharger
</div>
</div>
</div>
<div class="row my-5">
<div class="col-12 col-md-6">
<div class="form-group form__column column-choices">
<label for="name" class="column-choices__title">Colonnes disponibles</label>
<ul id="form__choices" class="column-choices__list">
{{#each form.columns.options}}
<li class="form__choice column-choice" data-value="{{this}}">
<label class="column-choice__inner">
<input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input" />
<span class="column-choice__text">{{this}}</span>
</label>
</li>
{{/each}}
</ul>
<div class="row my-5">
<div class="col-12 col-md-6">
<div class="form-group form__column column-choices">
<label for="name" class="column-choices__title">Colonnes disponibles</label>
<ul id="form__choices" class="column-choices__list">
{{#each form.columns.options}}
<li class="form__choice column-choice" data-value="{{this}}">
<label class="column-choice__inner">
<input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input" />
<span class="column-choice__text">{{this}}</span>
</label>
</li>
{{/each}}
</ul>
</div>
</div>
<div class="col-12 col-md-6">
<div class="form-group form__column column-choices">
<label for="name" class="column-choices__title column-choices__title--green">Colonnes
exportées</label>
<ul id="form__selected" class="column-choices__list">
{{#each form.columns.selected}}
<li class="form__choice column-choice" data-value="{{this}}">
<label class="column-choice__inner">
<input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input"
checked="checked" />
<span class="column-choice__text">{{this}}</span>
</label>
</li>
{{/each}}
</ul>
</div>
</div>
</div>
<div class="col-12 col-md-6">
<div class="form-group form__column column-choices">
<label for="name" class="column-choices__title column-choices__title--green">Colonnes
exportées</label>
<ul id="form__selected" class="column-choices__list">
{{#each form.columns.selected}}
<li class="form__choice column-choice" data-value="{{this}}">
<label class="column-choice__inner">
<input type="checkbox" name="columns[]" value="{{this}}" class="column-choice__input"
checked="checked" />
<span class="column-choice__text">{{this}}</span>
</label>
</li>
{{/each}}
</ul>
<hr />
<div class="d-flex justify-content-center">
<div class="spinner-border text-primary d-none" role="status" id="form__spinner" style="">
<span class="visually-hidden">Loading...</span>
</div>
</div>
</div>
<hr />
<div class="d-flex justify-content-center">
<div class="spinner-border text-primary d-none" role="status" id="form__spinner" style="">
<span class="visually-hidden">Loading...</span>
<div class="d-none my-3" id="form__result">
<strong>Lien vers le fichier généré : </strong>
<a href="" target="_blank" download></a>
</div>
<div class="d-grid gap-2 col-12 col-md-4 mx-auto my-3">
<button class="btn btn-primary" id="form__submit" type="submit">Télécharger</button>
</div>
</div>
<div class="d-none my-3" id="form__result">
<strong>Lien vers le fichier généré : </strong>
<a href="" target="_blank" download></a>
</div>
<div class="d-grid gap-2 col-12 col-md-4 mx-auto my-3">
<button class="btn btn-primary" id="form__submit" type="submit">Télécharger</button>
</div>
</form>
</form>
</div>
</div>
</div>
<!-- /.container-fluid -->
</div>
</div>
<!-- Script pour le fonctionnement de drag and drop et AJAX -->
<script>
$(document).ready(function () {
<!-- Script pour le fonctionnement de drag and drop et AJAX -->
<script>
$(document).ready(function () {
$("#form__selected").sortable({
revert: false,
revert: false,
helper: "clone"
});
});
</script>
</script>
</body>
</html>
Write Preview
Loading…
Cancel
Save