From f1d912638c12e6ffba4fc1d04f23ce5c9b27ff01 Mon Sep 17 00:00:00 2001 From: Onja Date: Tue, 17 Oct 2023 17:14:08 +0300 Subject: [PATCH 1/4] Reset result and URL input value in the form after submitting it and delete Blob function --- src/assets/js/main.js | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/assets/js/main.js b/src/assets/js/main.js index 73c7bc0..46e0451 100644 --- a/src/assets/js/main.js +++ b/src/assets/js/main.js @@ -36,22 +36,6 @@ const sendRequest = (url, data) => { type: 'POST', data: data, success: function(data, textStatus, jqXHR) { - // Créez un lien de téléchargement et définissez ses attributs - const blob = new Blob([data], { type: 'text/csv' }); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - let date = new Date(); - // format date to YYYY-MM-DD HH:MM - date = date.toISOString().slice(0, 16).replace('T', ' '); - a.href = url; - a.download = `export-${date}.csv`; // Nom du fichier - document.body.appendChild(a); - - // Cliquez sur le lien pour déclencher le téléchargement - a.click(); - - // Supprimez le lien du DOM - window.URL.revokeObjectURL(url); resolve({ message: 'Fichier généré' }); }, error: function(jqXHR, textStatus, errorThrown) { @@ -78,6 +62,11 @@ const initSubmitForm = () => { $submitBtn.prop('disabled', true); $spinner.removeClass('d-none'); + // reset result + const $form__result = $('#form__result'); + $form__result.addClass('d-none'); + $form__result.find('a').attr('href', '#').html(''); + if ( !$urlInput.val() ) { toastr.error('Veuillez saisir une URL'); $submitBtn.prop('disabled', false); @@ -101,7 +90,8 @@ const initSubmitForm = () => { const $this = $(this); data.columns.push($this.val()); }); - + + $urlInput.val(''); sendRequest($form.attr('action'), data) .then((response) => { From 1421af2bb2a0d3f6cf676c00890221d5312f84ce Mon Sep 17 00:00:00 2001 From: Onja Date: Tue, 17 Oct 2023 17:14:32 +0300 Subject: [PATCH 2/4] Add logic to count the number of processed lines and log the count when it reaches the specified limit in the 'parse.data' event listener --- src/subscribers/consoleSubscriber.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/subscribers/consoleSubscriber.js b/src/subscribers/consoleSubscriber.js index 76a281b..b534337 100644 --- a/src/subscribers/consoleSubscriber.js +++ b/src/subscribers/consoleSubscriber.js @@ -78,6 +78,16 @@ emitter.on('parse.error', ({ filepath, columns, error }) => { // emitter.on('parse.data', ({ filepath, columns, data, index }) => { // log('parse.data', `Parsed ${filepath} with columns at index ${index}`); // }); +let processed = 0; +let limit = 10000; +emitter.on('parse.data', ({ filepath, columns, data, index }) => { + processed++; + + if ( processed == limit ) { + log('parse.data', `[${index.toLocaleString()} lignes] traités`); + processed = 0; + } +}); // Create a new listener for the deleteOldFiles.start event From cd747af78ad9bd379e4cebee101b44732727847c Mon Sep 17 00:00:00 2001 From: Onja Date: Tue, 17 Oct 2023 17:15:12 +0300 Subject: [PATCH 3/4] Refactored the `FileService.parseFromUrl` method to return the filepath instead of a stream, and updated the `routes/index.js` file to handle the new return value --- src/models/file.js | 147 +++++++++++++++++++++---------------------- src/routes/index.js | 24 +++---- src/services/file.js | 17 ++--- 3 files changed, 90 insertions(+), 98 deletions(-) diff --git a/src/models/file.js b/src/models/file.js index 86600df..6199a94 100644 --- a/src/models/file.js +++ b/src/models/file.js @@ -18,15 +18,6 @@ const { basedir } = require('../config/constants'); const dest = path.join(basedir, 'public/csv'); -// Create a generateFilePath function witch returns a path with a filename and datetime -function generateFilePath(filename) { - return { - filepath: path.join(dest, `${filename}-${Date.now()}.csv`), - generatedpath: path.join(dest, `${filename}-generated-${Date.now()}.csv`), - }; -} - - // Create a class File that extends EventEmitter class File { @@ -38,7 +29,14 @@ class File { this.url = url; } - + // Create a generateFilePath function witch returns a path with a filename and datetime + generateFilePath(filename) { + const date = new Date(); + return { + filepath: path.join(dest, `${filename}-${date.toISOString().split('T')[0]}.csv`), + generatedpath: path.join(dest, `${filename}-generated-${date.toISOString().split('T')[0]}.csv`), + }; + } /** * Download a file from a url @@ -47,7 +45,7 @@ class File { async download() { const url = URL.parse(this.url); this.filename = slugify(url.hostname, { lower: true }); - const { filepath, generatedpath } = generateFilePath(this.filename); + const { filepath, generatedpath } = this.generateFilePath(this.filename); this.filepath = filepath; this.generatedpath = generatedpath; @@ -121,74 +119,73 @@ class File { // create a parse method which read the file and return a stream parse(columns) { - const stream = new PassThrough(); - const fileStream = fs.createWriteStream(this.generatedpath); - - // check if columns is valid - if (!columns || !columns.length) { - // return Promise.reject(new Error('Invalid columns')); - emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); - return false; - } + return new Promise((resolve, reject) => { + const fileStream = fs.createWriteStream(this.generatedpath); + + // check if columns is valid + if (!columns || !columns.length) { + // return Promise.reject(new Error('Invalid columns')); + emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); + reject(new Error('Invalid columns')); + return false; + } - // Create a variable to hold csv columns indexes - const columnsIndex = {}; - for (let column of columns) { - columnsIndex[column] = { - exist: false, - main: (String(column)).split('.')[0], - value: column, - rest: (String(column)).split('.').slice(1).join('.'), - last: (String(column)).split('.').pop(), - }; - } + // Create a variable to hold csv columns indexes + const columnsIndex = {}; + for (let column of columns) { + columnsIndex[column] = { + exist: false, + main: (String(column)).split('.')[0], + value: column, + rest: (String(column)).split('.').slice(1).join('.'), + last: (String(column)).split('.').pop(), + }; + } + + const columnsFiltered = []; - const columnsFiltered = []; - - let count = 1; - fs.createReadStream(this.filepath) - .pipe(csvParser({ separator: ';' })) - .on('headers', (headers) => { - headers = headers.map(header => typeof header === 'string' ? header.trim() : header); - - const result = []; - for (let key in columnsIndex) { - columnsIndex[key].exist = headers.includes(columnsIndex[key].main); - if ( columnsIndex[key].exist ) { - columnsFiltered.push(columnsIndex[key].value); - result.push(columnsIndex[key].last); + let count = 1; + fs.createReadStream(this.filepath) + .pipe(csvParser({ separator: ';' })) + .on('headers', (headers) => { + headers = headers.map(header => typeof header === 'string' ? header.trim() : header); + + const result = []; + for (let key in columnsIndex) { + columnsIndex[key].exist = headers.includes(columnsIndex[key].main); + if ( columnsIndex[key].exist ) { + columnsFiltered.push(columnsIndex[key].value); + result.push(columnsIndex[key].last); + } } - } - - // Emit a parse.start event with the url and filepath - emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); - - stream.write(result.join(';') + "\n"); - fileStream.write(result.join(';') + "\n"); - }) - .on('data', (row) => { - // Emit a parse.data event with the url, filepath and data - let result = this.processRow(row, columnsIndex, columnsFiltered); - emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); - stream.write(result.join(';') + "\n"); - fileStream.write(result.join(';') + "\n"); - count++; - }) - .on('error', (err) => { - // Emit a parse.error event with the error - emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); - - fileStream.close(); - fs.unlink(this.generatedpath, () => {}); - }) - .on('end', () => { - // Emit a parse.end event with the url and filepath - stream.end(); - fileStream.close(); - emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) }); - }); + + // Emit a parse.start event with the url and filepath + emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); - return stream; + fileStream.write(result.join(';') + "\n"); + }) + .on('data', (row) => { + // Emit a parse.data event with the url, filepath and data + let result = this.processRow(row, columnsIndex, columnsFiltered); + emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); + fileStream.write(result.join(';') + "\n"); + count++; + }) + .on('error', (err) => { + // Emit a parse.error event with the error + emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); + + fileStream.close(); + reject(err); + fs.unlink(this.generatedpath, () => {}); + }) + .on('end', () => { + // Emit a parse.end event with the url and filepath + fileStream.close(); + resolve(this.generatedpath); + emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) }); + }); + }); } /** diff --git a/src/routes/index.js b/src/routes/index.js index 1d8e98a..1b939f6 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -105,7 +105,7 @@ router.get('/', async function(req, res, next) { return res; }); -router.post('/', async function(req, res, next) { +router.post('/', function(req, res, next) { // const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B'; // get url from form @@ -120,22 +120,16 @@ router.post('/', async function(req, res, next) { return res.status(500).send('Invalid columns'); } - let stream = null; - try { - stream = await fileService.parseFromUrl(url, columns); - } catch (err) { + fileService.parseFromUrl(url, columns) + .then((filepath) => { + res.send({ + success: true + }) + }) + .catch(err => { console.error('routes [/] error', err.message); - } - - if ( !stream ) { return res.status(500).send('Invalid stream'); - } - - res.setHeader('Content-Disposition', 'attachment; filename="mon_fichier.csv"'); - res.setHeader('Content-Type', 'text/csv; charset=utf-8'); - stream.pipe(res); - - // res.render('index', { title: 'Express' }); + }); }); module.exports = router; diff --git a/src/services/file.js b/src/services/file.js index 26461f1..5cf446f 100644 --- a/src/services/file.js +++ b/src/services/file.js @@ -29,14 +29,15 @@ class FileService { const file = new File(url); const filepath = await file.download(); - const stream = file.parse(columns); - if ( !stream ) { - emitter.emit('parseFromUrl.error', { url, columns, error: 'Invalid stream' }); - return Promise.reject(new Error('Invalid stream')); - } - - emitter.emit('parseFromUrl.end', { url, columns, filepath }); - return Promise.resolve(stream); + return file.parse(columns) + .then((filepath) => { + emitter.emit('parseFromUrl.end', { url, columns, filepath }); + return filepath; + }) + .catch((err) => { + emitter.emit('parseFromUrl.error', { url, columns, error: err.message }); + return err; + }); } /** From 62798ab8690f4cb93a94f8f77ef5a211a497979c Mon Sep 17 00:00:00 2001 From: Onja Date: Tue, 17 Oct 2023 17:22:59 +0300 Subject: [PATCH 4/4] Refactor generateFilePath function to use custom date format instead of ISO format --- src/models/file.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/models/file.js b/src/models/file.js index 6199a94..f773220 100644 --- a/src/models/file.js +++ b/src/models/file.js @@ -28,13 +28,28 @@ class File { constructor(url) { this.url = url; } + + formatDateToCustomFormat(date) { + // Obtenir les composantes de la date et de l'heure + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc. + const day = String(date.getDate()).padStart(2, '0'); + const hours = String(date.getHours()).padStart(2, '0'); + const minutes = String(date.getMinutes()).padStart(2, '0'); + const seconds = String(date.getSeconds()).padStart(2, '0'); + + // Créer la chaîne au format personnalisé + const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; + + return formattedDate; + } // Create a generateFilePath function witch returns a path with a filename and datetime generateFilePath(filename) { const date = new Date(); return { - filepath: path.join(dest, `${filename}-${date.toISOString().split('T')[0]}.csv`), - generatedpath: path.join(dest, `${filename}-generated-${date.toISOString().split('T')[0]}.csv`), + filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), + generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), }; }