onja
/
bodacc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge branch 'node10' of gitea.invaders.stream:onja/bodacc into node10

node10
root 1 year ago
parent
e4d8e2a580 62798ab869
commit
ed3559b1f0
5 changed files with 122 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 24
      src/assets/js/main.js
  2. 162
      src/models/file.js
  3. 24
      src/routes/index.js
  4. 17
      src/services/file.js
  5. 10
      src/subscribers/consoleSubscriber.js

24
src/assets/js/main.js
View File

@ -36,22 +36,6 @@ const sendRequest = (url, data) => {
type: 'POST',
data: data,
success: function(data, textStatus, jqXHR) {
// Créez un lien de téléchargement et définissez ses attributs
const blob = new Blob([data], { type: 'text/csv' });
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
let date = new Date();
// format date to YYYY-MM-DD HH:MM
date = date.toISOString().slice(0, 16).replace('T', ' ');
a.href = url;
a.download = `export-${date}.csv`; // Nom du fichier
document.body.appendChild(a);
// Cliquez sur le lien pour déclencher le téléchargement
a.click();
// Supprimez le lien du DOM
window.URL.revokeObjectURL(url);
resolve({ message: 'Fichier généré' });
},
error: function(jqXHR, textStatus, errorThrown) {
@ -78,6 +62,11 @@ const initSubmitForm = () => {
$submitBtn.prop('disabled', true);
$spinner.removeClass('d-none');
// reset result
const $form__result = $('#form__result');
$form__result.addClass('d-none');
$form__result.find('a').attr('href', '#').html('');
if ( !$urlInput.val() ) {
toastr.error('Veuillez saisir une URL');
$submitBtn.prop('disabled', false);
@ -101,7 +90,8 @@ const initSubmitForm = () => {
const $this = $(this);
data.columns.push($this.val());
});
$urlInput.val('');
sendRequest($form.attr('action'), data)
.then((response) => {

162
src/models/file.js
View File

@ -18,15 +18,6 @@ const { basedir } = require('../config/constants');
const dest = path.join(basedir, 'public/csv');
// Create a generateFilePath function witch returns a path with a filename and datetime
function generateFilePath(filename) {
return {
filepath: path.join(dest, `${filename}-${Date.now()}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${Date.now()}.csv`),
};
}
// Create a class File that extends EventEmitter
class File {
@ -37,8 +28,30 @@ class File {
constructor(url) {
this.url = url;
}
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate;
}
// Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
};
}
/**
* Download a file from a url
@ -65,7 +78,7 @@ class File {
async download() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const { filepath, generatedpath } = generateFilePath(this.filename);
const { filepath, generatedpath } = this.generateFilePath(this.filename);
this.filepath = filepath;
this.generatedpath = generatedpath;
@ -139,74 +152,73 @@ class File {
// create a parse method which read the file and return a stream
parse(columns) {
const stream = new PassThrough();
const fileStream = fs.createWriteStream(this.generatedpath);
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
return false;
}
return new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(this.generatedpath);
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
reject(new Error('Invalid columns'));
return false;
}
// Create a variable to hold csv columns indexes
const columnsIndex = {};
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: (String(column)).split('.')[0],
value: column,
rest: (String(column)).split('.').slice(1).join('.'),
last: (String(column)).split('.').pop(),
};
}
// Create a variable to hold csv columns indexes
const columnsIndex = {};
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: (String(column)).split('.')[0],
value: column,
rest: (String(column)).split('.').slice(1).join('.'),
last: (String(column)).split('.').pop(),
};
}
const columnsFiltered = [];
let count = 1;
fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' }))
.on('headers', (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if ( columnsIndex[key].exist ) {
columnsFiltered.push(columnsIndex[key].value);
result.push(columnsIndex[key].last);
const columnsFiltered = [];
let count = 1;
fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' }))
.on('headers', (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if ( columnsIndex[key].exist ) {
columnsFiltered.push(columnsIndex[key].value);
result.push(columnsIndex[key].last);
}
}
}
// Emit a parse.start event with the url and filepath
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
stream.write(result.join(';') + "\n");
fileStream.write(result.join(';') + "\n");
})
.on('data', (row) => {
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered);
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
stream.write(result.join(';') + "\n");
fileStream.write(result.join(';') + "\n");
count++;
})
.on('error', (err) => {
// Emit a parse.error event with the error
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
fileStream.close();
fs.unlink(this.generatedpath, () => {});
})
.on('end', () => {
// Emit a parse.end event with the url and filepath
stream.end();
fileStream.close();
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
});
// Emit a parse.start event with the url and filepath
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
return stream;
fileStream.write(result.join(';') + "\n");
})
.on('data', (row) => {
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered);
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
fileStream.write(result.join(';') + "\n");
count++;
})
.on('error', (err) => {
// Emit a parse.error event with the error
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => {});
})
.on('end', () => {
// Emit a parse.end event with the url and filepath
fileStream.close();
resolve(this.generatedpath);
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
});
});
}
/**

24
src/routes/index.js
View File

@ -105,7 +105,7 @@ router.get('/', async function(req, res, next) {
return res;
});
router.post('/', async function(req, res, next) {
router.post('/', function(req, res, next) {
// const url = 'https://bodacc-datadila.opendatasoft.com/api/explore/v2.1/catalog/datasets/annonces-commerciales/exports/csv?lang=fr&refine=publicationavis%3A%22A%22&refine=publicationavis_facette%3A%22Bodacc%20A%22&refine=familleavis_lib%3A%22Ventes%20et%20cessions%22&timezone=Asia%2FBaghdad&use_labels=true&delimiter=%3B';
// get url from form
@ -120,22 +120,16 @@ router.post('/', async function(req, res, next) {
return res.status(500).send('Invalid columns');
}
let stream = null;
try {
stream = await fileService.parseFromUrl(url, columns);
} catch (err) {
fileService.parseFromUrl(url, columns)
.then((filepath) => {
res.send({
success: true
})
})
.catch(err => {
console.error('routes [/] error', err.message);
}
if ( !stream ) {
return res.status(500).send('Invalid stream');
}
res.setHeader('Content-Disposition', 'attachment; filename="mon_fichier.csv"');
res.setHeader('Content-Type', 'text/csv; charset=utf-8');
stream.pipe(res);
// res.render('index', { title: 'Express' });
});
});
module.exports = router;

17
src/services/file.js
View File

@ -29,14 +29,15 @@ class FileService {
const file = new File(url);
const filepath = await file.download();
const stream = file.parse(columns);
if ( !stream ) {
emitter.emit('parseFromUrl.error', { url, columns, error: 'Invalid stream' });
return Promise.reject(new Error('Invalid stream'));
}
emitter.emit('parseFromUrl.end', { url, columns, filepath });
return Promise.resolve(stream);
return file.parse(columns)
.then((filepath) => {
emitter.emit('parseFromUrl.end', { url, columns, filepath });
return filepath;
})
.catch((err) => {
emitter.emit('parseFromUrl.error', { url, columns, error: err.message });
return err;
});
}
/**

10
src/subscribers/consoleSubscriber.js
View File

@ -78,6 +78,16 @@ emitter.on('parse.error', ({ filepath, columns, error }) => {
// emitter.on('parse.data', ({ filepath, columns, data, index }) => {
// log('parse.data', `Parsed ${filepath} with columns at index ${index}`);
// });
let processed = 0;
let limit = 10000;
emitter.on('parse.data', ({ filepath, columns, data, index }) => {
processed++;
if ( processed == limit ) {
log('parse.data', `[${index.toLocaleString()} lignes] traités`);
processed = 0;
}
});
// Create a new listener for the deleteOldFiles.start event

Write Preview
Loading…
Cancel
Save