diff --git a/src/models/file.js b/src/models/file.js index 6cc45a1..71ef29f 100644 --- a/src/models/file.js +++ b/src/models/file.js @@ -1,55 +1,59 @@ /* global basedir */ -const path = require('path'); +const path = require("path"); const URL = require("url"); -const fs = require('fs'); -const util = require('util'); -const http = require('http'); -const https = require('https'); -const exec = util.promisify(require('child_process').exec); +const fs = require("fs"); +const util = require("util"); +const http = require("http"); +const https = require("https"); +const exec = util.promisify(require("child_process").exec); -const csvParser = require('csv-parser'); -const { PassThrough } = require('stream'); -const slugify = require('slugify'); -const emitter = require('../services/eventEmitter'); +const csvParser = require("csv-parser"); +const { PassThrough } = require("stream"); +const slugify = require("slugify"); +const emitter = require("../services/eventEmitter"); // Load project path from config/constants.js -const { basedir } = require('../config/constants'); - -const dest = path.join(basedir, 'public/csv'); +const { basedir } = require("../config/constants"); +const dest = path.join(basedir, "public/csv"); // Create a class File that extends EventEmitter class File { - /** * Create a new File instance - * @param {string} url + * @param {string} url */ constructor(url) { this.url = url; } - formatDateToCustomFormat(date) { + formatDateToCustomFormat(date) { // Obtenir les composantes de la date et de l'heure const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc. - const day = String(date.getDate()).padStart(2, '0'); - const hours = String(date.getHours()).padStart(2, '0'); - const minutes = String(date.getMinutes()).padStart(2, '0'); - const seconds = String(date.getSeconds()).padStart(2, '0'); - + const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc. + const day = String(date.getDate()).padStart(2, "0"); + const hours = String(date.getHours()).padStart(2, "0"); + const minutes = String(date.getMinutes()).padStart(2, "0"); + const seconds = String(date.getSeconds()).padStart(2, "0"); + // Créer la chaîne au format personnalisé const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; - + return formattedDate; } - + // Create a generateFilePath function witch returns a path with a filename and datetime generateFilePath(filename) { const date = new Date(); return { - filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), - generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), + filepath: path.join( + dest, + `${filename}-${this.formatDateToCustomFormat(date)}.csv` + ), + generatedpath: path.join( + dest, + `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv` + ), }; } @@ -60,37 +64,48 @@ class File { async fakeDownload() { const url = URL.parse(this.url); this.filename = slugify(url.hostname, { lower: true }); - const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); - const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); + const filepath = path.join( + dest, + `bodacc-datadila.opendatasoft.com-1697536785170.csv` + ); + const generatedpath = path.join( + dest, + `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv` + ); this.filepath = filepath; this.generatedpath = generatedpath; return new Promise((resolve, reject) => { resolve(filepath); }); } - - formatDateToCustomFormat(date) { + formatDateToCustomFormat(date) { // Obtenir les composantes de la date et de l'heure const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc. - const day = String(date.getDate()).padStart(2, '0'); - const hours = String(date.getHours()).padStart(2, '0'); - const minutes = String(date.getMinutes()).padStart(2, '0'); - const seconds = String(date.getSeconds()).padStart(2, '0'); - + const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc. + const day = String(date.getDate()).padStart(2, "0"); + const hours = String(date.getHours()).padStart(2, "0"); + const minutes = String(date.getMinutes()).padStart(2, "0"); + const seconds = String(date.getSeconds()).padStart(2, "0"); + // Créer la chaîne au format personnalisé const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; - + return formattedDate; } - + // Create a generateFilePath function witch returns a path with a filename and datetime generateFilePath(filename) { const date = new Date(); return { - filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), - generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), + filepath: path.join( + dest, + `${filename}-${this.formatDateToCustomFormat(date)}.csv` + ), + generatedpath: path.join( + dest, + `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv` + ), }; } @@ -101,15 +116,20 @@ class File { async fakeDownload() { const url = URL.parse(this.url); this.filename = slugify(url.hostname, { lower: true }); - const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); - const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); + const filepath = path.join( + dest, + `bodacc-datadila.opendatasoft.com-1697536785170.csv` + ); + const generatedpath = path.join( + dest, + `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv` + ); this.filepath = filepath; this.generatedpath = generatedpath; return new Promise((resolve, reject) => { resolve(filepath); }); } - /** * Download a file from a url @@ -122,74 +142,101 @@ class File { this.filepath = filepath; this.generatedpath = generatedpath; - const file = fs.createWriteStream(this.filepath); - const request = url.protocol === 'https:' ? https : http; + const file = fs.createWriteStream(this.filepath); + const request = url.protocol === "https:" ? https : http; // Emit a download.start event with the url and filepath - emitter.emit('download.start', { url: this.url, filepath: this.filepath }); + emitter.emit("download.start", { url: this.url, filepath: this.filepath }); return new Promise((resolve, reject) => { // Handle file errors - file.on('error', (err) => { + file.on("error", (err) => { fs.unlink(filepath, () => { // Emit a download.error event with the error - emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' }); + emitter.emit("download.error", { + url: this.url, + filepath: this.filepath, + error: err.message, + type: "file", + }); }); return reject(err.message); }); request - .get(url.href, (response) => { - // Check if response is valid - if (response.statusCode !== 200) { - // Emit a download.error event with the error - emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' }); - return reject(response.statusMessage); - } - - - emitter.emit('download.started', { url: this.url, filepath: this.filepath }); - - // Create a variable to hold the downloaded size - let downloaded = 0; - - // Handle response data - let i = 0; - response.on('data', (chunk) => { - downloaded += chunk.length; - i++; - - // Check if the downloaded size is bigger than the step limit - if (i === 500) { - // Emit a download.progress event with the url, filepath and downloaded size - emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded }); - i = 0; + .get(url.href, (response) => { + // Check if response is valid + if (response.statusCode !== 200) { + // Emit a download.error event with the error + emitter.emit("download.error", { + url: this.url, + filepath: this.filepath, + error: response.statusMessage, + type: "response", + }); + return reject(response.statusMessage); } - }); - response.pipe(file); + emitter.emit("download.started", { + url: this.url, + filepath: this.filepath, + }); - file.on('finish', () => { - emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded }); + // Create a variable to hold the downloaded size + let downloaded = 0; + + // Handle response data + let i = 0; + response.on("data", (chunk) => { + downloaded += chunk.length; + i++; + + // Check if the downloaded size is bigger than the step limit + if (i === 500) { + // Emit a download.progress event with the url, filepath and downloaded size + emitter.emit("download.progress", { + url: this.url, + filepath: this.filepath, + downloaded, + }); + i = 0; + } + }); - // Emit a download.end event with the url and filepath - emitter.emit('download.end', { url: this.url, filepath: this.filepath }); - file.close(() => { - resolve(filepath); + response.pipe(file); + + file.on("finish", () => { + emitter.emit("download.progress", { + url: this.url, + filepath: this.filepath, + downloaded, + }); + + // Emit a download.end event with the url and filepath + emitter.emit("download.end", { + url: this.url, + filepath: this.filepath, + }); + file.close(() => { + resolve(filepath); + }); + }); + }) + .on("error", (err) => { + fs.unlink(filepath, () => { + // Emit a download.error event with the error + emitter.emit("download.error", { + url: this.url, + filepath: this.filepath, + error: err.message, + type: "request", + }); + reject(err.message); }); }); - }) - .on('error', (err) => { - fs.unlink(filepath, () => { - // Emit a download.error event with the error - emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' }); - reject(err.message); - }); - }); }); } - // create a parse method which read the file and return a stream parse(columns) { return new Promise((resolve, reject) => { @@ -198,8 +245,12 @@ class File { // check if columns is valid if (!columns || !columns.length) { // return Promise.reject(new Error('Invalid columns')); - emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); - reject(new Error('Invalid columns')); + emitter.emit("parse.error", { + url: this.url, + filepath: this.filepath, + error: "Invalid columns", + }); + reject(new Error("Invalid columns")); return false; } @@ -208,10 +259,10 @@ class File { for (let column of columns) { columnsIndex[column] = { exist: false, - main: (String(column)).split('.')[0], + main: String(column).split(".")[0], value: column, - rest: (String(column)).split('.').slice(1).join('.'), - last: (String(column)).split('.').pop(), + rest: String(column).split(".").slice(1).join("."), + last: String(column).split(".").pop(), }; } @@ -219,84 +270,129 @@ class File { let count = 1; fs.createReadStream(this.filepath) - .pipe(csvParser({ separator: ';' })) - .on('headers', (headers) => { - headers = headers.map(header => typeof header === 'string' ? header.trim() : header); - - const result = []; - for (let key in columnsIndex) { - columnsIndex[key].exist = headers.includes(columnsIndex[key].main); - if ( columnsIndex[key].exist ) { - columnsFiltered.push(columnsIndex[key].value); - if(columnsIndex[key].main ==='listeprecedentexploitant'){ - result.push(`Prec Exp ${columnsIndex[key].last}`) - }else if(columnsIndex[key].main ==='listeprecedentproprietaire'){ - result.push(`Prec Prop ${columnsIndex[key].last}`) - }else{ - result.push(columnsIndex[key].last); + .pipe(csvParser({ separator: ";" })) + .on("headers", (headers) => { + headers = headers.map((header) => + typeof header === "string" ? header.trim() : header + ); + + const result = []; + for (let key in columnsIndex) { + columnsIndex[key].exist = headers.includes(columnsIndex[key].main); + if (columnsIndex[key].exist) { + columnsFiltered.push(columnsIndex[key].value); + if (columnsIndex[key].main === "listeprecedentexploitant") { + result.push(`Prec Exp ${columnsIndex[key].last}`); + } else if ( + columnsIndex[key].main === "listeprecedentproprietaire" + ) { + result.push(`Prec Prop ${columnsIndex[key].last}`); + } else if (columnsIndex[key].last === "origineFonds") { + result.push("Montant Achat"); + } else { + result.push(columnsIndex[key].last); + } } } - } - - // Emit a parse.start event with the url and filepath - emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); - - fileStream.write(result.join(';') + "\n"); - }) - .on('data', (row) => { - // Emit a parse.data event with the url, filepath and data - let result = this.processRow(row, columnsIndex, columnsFiltered); - emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); - fileStream.write(result.join(';') + "\n"); - count++; - }) - .on('error', (err) => { - // Emit a parse.error event with the error - emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); - - fileStream.close(); - reject(err); - fs.unlink(this.generatedpath, () => {}); - }) - .on('end', () => { - // Emit a parse.end event with the url and filepath - fileStream.close(); - emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) }); - resolve(path.basename(this.generatedpath)); - }); + + // Emit a parse.start event with the url and filepath + emitter.emit("parse.start", { + url: this.url, + filepath: this.filepath, + headers, + result: result, + }); + + fileStream.write(result.join(";") + "\n"); + }) + .on("data", (row) => { + // Emit a parse.data event with the url, filepath and data + let result = this.processRow(row, columnsIndex, columnsFiltered); + emitter.emit("parse.data", { + url: this.url, + filepath: this.filepath, + data: row, + result, + index: count, + }); + fileStream.write(result.join(";") + "\n"); + count++; + }) + .on("error", (err) => { + // Emit a parse.error event with the error + emitter.emit("parse.error", { + url: this.url, + filepath: this.filepath, + error: err.message, + }); + + fileStream.close(); + reject(err); + fs.unlink(this.generatedpath, () => {}); + }) + .on("end", () => { + // Emit a parse.end event with the url and filepath + fileStream.close(); + emitter.emit("parse.end", { + url: this.url, + filepath: this.filepath, + count: count - 1, + generated: path.basename(this.generatedpath), + }); + resolve(path.basename(this.generatedpath)); + }); }); } + getMontantFormString(chaine) { + const regex = /(\d[\d.,]*)/g; + const matches = chaine.match(regex); + if (matches) { + const result= matches.map(match => { + const parts = match.split(/[.,]/); + if (parts.length > 1 && parts[parts.length - 1].length === 2) { + return match.replace(/[^\d.,]/g, '').replace(/[.,]/g, ','); + } else { + return match.replace(/[^\d.,]/g, '').replace(/[^\d]/g, '.'); + } + }); + return `${result.at(-1)} EUR` + } + return []; + } + /** * Generate a new file with the columns - * + * * @param object obj * @param string path * @return string */ getValueByPath(obj, path) { - if ( !path || typeof path !== 'string' ) { + if (!path || typeof path !== "string") { return obj; } - const parts = path.split('.'); + const parts = path.split("."); let result = obj; for (let part of parts) { // Add a check before accessing the property if (result[part]) { - result = result[part]; + if (part === "origineFonds") { + result = this.getMontantFormString(result[part]); + } else { + result = result[part]; + } } else { - return ''; + return ""; } } return result; } - - /** * Generate exported row - * + * * @param object row * @param object columnsIndex * @return array @@ -306,11 +402,15 @@ class File { for (let key in row) { try { - if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) { - row[key] = JSON.parse(row[key]); + if ( + typeof row[key] === "string" && + row[key].startsWith("{") && + row[key].endsWith("}") + ) { + row[key] = JSON.parse(row[key]); } - - row[key.trim()] = row[key] + + row[key.trim()] = row[key]; } catch (err) { // result.push(''); console.log(err.message); @@ -323,22 +423,22 @@ class File { } const column = columnsIndex[key]; - const item = row[column.main] || ''; + const item = row[column.main] || ""; - if ( column.primary === column.value ) { + if (column.primary === column.value) { result.push(item); } else { - if ( typeof item === 'object' ) { + if (typeof item === "object") { result.push(this.getValueByPath(item, column.rest)); } else { result.push(item); } } - } - return result.map((res)=>{ - return typeof res === "string" ? res.replace(/;/g, ',') : res + } + return result.map((res) => { + return typeof res === "string" ? res.replace(/;/g, ",") : res; }); } } -module.exports = File; \ No newline at end of file +module.exports = File; diff --git a/src/routes/index.js b/src/routes/index.js index 0ec79fe..dfd6d2b 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -133,6 +133,7 @@ router.get("/", async function (req, res, next) { columns: { options: [ "id", + "listeetablissements.etablissement.origineFonds", "publicationavis", "publicationavis_facette", "parution",