onja
/
bodacc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge pull request 'fix: montant achat' (#8) from fix/montant-achat into master 

Reviewed-on: #8
master
diary 5 months ago
parent
2a9d3dec3a a24de25583
commit
90a9b69907
2 changed files with 258 additions and 157 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 414
      src/models/file.js
  2. 1
      src/routes/index.js

414
src/models/file.js
View File

@ -1,55 +1,59 @@
/* global basedir */ /* global basedir */
const path = require('path'); const path = require("path");
const URL = require("url"); const URL = require("url");
const fs = require('fs'); const fs = require("fs");
const util = require('util'); const util = require("util");
const http = require('http'); const http = require("http");
const https = require('https'); const https = require("https");
const exec = util.promisify(require('child_process').exec); const exec = util.promisify(require("child_process").exec);
const csvParser = require('csv-parser'); const csvParser = require("csv-parser");
const { PassThrough } = require('stream'); const { PassThrough } = require("stream");
const slugify = require('slugify'); const slugify = require("slugify");
const emitter = require('../services/eventEmitter'); const emitter = require("../services/eventEmitter");
// Load project path from config/constants.js // Load project path from config/constants.js
const { basedir } = require('../config/constants'); const { basedir } = require("../config/constants");
const dest = path.join(basedir, 'public/csv');
const dest = path.join(basedir, "public/csv");
// Create a class File that extends EventEmitter // Create a class File that extends EventEmitter
class File { class File {
/** /**
* Create a new File instance * Create a new File instance
* @param {string} url * @param {string} url
*/ */
constructor(url) { constructor(url) {
this.url = url; this.url = url;
} }
formatDateToCustomFormat(date) { formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure // Obtenir les composantes de la date et de l'heure
const year = date.getFullYear(); const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc. const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0'); const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, '0'); const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, '0'); const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, '0'); const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé // Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate; return formattedDate;
} }
// Create a generateFilePath function witch returns a path with a filename and datetime // Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) { generateFilePath(filename) {
const date = new Date(); const date = new Date();
return { return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), filepath: path.join(
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
}; };
} }
@ -60,37 +64,48 @@ class File {
async fakeDownload() { async fakeDownload() {
const url = URL.parse(this.url); const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true }); this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); const filepath = path.join(
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath; this.filepath = filepath;
this.generatedpath = generatedpath; this.generatedpath = generatedpath;
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
resolve(filepath); resolve(filepath);
}); });
} }
formatDateToCustomFormat(date) { formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure // Obtenir les composantes de la date et de l'heure
const year = date.getFullYear(); const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc. const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0'); const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, '0'); const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, '0'); const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, '0'); const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé // Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate; return formattedDate;
} }
// Create a generateFilePath function witch returns a path with a filename and datetime // Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) { generateFilePath(filename) {
const date = new Date(); const date = new Date();
return { return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), filepath: path.join(
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
}; };
} }
@ -101,15 +116,20 @@ class File {
async fakeDownload() { async fakeDownload() {
const url = URL.parse(this.url); const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true }); this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); const filepath = path.join(
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath; this.filepath = filepath;
this.generatedpath = generatedpath; this.generatedpath = generatedpath;
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
resolve(filepath); resolve(filepath);
}); });
} }
/** /**
* Download a file from a url * Download a file from a url
@ -122,74 +142,101 @@ class File {
this.filepath = filepath; this.filepath = filepath;
this.generatedpath = generatedpath; this.generatedpath = generatedpath;
const file = fs.createWriteStream(this.filepath); const file = fs.createWriteStream(this.filepath);
const request = url.protocol === 'https:' ? https : http; const request = url.protocol === "https:" ? https : http;
// Emit a download.start event with the url and filepath // Emit a download.start event with the url and filepath
emitter.emit('download.start', { url: this.url, filepath: this.filepath }); emitter.emit("download.start", { url: this.url, filepath: this.filepath });
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// Handle file errors // Handle file errors
file.on('error', (err) => { file.on("error", (err) => {
fs.unlink(filepath, () => { fs.unlink(filepath, () => {
// Emit a download.error event with the error // Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' }); emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "file",
});
}); });
return reject(err.message); return reject(err.message);
}); });
request request
.get(url.href, (response) => { .get(url.href, (response) => {
// Check if response is valid // Check if response is valid
if (response.statusCode !== 200) { if (response.statusCode !== 200) {
// Emit a download.error event with the error // Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' }); emitter.emit("download.error", {
return reject(response.statusMessage); url: this.url,
} filepath: this.filepath,
error: response.statusMessage,
type: "response",
emitter.emit('download.started', { url: this.url, filepath: this.filepath }); });
return reject(response.statusMessage);
// Create a variable to hold the downloaded size
let downloaded = 0;
// Handle response data
let i = 0;
response.on('data', (chunk) => {
downloaded += chunk.length;
i++;
// Check if the downloaded size is bigger than the step limit
if (i === 500) {
// Emit a download.progress event with the url, filepath and downloaded size
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded });
i = 0;
} }
});
response.pipe(file); emitter.emit("download.started", {
url: this.url,
filepath: this.filepath,
});
file.on('finish', () => { // Create a variable to hold the downloaded size
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded }); let downloaded = 0;
// Handle response data
let i = 0;
response.on("data", (chunk) => {
downloaded += chunk.length;
i++;
// Check if the downloaded size is bigger than the step limit
if (i === 500) {
// Emit a download.progress event with the url, filepath and downloaded size
emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
i = 0;
}
});
// Emit a download.end event with the url and filepath response.pipe(file);
emitter.emit('download.end', { url: this.url, filepath: this.filepath });
file.close(() => { file.on("finish", () => {
resolve(filepath); emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
// Emit a download.end event with the url and filepath
emitter.emit("download.end", {
url: this.url,
filepath: this.filepath,
});
file.close(() => {
resolve(filepath);
});
});
})
.on("error", (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "request",
});
reject(err.message);
}); });
}); });
})
.on('error', (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' });
reject(err.message);
});
});
}); });
} }
// create a parse method which read the file and return a stream // create a parse method which read the file and return a stream
parse(columns) { parse(columns) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -198,8 +245,12 @@ class File {
// check if columns is valid // check if columns is valid
if (!columns || !columns.length) { if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns')); // return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); emitter.emit("parse.error", {
reject(new Error('Invalid columns')); url: this.url,
filepath: this.filepath,
error: "Invalid columns",
});
reject(new Error("Invalid columns"));
return false; return false;
} }
@ -208,10 +259,10 @@ class File {
for (let column of columns) { for (let column of columns) {
columnsIndex[column] = { columnsIndex[column] = {
exist: false, exist: false,
main: (String(column)).split('.')[0], main: String(column).split(".")[0],
value: column, value: column,
rest: (String(column)).split('.').slice(1).join('.'), rest: String(column).split(".").slice(1).join("."),
last: (String(column)).split('.').pop(), last: String(column).split(".").pop(),
}; };
} }
@ -219,84 +270,129 @@ class File {
let count = 1; let count = 1;
fs.createReadStream(this.filepath) fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' })) .pipe(csvParser({ separator: ";" }))
.on('headers', (headers) => { .on("headers", (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header); headers = headers.map((header) =>
typeof header === "string" ? header.trim() : header
const result = []; );
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main); const result = [];
if ( columnsIndex[key].exist ) { for (let key in columnsIndex) {
columnsFiltered.push(columnsIndex[key].value); columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if(columnsIndex[key].main ==='listeprecedentexploitant'){ if (columnsIndex[key].exist) {
result.push(`Prec Exp ${columnsIndex[key].last}`) columnsFiltered.push(columnsIndex[key].value);
}else if(columnsIndex[key].main ==='listeprecedentproprietaire'){ if (columnsIndex[key].main === "listeprecedentexploitant") {
result.push(`Prec Prop ${columnsIndex[key].last}`) result.push(`Prec Exp ${columnsIndex[key].last}`);
}else{ } else if (
result.push(columnsIndex[key].last); columnsIndex[key].main === "listeprecedentproprietaire"
) {
result.push(`Prec Prop ${columnsIndex[key].last}`);
} else if (columnsIndex[key].last === "origineFonds") {
result.push("Montant Achat");
} else {
result.push(columnsIndex[key].last);
}
} }
} }
}
// Emit a parse.start event with the url and filepath
// Emit a parse.start event with the url and filepath emitter.emit("parse.start", {
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); url: this.url,
filepath: this.filepath,
fileStream.write(result.join(';') + "\n"); headers,
}) result: result,
.on('data', (row) => { });
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered); fileStream.write(result.join(";") + "\n");
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); })
fileStream.write(result.join(';') + "\n"); .on("data", (row) => {
count++; // Emit a parse.data event with the url, filepath and data
}) let result = this.processRow(row, columnsIndex, columnsFiltered);
.on('error', (err) => { emitter.emit("parse.data", {
// Emit a parse.error event with the error url: this.url,
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); filepath: this.filepath,
data: row,
fileStream.close(); result,
reject(err); index: count,
fs.unlink(this.generatedpath, () => {}); });
}) fileStream.write(result.join(";") + "\n");
.on('end', () => { count++;
// Emit a parse.end event with the url and filepath })
fileStream.close(); .on("error", (err) => {
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) }); // Emit a parse.error event with the error
resolve(path.basename(this.generatedpath)); emitter.emit("parse.error", {
}); url: this.url,
filepath: this.filepath,
error: err.message,
});
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => {});
})
.on("end", () => {
// Emit a parse.end event with the url and filepath
fileStream.close();
emitter.emit("parse.end", {
url: this.url,
filepath: this.filepath,
count: count - 1,
generated: path.basename(this.generatedpath),
});
resolve(path.basename(this.generatedpath));
});
}); });
} }
getMontantFormString(chaine) {
const regex = /(\d[\d.,]*)/g;
const matches = chaine.match(regex);
if (matches) {
const result= matches.map(match => {
const parts = match.split(/[.,]/);
if (parts.length > 1 && parts[parts.length - 1].length === 2) {
return match.replace(/[^\d.,]/g, '').replace(/[.,]/g, ',');
} else {
return match.replace(/[^\d.,]/g, '').replace(/[^\d]/g, '.');
}
});
return `${result.at(-1)} EUR`
}
return [];
}
/** /**
* Generate a new file with the columns * Generate a new file with the columns
* *
* @param object obj * @param object obj
* @param string path * @param string path
* @return string * @return string
*/ */
getValueByPath(obj, path) { getValueByPath(obj, path) {
if ( !path || typeof path !== 'string' ) { if (!path || typeof path !== "string") {
return obj; return obj;
} }
const parts = path.split('.'); const parts = path.split(".");
let result = obj; let result = obj;
for (let part of parts) { for (let part of parts) {
// Add a check before accessing the property // Add a check before accessing the property
if (result[part]) { if (result[part]) {
result = result[part]; if (part === "origineFonds") {
result = this.getMontantFormString(result[part]);
} else {
result = result[part];
}
} else { } else {
return ''; return "";
} }
} }
return result; return result;
} }
/** /**
* Generate exported row * Generate exported row
* *
* @param object row * @param object row
* @param object columnsIndex * @param object columnsIndex
* @return array * @return array
@ -306,11 +402,15 @@ class File {
for (let key in row) { for (let key in row) {
try { try {
if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) { if (
row[key] = JSON.parse(row[key]); typeof row[key] === "string" &&
row[key].startsWith("{") &&
row[key].endsWith("}")
) {
row[key] = JSON.parse(row[key]);
} }
row[key.trim()] = row[key] row[key.trim()] = row[key];
} catch (err) { } catch (err) {
// result.push(''); // result.push('');
console.log(err.message); console.log(err.message);
@ -323,22 +423,22 @@ class File {
} }
const column = columnsIndex[key]; const column = columnsIndex[key];
const item = row[column.main] || ''; const item = row[column.main] || "";
if ( column.primary === column.value ) { if (column.primary === column.value) {
result.push(item); result.push(item);
} else { } else {
if ( typeof item === 'object' ) { if (typeof item === "object") {
result.push(this.getValueByPath(item, column.rest)); result.push(this.getValueByPath(item, column.rest));
} else { } else {
result.push(item); result.push(item);
} }
} }
} }
return result.map((res)=>{ return result.map((res) => {
return typeof res === "string" ? res.replace(/;/g, ',') : res return typeof res === "string" ? res.replace(/;/g, ",") : res;
}); });
} }
} }
module.exports = File; module.exports = File;

1
src/routes/index.js
View File

@ -133,6 +133,7 @@ router.get("/", async function (req, res, next) {
columns: { columns: {
options: [ options: [
"id", "id",
"listeetablissements.etablissement.origineFonds",
"publicationavis", "publicationavis",
"publicationavis_facette", "publicationavis_facette",
"parution", "parution",

Write Preview
Loading…
Cancel
Save