onja
/
bodacc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge pull request 'fix: montant achat' (#8) from fix/montant-achat into master 

Reviewed-on: #8
master
diary 5 months ago
parent
2a9d3dec3a a24de25583
commit
90a9b69907
2 changed files with 258 additions and 157 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 414
      src/models/file.js
  2. 1
      src/routes/index.js

414
src/models/file.js
View File

@ -1,55 +1,59 @@
/* global basedir */
const path = require('path');
const path = require("path");
const URL = require("url");
const fs = require('fs');
const util = require('util');
const http = require('http');
const https = require('https');
const exec = util.promisify(require('child_process').exec);
const fs = require("fs");
const util = require("util");
const http = require("http");
const https = require("https");
const exec = util.promisify(require("child_process").exec);
const csvParser = require('csv-parser');
const { PassThrough } = require('stream');
const slugify = require('slugify');
const emitter = require('../services/eventEmitter');
const csvParser = require("csv-parser");
const { PassThrough } = require("stream");
const slugify = require("slugify");
const emitter = require("../services/eventEmitter");
// Load project path from config/constants.js
const { basedir } = require('../config/constants');
const dest = path.join(basedir, 'public/csv');
const { basedir } = require("../config/constants");
const dest = path.join(basedir, "public/csv");
// Create a class File that extends EventEmitter
class File {
/**
* Create a new File instance
* @param {string} url
* @param {string} url
*/
constructor(url) {
this.url = url;
}
formatDateToCustomFormat(date) {
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate;
}
// Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
filepath: path.join(
dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
};
}
@ -60,37 +64,48 @@ class File {
async fakeDownload() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
const filepath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath;
this.generatedpath = generatedpath;
return new Promise((resolve, reject) => {
resolve(filepath);
});
}
formatDateToCustomFormat(date) {
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
return formattedDate;
}
// Create a generateFilePath function witch returns a path with a filename and datetime
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
filepath: path.join(
dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
};
}
@ -101,15 +116,20 @@ class File {
async fakeDownload() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
const filepath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath;
this.generatedpath = generatedpath;
return new Promise((resolve, reject) => {
resolve(filepath);
});
}
/**
* Download a file from a url
@ -122,74 +142,101 @@ class File {
this.filepath = filepath;
this.generatedpath = generatedpath;
const file = fs.createWriteStream(this.filepath);
const request = url.protocol === 'https:' ? https : http;
const file = fs.createWriteStream(this.filepath);
const request = url.protocol === "https:" ? https : http;
// Emit a download.start event with the url and filepath
emitter.emit('download.start', { url: this.url, filepath: this.filepath });
emitter.emit("download.start", { url: this.url, filepath: this.filepath });
return new Promise((resolve, reject) => {
// Handle file errors
file.on('error', (err) => {
file.on("error", (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' });
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "file",
});
});
return reject(err.message);
});
request
.get(url.href, (response) => {
// Check if response is valid
if (response.statusCode !== 200) {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' });
return reject(response.statusMessage);
}
emitter.emit('download.started', { url: this.url, filepath: this.filepath });
// Create a variable to hold the downloaded size
let downloaded = 0;
// Handle response data
let i = 0;
response.on('data', (chunk) => {
downloaded += chunk.length;
i++;
// Check if the downloaded size is bigger than the step limit
if (i === 500) {
// Emit a download.progress event with the url, filepath and downloaded size
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded });
i = 0;
.get(url.href, (response) => {
// Check if response is valid
if (response.statusCode !== 200) {
// Emit a download.error event with the error
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: response.statusMessage,
type: "response",
});
return reject(response.statusMessage);
}
});
response.pipe(file);
emitter.emit("download.started", {
url: this.url,
filepath: this.filepath,
});
file.on('finish', () => {
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded });
// Create a variable to hold the downloaded size
let downloaded = 0;
// Handle response data
let i = 0;
response.on("data", (chunk) => {
downloaded += chunk.length;
i++;
// Check if the downloaded size is bigger than the step limit
if (i === 500) {
// Emit a download.progress event with the url, filepath and downloaded size
emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
i = 0;
}
});
// Emit a download.end event with the url and filepath
emitter.emit('download.end', { url: this.url, filepath: this.filepath });
file.close(() => {
resolve(filepath);
response.pipe(file);
file.on("finish", () => {
emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
// Emit a download.end event with the url and filepath
emitter.emit("download.end", {
url: this.url,
filepath: this.filepath,
});
file.close(() => {
resolve(filepath);
});
});
})
.on("error", (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "request",
});
reject(err.message);
});
});
})
.on('error', (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' });
reject(err.message);
});
});
});
}
// create a parse method which read the file and return a stream
parse(columns) {
return new Promise((resolve, reject) => {
@ -198,8 +245,12 @@ class File {
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
reject(new Error('Invalid columns'));
emitter.emit("parse.error", {
url: this.url,
filepath: this.filepath,
error: "Invalid columns",
});
reject(new Error("Invalid columns"));
return false;
}
@ -208,10 +259,10 @@ class File {
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: (String(column)).split('.')[0],
main: String(column).split(".")[0],
value: column,
rest: (String(column)).split('.').slice(1).join('.'),
last: (String(column)).split('.').pop(),
rest: String(column).split(".").slice(1).join("."),
last: String(column).split(".").pop(),
};
}
@ -219,84 +270,129 @@ class File {
let count = 1;
fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' }))
.on('headers', (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if ( columnsIndex[key].exist ) {
columnsFiltered.push(columnsIndex[key].value);
if(columnsIndex[key].main ==='listeprecedentexploitant'){
result.push(`Prec Exp ${columnsIndex[key].last}`)
}else if(columnsIndex[key].main ==='listeprecedentproprietaire'){
result.push(`Prec Prop ${columnsIndex[key].last}`)
}else{
result.push(columnsIndex[key].last);
.pipe(csvParser({ separator: ";" }))
.on("headers", (headers) => {
headers = headers.map((header) =>
typeof header === "string" ? header.trim() : header
);
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if (columnsIndex[key].exist) {
columnsFiltered.push(columnsIndex[key].value);
if (columnsIndex[key].main === "listeprecedentexploitant") {
result.push(`Prec Exp ${columnsIndex[key].last}`);
} else if (
columnsIndex[key].main === "listeprecedentproprietaire"
) {
result.push(`Prec Prop ${columnsIndex[key].last}`);
} else if (columnsIndex[key].last === "origineFonds") {
result.push("Montant Achat");
} else {
result.push(columnsIndex[key].last);
}
}
}
}
// Emit a parse.start event with the url and filepath
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
fileStream.write(result.join(';') + "\n");
})
.on('data', (row) => {
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered);
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
fileStream.write(result.join(';') + "\n");
count++;
})
.on('error', (err) => {
// Emit a parse.error event with the error
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => {});
})
.on('end', () => {
// Emit a parse.end event with the url and filepath
fileStream.close();
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
resolve(path.basename(this.generatedpath));
});
// Emit a parse.start event with the url and filepath
emitter.emit("parse.start", {
url: this.url,
filepath: this.filepath,
headers,
result: result,
});
fileStream.write(result.join(";") + "\n");
})
.on("data", (row) => {
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered);
emitter.emit("parse.data", {
url: this.url,
filepath: this.filepath,
data: row,
result,
index: count,
});
fileStream.write(result.join(";") + "\n");
count++;
})
.on("error", (err) => {
// Emit a parse.error event with the error
emitter.emit("parse.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
});
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => {});
})
.on("end", () => {
// Emit a parse.end event with the url and filepath
fileStream.close();
emitter.emit("parse.end", {
url: this.url,
filepath: this.filepath,
count: count - 1,
generated: path.basename(this.generatedpath),
});
resolve(path.basename(this.generatedpath));
});
});
}
getMontantFormString(chaine) {
const regex = /(\d[\d.,]*)/g;
const matches = chaine.match(regex);
if (matches) {
const result= matches.map(match => {
const parts = match.split(/[.,]/);
if (parts.length > 1 && parts[parts.length - 1].length === 2) {
return match.replace(/[^\d.,]/g, '').replace(/[.,]/g, ',');
} else {
return match.replace(/[^\d.,]/g, '').replace(/[^\d]/g, '.');
}
});
return `${result.at(-1)} EUR`
}
return [];
}
/**
* Generate a new file with the columns
*
*
* @param object obj
* @param string path
* @return string
*/
getValueByPath(obj, path) {
if ( !path || typeof path !== 'string' ) {
if (!path || typeof path !== "string") {
return obj;
}
const parts = path.split('.');
const parts = path.split(".");
let result = obj;
for (let part of parts) {
// Add a check before accessing the property
if (result[part]) {
result = result[part];
if (part === "origineFonds") {
result = this.getMontantFormString(result[part]);
} else {
result = result[part];
}
} else {
return '';
return "";
}
}
return result;
}
/**
* Generate exported row
*
*
* @param object row
* @param object columnsIndex
* @return array
@ -306,11 +402,15 @@ class File {
for (let key in row) {
try {
if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) {
row[key] = JSON.parse(row[key]);
if (
typeof row[key] === "string" &&
row[key].startsWith("{") &&
row[key].endsWith("}")
) {
row[key] = JSON.parse(row[key]);
}
row[key.trim()] = row[key]
row[key.trim()] = row[key];
} catch (err) {
// result.push('');
console.log(err.message);
@ -323,22 +423,22 @@ class File {
}
const column = columnsIndex[key];
const item = row[column.main] || '';
const item = row[column.main] || "";
if ( column.primary === column.value ) {
if (column.primary === column.value) {
result.push(item);
} else {
if ( typeof item === 'object' ) {
if (typeof item === "object") {
result.push(this.getValueByPath(item, column.rest));
} else {
result.push(item);
}
}
}
return result.map((res)=>{
return typeof res === "string" ? res.replace(/;/g, ',') : res
}
return result.map((res) => {
return typeof res === "string" ? res.replace(/;/g, ",") : res;
});
}
}
module.exports = File;
module.exports = File;

1
src/routes/index.js
View File

@ -133,6 +133,7 @@ router.get("/", async function (req, res, next) {
columns: {
options: [
"id",
"listeetablissements.etablissement.origineFonds",
"publicationavis",
"publicationavis_facette",
"parution",

Write Preview
Loading…
Cancel
Save