onja
/
bodacc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: onja:2a9d3dec3a77ccf64389030f6cc005cbd1d02050
Branches Tags
onja:fix-feed
onja:fix/feedback
onja:master
onja:node10
onja:node16
...
compare: onja:90a9b6990720549be8495483e61f373838da45f6
Branches Tags
onja:fix-feed
onja:fix/feedback
onja:master
onja:node10
onja:node16

2 Commits
2a9d3dec3a ... 90a9b69907

Author SHA1 Message Date
diary 90a9b69907 Merge pull request 'fix: montant achat' (#8) from fix/montant-achat into master
7 months ago
Diary Narindra a24de25583 fix: montant achat
7 months ago
2 changed files with 258 additions and 157 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats
  1. 260
      src/models/file.js
  2. 1
      src/routes/index.js

260
src/models/file.js
View File

@ -1,26 +1,24 @@
/* global basedir */
const path = require('path');
const path = require("path");
const URL = require("url");
const fs = require('fs');
const util = require('util');
const http = require('http');
const https = require('https');
const exec = util.promisify(require('child_process').exec);
const fs = require("fs");
const util = require("util");
const http = require("http");
const https = require("https");
const exec = util.promisify(require("child_process").exec);
const csvParser = require('csv-parser');
const { PassThrough } = require('stream');
const slugify = require('slugify');
const emitter = require('../services/eventEmitter');
const csvParser = require("csv-parser");
const { PassThrough } = require("stream");
const slugify = require("slugify");
const emitter = require("../services/eventEmitter");
// Load project path from config/constants.js
const { basedir } = require('../config/constants');
const dest = path.join(basedir, 'public/csv');
const { basedir } = require("../config/constants");
const dest = path.join(basedir, "public/csv");
// Create a class File that extends EventEmitter
class File {
/**
* Create a new File instance
* @param {string} url
@ -32,11 +30,11 @@ class File {
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
@ -48,8 +46,14 @@ class File {
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
filepath: path.join(
dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
};
}
@ -60,8 +64,14 @@ class File {
async fakeDownload() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
const filepath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath;
this.generatedpath = generatedpath;
return new Promise((resolve, reject) => {
@ -69,15 +79,14 @@ class File {
});
}
formatDateToCustomFormat(date) {
// Obtenir les composantes de la date et de l'heure
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
const month = String(date.getMonth() + 1).padStart(2, "0"); // Janvier est 0, février est 1, etc.
const day = String(date.getDate()).padStart(2, "0");
const hours = String(date.getHours()).padStart(2, "0");
const minutes = String(date.getMinutes()).padStart(2, "0");
const seconds = String(date.getSeconds()).padStart(2, "0");
// Créer la chaîne au format personnalisé
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
@ -89,8 +98,14 @@ class File {
generateFilePath(filename) {
const date = new Date();
return {
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`),
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`),
filepath: path.join(
dest,
`${filename}-${this.formatDateToCustomFormat(date)}.csv`
),
generatedpath: path.join(
dest,
`${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`
),
};
}
@ -101,8 +116,14 @@ class File {
async fakeDownload() {
const url = URL.parse(this.url);
this.filename = slugify(url.hostname, { lower: true });
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`);
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`);
const filepath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-1697536785170.csv`
);
const generatedpath = path.join(
dest,
`bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`
);
this.filepath = filepath;
this.generatedpath = generatedpath;
return new Promise((resolve, reject) => {
@ -110,7 +131,6 @@ class File {
});
}
/**
* Download a file from a url
* @returns {Promise<string>} filepath
@ -123,17 +143,22 @@ class File {
this.generatedpath = generatedpath;
const file = fs.createWriteStream(this.filepath);
const request = url.protocol === 'https:' ? https : http;
const request = url.protocol === "https:" ? https : http;
// Emit a download.start event with the url and filepath
emitter.emit('download.start', { url: this.url, filepath: this.filepath });
emitter.emit("download.start", { url: this.url, filepath: this.filepath });
return new Promise((resolve, reject) => {
// Handle file errors
file.on('error', (err) => {
file.on("error", (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' });
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "file",
});
});
return reject(err.message);
});
@ -143,53 +168,75 @@ class File {
// Check if response is valid
if (response.statusCode !== 200) {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' });
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: response.statusMessage,
type: "response",
});
return reject(response.statusMessage);
}
emitter.emit('download.started', { url: this.url, filepath: this.filepath });
emitter.emit("download.started", {
url: this.url,
filepath: this.filepath,
});
// Create a variable to hold the downloaded size
let downloaded = 0;
// Handle response data
let i = 0;
response.on('data', (chunk) => {
response.on("data", (chunk) => {
downloaded += chunk.length;
i++;
// Check if the downloaded size is bigger than the step limit
if (i === 500) {
// Emit a download.progress event with the url, filepath and downloaded size
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded });
emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
i = 0;
}
});
response.pipe(file);
file.on('finish', () => {
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, downloaded });
file.on("finish", () => {
emitter.emit("download.progress", {
url: this.url,
filepath: this.filepath,
downloaded,
});
// Emit a download.end event with the url and filepath
emitter.emit('download.end', { url: this.url, filepath: this.filepath });
emitter.emit("download.end", {
url: this.url,
filepath: this.filepath,
});
file.close(() => {
resolve(filepath);
});
});
})
.on('error', (err) => {
.on("error", (err) => {
fs.unlink(filepath, () => {
// Emit a download.error event with the error
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' });
emitter.emit("download.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
type: "request",
});
reject(err.message);
});
});
});
}
// create a parse method which read the file and return a stream
parse(columns) {
return new Promise((resolve, reject) => {
@ -198,8 +245,12 @@ class File {
// check if columns is valid
if (!columns || !columns.length) {
// return Promise.reject(new Error('Invalid columns'));
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' });
reject(new Error('Invalid columns'));
emitter.emit("parse.error", {
url: this.url,
filepath: this.filepath,
error: "Invalid columns",
});
reject(new Error("Invalid columns"));
return false;
}
@ -208,10 +259,10 @@ class File {
for (let column of columns) {
columnsIndex[column] = {
exist: false,
main: (String(column)).split('.')[0],
main: String(column).split(".")[0],
value: column,
rest: (String(column)).split('.').slice(1).join('.'),
last: (String(column)).split('.').pop(),
rest: String(column).split(".").slice(1).join("."),
last: String(column).split(".").pop(),
};
}
@ -219,19 +270,25 @@ class File {
let count = 1;
fs.createReadStream(this.filepath)
.pipe(csvParser({ separator: ';' }))
.on('headers', (headers) => {
headers = headers.map(header => typeof header === 'string' ? header.trim() : header);
.pipe(csvParser({ separator: ";" }))
.on("headers", (headers) => {
headers = headers.map((header) =>
typeof header === "string" ? header.trim() : header
);
const result = [];
for (let key in columnsIndex) {
columnsIndex[key].exist = headers.includes(columnsIndex[key].main);
if (columnsIndex[key].exist) {
columnsFiltered.push(columnsIndex[key].value);
if(columnsIndex[key].main ==='listeprecedentexploitant'){
result.push(`Prec Exp ${columnsIndex[key].last}`)
}else if(columnsIndex[key].main ==='listeprecedentproprietaire'){
result.push(`Prec Prop ${columnsIndex[key].last}`)
if (columnsIndex[key].main === "listeprecedentexploitant") {
result.push(`Prec Exp ${columnsIndex[key].last}`);
} else if (
columnsIndex[key].main === "listeprecedentproprietaire"
) {
result.push(`Prec Prop ${columnsIndex[key].last}`);
} else if (columnsIndex[key].last === "origineFonds") {
result.push("Montant Achat");
} else {
result.push(columnsIndex[key].last);
}
@ -239,34 +296,71 @@ class File {
}
// Emit a parse.start event with the url and filepath
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result });
emitter.emit("parse.start", {
url: this.url,
filepath: this.filepath,
headers,
result: result,
});
fileStream.write(result.join(';') + "\n");
fileStream.write(result.join(";") + "\n");
})
.on('data', (row) => {
.on("data", (row) => {
// Emit a parse.data event with the url, filepath and data
let result = this.processRow(row, columnsIndex, columnsFiltered);
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count });
fileStream.write(result.join(';') + "\n");
emitter.emit("parse.data", {
url: this.url,
filepath: this.filepath,
data: row,
result,
index: count,
});
fileStream.write(result.join(";") + "\n");
count++;
})
.on('error', (err) => {
.on("error", (err) => {
// Emit a parse.error event with the error
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message });
emitter.emit("parse.error", {
url: this.url,
filepath: this.filepath,
error: err.message,
});
fileStream.close();
reject(err);
fs.unlink(this.generatedpath, () => {});
})
.on('end', () => {
.on("end", () => {
// Emit a parse.end event with the url and filepath
fileStream.close();
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) });
emitter.emit("parse.end", {
url: this.url,
filepath: this.filepath,
count: count - 1,
generated: path.basename(this.generatedpath),
});
resolve(path.basename(this.generatedpath));
});
});
}
getMontantFormString(chaine) {
const regex = /(\d[\d.,]*)/g;
const matches = chaine.match(regex);
if (matches) {
const result= matches.map(match => {
const parts = match.split(/[.,]/);
if (parts.length > 1 && parts[parts.length - 1].length === 2) {
return match.replace(/[^\d.,]/g, '').replace(/[.,]/g, ',');
} else {
return match.replace(/[^\d.,]/g, '').replace(/[^\d]/g, '.');
}
});
return `${result.at(-1)} EUR`
}
return [];
}
/**
* Generate a new file with the columns
*
@ -275,25 +369,27 @@ class File {
* @return string
*/
getValueByPath(obj, path) {
if ( !path || typeof path !== 'string' ) {
if (!path || typeof path !== "string") {
return obj;
}
const parts = path.split('.');
const parts = path.split(".");
let result = obj;
for (let part of parts) {
// Add a check before accessing the property
if (result[part]) {
if (part === "origineFonds") {
result = this.getMontantFormString(result[part]);
} else {
result = result[part];
}
} else {
return '';
return "";
}
}
return result;
}
/**
* Generate exported row
*
@ -306,11 +402,15 @@ class File {
for (let key in row) {
try {
if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) {
if (
typeof row[key] === "string" &&
row[key].startsWith("{") &&
row[key].endsWith("}")
) {
row[key] = JSON.parse(row[key]);
}
row[key.trim()] = row[key]
row[key.trim()] = row[key];
} catch (err) {
// result.push('');
console.log(err.message);
@ -323,12 +423,12 @@ class File {
}
const column = columnsIndex[key];
const item = row[column.main] || '';
const item = row[column.main] || "";
if (column.primary === column.value) {
result.push(item);
} else {
if ( typeof item === 'object' ) {
if (typeof item === "object") {
result.push(this.getValueByPath(item, column.rest));
} else {
result.push(item);
@ -336,7 +436,7 @@ class File {
}
}
return result.map((res) => {
return typeof res === "string" ? res.replace(/;/g, ',') : res
return typeof res === "string" ? res.replace(/;/g, ",") : res;
});
}
}

1
src/routes/index.js
View File

@ -133,6 +133,7 @@ router.get("/", async function (req, res, next) {
columns: {
options: [
"id",
"listeetablissements.etablissement.origineFonds",
"publicationavis",
"publicationavis_facette",
"parution",

Write Preview
Loading…
Cancel
Save