|
|
@ -5,7 +5,7 @@ const fs = require('fs'); |
|
|
const util = require('util'); |
|
|
const util = require('util'); |
|
|
const http = require('http'); |
|
|
const http = require('http'); |
|
|
const https = require('https'); |
|
|
const https = require('https'); |
|
|
const exec = util.promisify(require('child_process').exec); |
|
|
const exec = util.promisify(require('child_process').exec); |
|
|
|
|
|
|
|
|
const csvParser = require('csv-parser'); |
|
|
const csvParser = require('csv-parser'); |
|
|
const { PassThrough } = require('stream'); |
|
|
const { PassThrough } = require('stream'); |
|
|
@ -18,15 +18,6 @@ const { basedir } = require('../config/constants'); |
|
|
const dest = path.join(basedir, 'public/csv'); |
|
|
const dest = path.join(basedir, 'public/csv'); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Create a generateFilePath function witch returns a path with a filename and datetime
|
|
|
|
|
|
function generateFilePath(filename) { |
|
|
|
|
|
return { |
|
|
|
|
|
filepath: path.join(dest, `${filename}-${Date.now()}.csv`), |
|
|
|
|
|
generatedpath: path.join('csv', `${filename}-generated-${Date.now()}.csv`), |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Create a class File that extends EventEmitter
|
|
|
// Create a class File that extends EventEmitter
|
|
|
class File { |
|
|
class File { |
|
|
|
|
|
|
|
|
@ -37,8 +28,88 @@ class File { |
|
|
constructor(url) { |
|
|
constructor(url) { |
|
|
this.url = url; |
|
|
this.url = url; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
formatDateToCustomFormat(date) { |
|
|
|
|
|
// Obtenir les composantes de la date et de l'heure
|
|
|
|
|
|
const year = date.getFullYear(); |
|
|
|
|
|
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
|
|
|
|
|
|
const day = String(date.getDate()).padStart(2, '0'); |
|
|
|
|
|
const hours = String(date.getHours()).padStart(2, '0'); |
|
|
|
|
|
const minutes = String(date.getMinutes()).padStart(2, '0'); |
|
|
|
|
|
const seconds = String(date.getSeconds()).padStart(2, '0'); |
|
|
|
|
|
|
|
|
|
|
|
// Créer la chaîne au format personnalisé
|
|
|
|
|
|
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; |
|
|
|
|
|
|
|
|
|
|
|
return formattedDate; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Create a generateFilePath function witch returns a path with a filename and datetime
|
|
|
|
|
|
generateFilePath(filename) { |
|
|
|
|
|
const date = new Date(); |
|
|
|
|
|
return { |
|
|
|
|
|
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), |
|
|
|
|
|
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* Download a file from a url |
|
|
|
|
|
* @returns {Promise<string>} filepath |
|
|
|
|
|
*/ |
|
|
|
|
|
async fakeDownload() { |
|
|
|
|
|
const url = URL.parse(this.url); |
|
|
|
|
|
this.filename = slugify(url.hostname, { lower: true }); |
|
|
|
|
|
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); |
|
|
|
|
|
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); |
|
|
|
|
|
this.filepath = filepath; |
|
|
|
|
|
this.generatedpath = generatedpath; |
|
|
|
|
|
return new Promise((resolve, reject) => { |
|
|
|
|
|
resolve(filepath); |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
formatDateToCustomFormat(date) { |
|
|
|
|
|
// Obtenir les composantes de la date et de l'heure
|
|
|
|
|
|
const year = date.getFullYear(); |
|
|
|
|
|
const month = String(date.getMonth() + 1).padStart(2, '0'); // Janvier est 0, février est 1, etc.
|
|
|
|
|
|
const day = String(date.getDate()).padStart(2, '0'); |
|
|
|
|
|
const hours = String(date.getHours()).padStart(2, '0'); |
|
|
|
|
|
const minutes = String(date.getMinutes()).padStart(2, '0'); |
|
|
|
|
|
const seconds = String(date.getSeconds()).padStart(2, '0'); |
|
|
|
|
|
|
|
|
|
|
|
// Créer la chaîne au format personnalisé
|
|
|
|
|
|
const formattedDate = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`; |
|
|
|
|
|
|
|
|
|
|
|
return formattedDate; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Create a generateFilePath function witch returns a path with a filename and datetime
|
|
|
|
|
|
generateFilePath(filename) { |
|
|
|
|
|
const date = new Date(); |
|
|
|
|
|
return { |
|
|
|
|
|
filepath: path.join(dest, `${filename}-${this.formatDateToCustomFormat(date)}.csv`), |
|
|
|
|
|
generatedpath: path.join(dest, `${filename}-generated-${this.formatDateToCustomFormat(date)}.csv`), |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* Download a file from a url |
|
|
|
|
|
* @returns {Promise<string>} filepath |
|
|
|
|
|
*/ |
|
|
|
|
|
async fakeDownload() { |
|
|
|
|
|
const url = URL.parse(this.url); |
|
|
|
|
|
this.filename = slugify(url.hostname, { lower: true }); |
|
|
|
|
|
const filepath = path.join(dest, `bodacc-datadila.opendatasoft.com-1697536785170.csv`); |
|
|
|
|
|
const generatedpath = path.join(dest, `bodacc-datadila.opendatasoft.com-generated-1697536785170-2.csv`); |
|
|
|
|
|
this.filepath = filepath; |
|
|
|
|
|
this.generatedpath = generatedpath; |
|
|
|
|
|
return new Promise((resolve, reject) => { |
|
|
|
|
|
resolve(filepath); |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* Download a file from a url |
|
|
* Download a file from a url |
|
|
@ -47,11 +118,11 @@ class File { |
|
|
async download() { |
|
|
async download() { |
|
|
const url = URL.parse(this.url); |
|
|
const url = URL.parse(this.url); |
|
|
this.filename = slugify(url.hostname, { lower: true }); |
|
|
this.filename = slugify(url.hostname, { lower: true }); |
|
|
const { filepath, generatedpath } = generateFilePath(this.filename); |
|
|
const { filepath, generatedpath } = this.generateFilePath(this.filename); |
|
|
this.filepath = filepath; |
|
|
this.filepath = filepath; |
|
|
this.generatedpath = generatedpath; |
|
|
this.generatedpath = generatedpath; |
|
|
|
|
|
|
|
|
const file = fs.createWriteStream(this.filepath); |
|
|
const file = fs.createWriteStream(this.filepath); |
|
|
const request = url.protocol === 'https:' ? https : http; |
|
|
const request = url.protocol === 'https:' ? https : http; |
|
|
|
|
|
|
|
|
// Emit a download.start event with the url and filepath
|
|
|
// Emit a download.start event with the url and filepath
|
|
|
@ -121,67 +192,79 @@ class File { |
|
|
|
|
|
|
|
|
// create a parse method which read the file and return a stream
|
|
|
// create a parse method which read the file and return a stream
|
|
|
parse(columns) { |
|
|
parse(columns) { |
|
|
const stream = new PassThrough(); |
|
|
return new Promise((resolve, reject) => { |
|
|
|
|
|
const fileStream = fs.createWriteStream(this.generatedpath); |
|
|
|
|
|
|
|
|
|
|
|
// check if columns is valid
|
|
|
|
|
|
if (!columns || !columns.length) { |
|
|
|
|
|
// return Promise.reject(new Error('Invalid columns'));
|
|
|
|
|
|
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); |
|
|
|
|
|
reject(new Error('Invalid columns')); |
|
|
|
|
|
return false; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
// check if columns is valid
|
|
|
// Create a variable to hold csv columns indexes
|
|
|
if (!columns || !columns.length) { |
|
|
const columnsIndex = {}; |
|
|
// return Promise.reject(new Error('Invalid columns'));
|
|
|
for (let column of columns) { |
|
|
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: 'Invalid columns' }); |
|
|
columnsIndex[column] = { |
|
|
return false; |
|
|
exist: false, |
|
|
} |
|
|
main: (String(column)).split('.')[0], |
|
|
|
|
|
value: column, |
|
|
|
|
|
rest: (String(column)).split('.').slice(1).join('.'), |
|
|
|
|
|
last: (String(column)).split('.').pop(), |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
// Create a variable to hold csv columns indexes
|
|
|
const columnsFiltered = []; |
|
|
const columnsIndex = {}; |
|
|
|
|
|
for (let column of columns) { |
|
|
|
|
|
columnsIndex[column] = { |
|
|
|
|
|
exist: false, |
|
|
|
|
|
main: (String(column)).split('.')[0], |
|
|
|
|
|
value: column, |
|
|
|
|
|
rest: (String(column)).split('.').slice(1).join('.'), |
|
|
|
|
|
last: (String(column)).split('.').pop(), |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
const columnsFiltered = []; |
|
|
let count = 1; |
|
|
|
|
|
fs.createReadStream(this.filepath) |
|
|
let count = 1; |
|
|
.pipe(csvParser({ separator: ';' })) |
|
|
fs.createReadStream(this.filepath) |
|
|
.on('headers', (headers) => { |
|
|
.pipe(csvParser({ separator: ';' })) |
|
|
headers = headers.map(header => typeof header === 'string' ? header.trim() : header); |
|
|
.on('headers', (headers) => { |
|
|
|
|
|
headers = headers.map(header => typeof header === 'string' ? header.trim() : header); |
|
|
const result = []; |
|
|
|
|
|
for (let key in columnsIndex) { |
|
|
const result = []; |
|
|
columnsIndex[key].exist = headers.includes(columnsIndex[key].main); |
|
|
for (let key in columnsIndex) { |
|
|
if ( columnsIndex[key].exist ) { |
|
|
columnsIndex[key].exist = headers.includes(columnsIndex[key].main); |
|
|
columnsFiltered.push(columnsIndex[key].value); |
|
|
if ( columnsIndex[key].exist ) { |
|
|
if(columnsIndex[key].main ==='listeprecedentexploitant'){ |
|
|
columnsFiltered.push(columnsIndex[key].value); |
|
|
result.push(`Prec Exp ${columnsIndex[key].last}`) |
|
|
result.push(columnsIndex[key].last); |
|
|
}else if(columnsIndex[key].main ==='listeprecedentproprietaire'){ |
|
|
|
|
|
result.push(`Prec Prop ${columnsIndex[key].last}`) |
|
|
|
|
|
}else{ |
|
|
|
|
|
result.push(columnsIndex[key].last); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// Emit a parse.start event with the url and filepath
|
|
|
// Emit a parse.start event with the url and filepath
|
|
|
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); |
|
|
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers, result: result }); |
|
|
|
|
|
|
|
|
fileStream.write(result.join(';') + "\n"); |
|
|
stream.write(result.join(';') + "\n"); |
|
|
}) |
|
|
}) |
|
|
.on('data', (row) => { |
|
|
.on('data', (row) => { |
|
|
// Emit a parse.data event with the url, filepath and data
|
|
|
// Emit a parse.data event with the url, filepath and data
|
|
|
let result = this.processRow(row, columnsIndex, columnsFiltered); |
|
|
let result = this.processRow(row, columnsIndex, columnsFiltered); |
|
|
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); |
|
|
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, result, index: count }); |
|
|
fileStream.write(result.join(';') + "\n"); |
|
|
stream.write(result.join(';') + "\n"); |
|
|
count++; |
|
|
count++; |
|
|
}) |
|
|
}) |
|
|
.on('error', (err) => { |
|
|
.on('error', (err) => { |
|
|
// Emit a parse.error event with the error
|
|
|
// Emit a parse.error event with the error
|
|
|
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); |
|
|
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); |
|
|
|
|
|
}) |
|
|
|
|
|
.on('end', () => { |
|
|
|
|
|
// Emit a parse.end event with the url and filepath
|
|
|
|
|
|
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1 }); |
|
|
|
|
|
stream.end(); |
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
return stream; |
|
|
fileStream.close(); |
|
|
|
|
|
reject(err); |
|
|
|
|
|
fs.unlink(this.generatedpath, () => {}); |
|
|
|
|
|
}) |
|
|
|
|
|
.on('end', () => { |
|
|
|
|
|
// Emit a parse.end event with the url and filepath
|
|
|
|
|
|
fileStream.close(); |
|
|
|
|
|
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: count - 1, generated: path.basename(this.generatedpath) }); |
|
|
|
|
|
resolve(path.basename(this.generatedpath)); |
|
|
|
|
|
}); |
|
|
|
|
|
}); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -223,8 +306,8 @@ class File { |
|
|
|
|
|
|
|
|
for (let key in row) { |
|
|
for (let key in row) { |
|
|
try { |
|
|
try { |
|
|
if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) { |
|
|
if ( typeof row[key] === 'string' && row[key].startsWith('{') && row[key].endsWith('}') ) { |
|
|
row[key] = JSON.parse(row[key]); |
|
|
row[key] = JSON.parse(row[key]); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
row[key.trim()] = row[key] |
|
|
row[key.trim()] = row[key] |
|
|
@ -240,7 +323,7 @@ class File { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
const column = columnsIndex[key]; |
|
|
const column = columnsIndex[key]; |
|
|
const item = row[column.main] || ''; |
|
|
const item = row[column.main] || ''; |
|
|
|
|
|
|
|
|
if ( column.primary === column.value ) { |
|
|
if ( column.primary === column.value ) { |
|
|
result.push(item); |
|
|
result.push(item); |
|
|
@ -251,9 +334,8 @@ class File { |
|
|
result.push(item); |
|
|
result.push(item); |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
return result.map((res)=>res.replace(/;/g, ',')); |
|
|
return result; |
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
