|
@ -7,11 +7,10 @@ const http = require('http'); |
|
|
const https = require('https'); |
|
|
const https = require('https'); |
|
|
const exec = util.promisify(require('child_process').exec); |
|
|
const exec = util.promisify(require('child_process').exec); |
|
|
|
|
|
|
|
|
const EventEmitter = require('events'); |
|
|
|
|
|
const csvParser = require('csv-parser'); |
|
|
const csvParser = require('csv-parser'); |
|
|
const { PassThrough } = require('stream'); |
|
|
const { PassThrough } = require('stream'); |
|
|
const slugify = require('slugify'); |
|
|
const slugify = require('slugify'); |
|
|
const e = require('express'); |
|
|
const emitter = require('../services/eventEmitter'); |
|
|
|
|
|
|
|
|
// Load project path from config/constants.js
|
|
|
// Load project path from config/constants.js
|
|
|
const { basedir } = require('../config/constants'); |
|
|
const { basedir } = require('../config/constants'); |
|
@ -29,7 +28,7 @@ function generateFilePath(filename) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Create a class File that extends EventEmitter
|
|
|
// Create a class File that extends EventEmitter
|
|
|
class File extends EventEmitter { |
|
|
class File { |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* Create a new File instance |
|
|
* Create a new File instance |
|
@ -57,13 +56,13 @@ class File extends EventEmitter { |
|
|
const request = url.protocol === 'https:' ? https : http; |
|
|
const request = url.protocol === 'https:' ? https : http; |
|
|
|
|
|
|
|
|
// Emit a download.start event with the url and filepath
|
|
|
// Emit a download.start event with the url and filepath
|
|
|
this.emit('download.start', { url: this.url, filepath: this.filepath }); |
|
|
emitter.emit('download.start', { url: this.url, filepath: this.filepath }); |
|
|
|
|
|
|
|
|
// Handle file errors
|
|
|
// Handle file errors
|
|
|
file.on('error', (err) => { |
|
|
file.on('error', (err) => { |
|
|
fs.unlink(filepath, () => { |
|
|
fs.unlink(filepath, () => { |
|
|
// Emit a download.error event with the error
|
|
|
// Emit a download.error event with the error
|
|
|
this.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' }); |
|
|
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'file' }); |
|
|
}); |
|
|
}); |
|
|
}); |
|
|
}); |
|
|
|
|
|
|
|
@ -73,7 +72,7 @@ class File extends EventEmitter { |
|
|
// Check if response is valid
|
|
|
// Check if response is valid
|
|
|
if (response.statusCode !== 200) { |
|
|
if (response.statusCode !== 200) { |
|
|
// Emit a download.error event with the error
|
|
|
// Emit a download.error event with the error
|
|
|
this.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' }); |
|
|
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: response.statusMessage, type: 'response' }); |
|
|
return reject(response.statusMessage); |
|
|
return reject(response.statusMessage); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
@ -93,7 +92,7 @@ class File extends EventEmitter { |
|
|
// Check if the downloaded size is bigger than the step limit
|
|
|
// Check if the downloaded size is bigger than the step limit
|
|
|
if (downloaded > stepLimit) { |
|
|
if (downloaded > stepLimit) { |
|
|
// Emit a download.progress event with the url, filepath and downloaded size
|
|
|
// Emit a download.progress event with the url, filepath and downloaded size
|
|
|
this.emit('download.progress', { url: this.url, filepath: this.filepath, percentage: Math.floor((downloaded / contentLength) * 100) }); |
|
|
emitter.emit('download.progress', { url: this.url, filepath: this.filepath, percentage: Math.floor((downloaded / contentLength) * 100) }); |
|
|
stepLimit += step; |
|
|
stepLimit += step; |
|
|
} |
|
|
} |
|
|
}); |
|
|
}); |
|
@ -102,7 +101,7 @@ class File extends EventEmitter { |
|
|
|
|
|
|
|
|
file.on('finish', () => { |
|
|
file.on('finish', () => { |
|
|
// Emit a download.end event with the url and filepath
|
|
|
// Emit a download.end event with the url and filepath
|
|
|
this.emit('download.end', { url: this.url, filepath: this.filepath }); |
|
|
emitter.emit('download.end', { url: this.url, filepath: this.filepath }); |
|
|
file.close(() => { |
|
|
file.close(() => { |
|
|
resolve(filepath); |
|
|
resolve(filepath); |
|
|
}); |
|
|
}); |
|
@ -111,7 +110,7 @@ class File extends EventEmitter { |
|
|
.on('error', (err) => { |
|
|
.on('error', (err) => { |
|
|
fs.unlink(filepath, () => { |
|
|
fs.unlink(filepath, () => { |
|
|
// Emit a download.error event with the error
|
|
|
// Emit a download.error event with the error
|
|
|
this.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' }); |
|
|
emitter.emit('download.error', { url: this.url, filepath: this.filepath, error: err.message, type: 'request' }); |
|
|
reject(err.message); |
|
|
reject(err.message); |
|
|
}); |
|
|
}); |
|
|
}); |
|
|
}); |
|
@ -145,11 +144,11 @@ class File extends EventEmitter { |
|
|
.pipe(csvParser({ separator: ';' })) |
|
|
.pipe(csvParser({ separator: ';' })) |
|
|
.on('headers', (headers) => { |
|
|
.on('headers', (headers) => { |
|
|
// Emit a parse.start event with the url and filepath
|
|
|
// Emit a parse.start event with the url and filepath
|
|
|
this.emit('parse.start', { url: this.url, filepath: this.filepath, headers }); |
|
|
emitter.emit('parse.start', { url: this.url, filepath: this.filepath, headers }); |
|
|
}) |
|
|
}) |
|
|
.on('data', (row) => { |
|
|
.on('data', (row) => { |
|
|
// Emit a parse.data event with the url, filepath and data
|
|
|
// Emit a parse.data event with the url, filepath and data
|
|
|
this.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, index: i }); |
|
|
emitter.emit('parse.data', { url: this.url, filepath: this.filepath, data: row, index: i }); |
|
|
|
|
|
|
|
|
let result = []; |
|
|
let result = []; |
|
|
if ( i === 0 ) { |
|
|
if ( i === 0 ) { |
|
@ -170,11 +169,11 @@ class File extends EventEmitter { |
|
|
}) |
|
|
}) |
|
|
.on('error', (err) => { |
|
|
.on('error', (err) => { |
|
|
// Emit a parse.error event with the error
|
|
|
// Emit a parse.error event with the error
|
|
|
this.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); |
|
|
emitter.emit('parse.error', { url: this.url, filepath: this.filepath, error: err.message }); |
|
|
}) |
|
|
}) |
|
|
.on('end', () => { |
|
|
.on('end', () => { |
|
|
// Emit a parse.end event with the url and filepath
|
|
|
// Emit a parse.end event with the url and filepath
|
|
|
this.emit('parse.end', { url: this.url, filepath: this.filepath, count: i }); |
|
|
emitter.emit('parse.end', { url: this.url, filepath: this.filepath, count: i }); |
|
|
stream.end(); |
|
|
stream.end(); |
|
|
}); |
|
|
}); |
|
|
|
|
|
|
|
|