const path = require('path'); const fs = require('fs'); const axios = require('axios'); const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./metrics/utils/progress'); // Change working directory to script directory process.chdir(path.dirname(__filename)); require('dotenv').config({ path: path.resolve(__dirname, '..', '.env') }); const FILES = [ { name: '39f2x83-products.csv', url: process.env.PRODUCTS_CSV_URL }, { name: '39f2x83-orders.csv', url: process.env.ORDERS_CSV_URL }, { name: '39f2x83-purchase_orders.csv', url: process.env.PURCHASE_ORDERS_CSV_URL } ]; let isCancelled = false; function cancelUpdate() { isCancelled = true; outputProgress({ status: 'cancelled', operation: 'CSV update cancelled', current: 0, total: FILES.length, elapsed: null, remaining: null, rate: 0 }); } async function downloadFile(file, index, startTime) { if (isCancelled) return; const csvDir = path.join(__dirname, '../csv'); if (!fs.existsSync(csvDir)) { fs.mkdirSync(csvDir, { recursive: true }); } const writer = fs.createWriteStream(path.join(csvDir, file.name)); try { const response = await axios({ url: file.url, method: 'GET', responseType: 'stream' }); const totalLength = response.headers['content-length']; let downloadedLength = 0; let lastProgressUpdate = Date.now(); const PROGRESS_INTERVAL = 1000; // Update progress every second response.data.on('data', (chunk) => { if (isCancelled) { writer.end(); return; } downloadedLength += chunk.length; // Update progress based on time interval const now = Date.now(); if (now - lastProgressUpdate >= PROGRESS_INTERVAL) { const progress = (downloadedLength / totalLength) * 100; outputProgress({ status: 'running', operation: `Downloading ${file.name}`, current: index + (downloadedLength / totalLength), total: FILES.length, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, index + (downloadedLength / totalLength), FILES.length), rate: calculateRate(startTime, index + (downloadedLength / totalLength)), percentage: progress.toFixed(1), file_progress: { name: file.name, downloaded: downloadedLength, total: totalLength, percentage: progress.toFixed(1) } }); lastProgressUpdate = now; } }); response.data.pipe(writer); return new Promise((resolve, reject) => { writer.on('finish', resolve); writer.on('error', reject); }); } catch (error) { fs.unlinkSync(path.join(csvDir, file.name)); throw error; } } // Main function to update all files async function updateFiles() { const startTime = Date.now(); outputProgress({ status: 'running', operation: 'Starting CSV update', current: 0, total: FILES.length, elapsed: '0s', remaining: null, rate: 0, percentage: '0' }); try { for (let i = 0; i < FILES.length; i++) { if (isCancelled) { return; } const file = FILES[i]; await downloadFile(file, i, startTime); outputProgress({ status: 'running', operation: 'CSV update in progress', current: i + 1, total: FILES.length, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, i + 1, FILES.length), rate: calculateRate(startTime, i + 1), percentage: (((i + 1) / FILES.length) * 100).toFixed(1) }); } outputProgress({ status: 'complete', operation: 'CSV update complete', current: FILES.length, total: FILES.length, elapsed: formatElapsedTime(startTime), remaining: '0s', rate: calculateRate(startTime, FILES.length), percentage: '100' }); } catch (error) { outputProgress({ status: 'error', operation: 'CSV update failed', error: error.message, current: 0, total: FILES.length, elapsed: formatElapsedTime(startTime), remaining: null, rate: 0 }); throw error; } } // Run the update only if this is the main module if (require.main === module) { updateFiles().catch((error) => { console.error('Error updating CSV files:', error); process.exit(1); }); } // Export the functions needed by the route module.exports = { updateFiles, cancelUpdate };