const fs = require('fs'); const path = require('path'); // Helper function to format elapsed time function formatElapsedTime(elapsed) { // If elapsed is a timestamp, convert to elapsed milliseconds if (elapsed instanceof Date || elapsed > 1000000000000) { elapsed = Date.now() - elapsed; } else { // If elapsed is in seconds, convert to milliseconds elapsed = elapsed * 1000; } const seconds = Math.floor(elapsed / 1000); const minutes = Math.floor(seconds / 60); const hours = Math.floor(minutes / 60); if (hours > 0) { return `${hours}h ${minutes % 60}m`; } else if (minutes > 0) { return `${minutes}m ${seconds % 60}s`; } else { return `${seconds}s`; } } // Helper function to estimate remaining time function estimateRemaining(startTime, current, total) { if (current === 0) return null; const elapsed = Date.now() - startTime; const rate = current / elapsed; const remaining = (total - current) / rate; const minutes = Math.floor(remaining / 60000); const seconds = Math.floor((remaining % 60000) / 1000); if (minutes > 0) { return `${minutes}m ${seconds}s`; } else { return `${seconds}s`; } } // Helper function to calculate rate function calculateRate(startTime, current) { const elapsed = (Date.now() - startTime) / 1000; // Convert to seconds return elapsed > 0 ? Math.round(current / elapsed) : 0; } // Set up logging const LOG_DIR = path.join(__dirname, '../../../logs'); const ERROR_LOG = path.join(LOG_DIR, 'import-errors.log'); const IMPORT_LOG = path.join(LOG_DIR, 'import.log'); const STATUS_FILE = path.join(LOG_DIR, 'metrics-status.json'); // Ensure log directory exists if (!fs.existsSync(LOG_DIR)) { fs.mkdirSync(LOG_DIR, { recursive: true }); } // Helper function to log errors function logError(error, context = '') { const timestamp = new Date().toISOString(); const errorMessage = `[${timestamp}] ${context}\nError: ${error.message}\nStack: ${error.stack}\n\n`; // Log to error file fs.appendFileSync(ERROR_LOG, errorMessage); // Also log to console console.error(`\n${context}\nError: ${error.message}`); } // Helper function to log import progress function logImport(message) { const timestamp = new Date().toISOString(); const logMessage = `[${timestamp}] ${message}\n`; fs.appendFileSync(IMPORT_LOG, logMessage); } // Helper function to output progress function outputProgress(data) { // Save progress to file for resumption saveProgress(data); // Format as SSE event const event = { progress: data }; // Always send to stdout for frontend process.stdout.write(JSON.stringify(event) + '\n'); // Log significant events to disk const isSignificant = // Operation starts (data.operation && !data.current) || // Operation completions and errors data.status === 'complete' || data.status === 'error' || // Major phase changes data.operation?.includes('Starting ABC classification') || data.operation?.includes('Starting time-based aggregates') || data.operation?.includes('Starting vendor metrics'); if (isSignificant) { logImport(`${data.operation || 'Operation'}${data.message ? ': ' + data.message : ''}${data.error ? ' Error: ' + data.error : ''}${data.status ? ' Status: ' + data.status : ''}`); } } function saveProgress(progress) { try { fs.writeFileSync(STATUS_FILE, JSON.stringify({ ...progress, timestamp: Date.now() })); } catch (err) { console.error('Failed to save progress:', err); } } function clearProgress() { try { if (fs.existsSync(STATUS_FILE)) { fs.unlinkSync(STATUS_FILE); } } catch (err) { console.error('Failed to clear progress:', err); } } function getProgress() { try { if (fs.existsSync(STATUS_FILE)) { const progress = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf8')); // Check if the progress is still valid (less than 1 hour old) if (progress.timestamp && Date.now() - progress.timestamp < 3600000) { return progress; } else { // Clear old progress clearProgress(); } } } catch (err) { console.error('Failed to read progress:', err); clearProgress(); } return null; } module.exports = { formatElapsedTime, estimateRemaining, calculateRate, logError, logImport, outputProgress, saveProgress, clearProgress, getProgress };