Enhance metrics calculation scripts with improved progress tracking and cancellation support

This commit is contained in:
2025-01-28 20:54:05 -05:00
parent a1e3803ca3
commit 9c34e24909
12 changed files with 915 additions and 327 deletions

View File

@@ -3,6 +3,7 @@ const path = require('path');
const csv = require('csv-parse');
const mysql = require('mysql2/promise');
const dotenv = require('dotenv');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./metrics/utils/progress');
// Get test limits from environment variables
const PRODUCTS_TEST_LIMIT = parseInt(process.env.PRODUCTS_TEST_LIMIT || '0');
@@ -106,20 +107,19 @@ async function countRows(filePath) {
}
// Helper function to update progress with time estimate
function updateProgress(current, total, operation, startTime) {
const elapsed = (Date.now() - startTime) / 1000;
const rate = current / elapsed; // rows per second
const remaining = (total - current) / rate;
function updateProgress(current, total, operation, startTime, added = 0, updated = 0, skipped = 0) {
outputProgress({
status: 'running',
operation,
current,
total,
rate,
elapsed: formatDuration(elapsed),
remaining: formatDuration(remaining),
percentage: ((current / total) * 100).toFixed(1)
rate: calculateRate(startTime, current),
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, current, total),
percentage: ((current / total) * 100).toFixed(1),
added,
updated,
skipped
});
}
@@ -474,7 +474,7 @@ async function importProducts(pool, filePath) {
// Update progress every 100ms to avoid console flooding
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Products import', startTime);
updateProgress(rowCount, totalRows, 'Products import', startTime, added, updated, 0);
lastUpdate = now;
}
@@ -678,7 +678,7 @@ async function importOrders(pool, filePath) {
// Update progress every 100ms
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Orders import', startTime);
updateProgress(rowCount, totalRows, 'Orders import', startTime, added, updated, skipped);
lastUpdate = now;
}
@@ -845,7 +845,7 @@ async function importPurchaseOrders(pool, filePath) {
// Update progress every 100ms
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Purchase orders import', startTime);
updateProgress(rowCount, totalRows, 'Purchase orders import', startTime, added, updated, skipped);
lastUpdate = now;
}