matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance metrics calculation scripts with improved progress tracking and cancellation support

Browse Source
This commit is contained in:
Matt
2025-01-28 20:54:05 -05:00
parent a1e3803ca3
commit 9c34e24909
12 changed files with 915 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
inventory-server/scripts/calculate-metrics.js
View File

@@ -186,6 +186,19 @@ async function calculateMetrics() {
}
// Calculate ABC classification
outputProgress({
status: 'running',
operation: 'Starting ABC classification',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
const [abcConfig] = await connection.query('SELECT a_threshold, b_threshold FROM abc_classification_config WHERE id = 1');
const abcThresholds = abcConfig[0] || { a_threshold: 20, b_threshold: 50 };
@@ -202,6 +215,19 @@ async function calculateMetrics() {
) ENGINE=MEMORY
`);
outputProgress({
status: 'running',
operation: 'Creating revenue rankings',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
await connection.query(`
INSERT INTO temp_revenue_ranks
SELECT
@@ -222,11 +248,26 @@ async function calculateMetrics() {
const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
const totalCount = rankingCount[0].total_count || 1;
outputProgress({
status: 'running',
operation: 'Updating ABC classifications',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Process updates in batches
let abcProcessedCount = 0;
const batchSize = 5000;
while (true) {
if (isCancelled) return processedCount;
// First get a batch of PIDs that need updating
const [pids] = await connection.query(`
SELECT pm.pid
@@ -267,6 +308,18 @@ async function calculateMetrics() {
pids.map(row => row.pid)]);
abcProcessedCount += result.affectedRows;
processedCount = Math.floor(totalProducts * (0.99 + (abcProcessedCount / totalCount) * 0.01));
outputProgress({
status: 'running',
operation: 'ABC classification progress',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Small delay between batches to allow other transactions
await new Promise(resolve => setTimeout(resolve, 100));
@@ -276,14 +329,14 @@ async function calculateMetrics() {
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_revenue_ranks');
// Final success message
global.outputProgress({
outputProgress({
status: 'complete',
operation: 'Metrics calculation complete',
current: totalProducts,
total: totalProducts,
elapsed: global.formatElapsedTime(startTime),
elapsed: formatElapsedTime(startTime),
remaining: '0s',
rate: global.calculateRate(startTime, totalProducts),
rate: calculateRate(startTime, totalProducts),
percentage: '100'
});

24
inventory-server/scripts/import-csv.js
View File

@@ -3,6 +3,7 @@ const path = require('path');
const csv = require('csv-parse');
const mysql = require('mysql2/promise');
const dotenv = require('dotenv');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./metrics/utils/progress');
// Get test limits from environment variables
const PRODUCTS_TEST_LIMIT = parseInt(process.env.PRODUCTS_TEST_LIMIT || '0');
@@ -106,20 +107,19 @@ async function countRows(filePath) {
}
// Helper function to update progress with time estimate
function updateProgress(current, total, operation, startTime) {
const elapsed = (Date.now() - startTime) / 1000;
const rate = current / elapsed; // rows per second
const remaining = (total - current) / rate;
function updateProgress(current, total, operation, startTime, added = 0, updated = 0, skipped = 0) {
outputProgress({
status: 'running',
operation,
current,
total,
rate,
elapsed: formatDuration(elapsed),
remaining: formatDuration(remaining),
percentage: ((current / total) * 100).toFixed(1)
rate: calculateRate(startTime, current),
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, current, total),
percentage: ((current / total) * 100).toFixed(1),
added,
updated,
skipped
});
}
@@ -474,7 +474,7 @@ async function importProducts(pool, filePath) {
// Update progress every 100ms to avoid console flooding
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Products import', startTime);
updateProgress(rowCount, totalRows, 'Products import', startTime, added, updated, 0);
lastUpdate = now;
}
@@ -678,7 +678,7 @@ async function importOrders(pool, filePath) {
// Update progress every 100ms
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Orders import', startTime);
updateProgress(rowCount, totalRows, 'Orders import', startTime, added, updated, skipped);
lastUpdate = now;
}
@@ -845,7 +845,7 @@ async function importPurchaseOrders(pool, filePath) {
// Update progress every 100ms
const now = Date.now();
if (now - lastUpdate > 100) {
updateProgress(rowCount, totalRows, 'Purchase orders import', startTime);
updateProgress(rowCount, totalRows, 'Purchase orders import', startTime, added, updated, skipped);
lastUpdate = now;
}

74
inventory-server/scripts/import-from-prod.js
View File

@@ -2,6 +2,7 @@ const mysql = require("mysql2/promise");
const { Client } = require("ssh2");
const dotenv = require("dotenv");
const path = require("path");
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./metrics/utils/progress');
dotenv.config({ path: path.join(__dirname, "../.env") });
@@ -43,42 +44,9 @@ const localDbConfig = {
namedPlaceholders: true,
};
// Helper function to output progress
function outputProgress(data) {
process.stdout.write(JSON.stringify(data) + "\n");
}
// Helper function to format duration
function formatDuration(seconds) {
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
seconds = Math.floor(seconds % 60);
const parts = [];
if (hours > 0) parts.push(`${hours}h`);
if (minutes > 0) parts.push(`${minutes}m`);
if (seconds > 0 || parts.length === 0) parts.push(`${seconds}s`);
return parts.join(" ");
}
// Helper function to update progress with time estimate
function updateProgress(current, total, operation, startTime) {
const elapsed = (Date.now() - startTime) / 1000;
const rate = current / elapsed;
const remaining = (total - current) / rate;
outputProgress({
status: "running",
operation,
current,
total,
rate,
elapsed: formatDuration(elapsed),
remaining: formatDuration(remaining),
percentage: ((current / total) * 100).toFixed(1),
});
}
// Constants
const BATCH_SIZE = 1000;
const PROGRESS_INTERVAL = 1000; // Update progress every second
let isImportCancelled = false;
@@ -86,8 +54,27 @@ let isImportCancelled = false;
function cancelImport() {
isImportCancelled = true;
outputProgress({
status: "cancelled",
operation: "Import cancelled",
status: 'cancelled',
operation: 'Import cancelled',
current: 0,
total: 0,
elapsed: null,
remaining: null,
rate: 0
});
}
// Helper function to update progress with time estimate
function updateProgress(current, total, operation, startTime) {
outputProgress({
status: 'running',
operation,
current,
total,
rate: calculateRate(startTime, current),
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, current, total),
percentage: ((current / total) * 100).toFixed(1)
});
}
@@ -276,7 +263,7 @@ async function importCategories(prodConnection, localConnection) {
operation: "Categories import completed",
current: totalInserted,
total: totalInserted,
duration: formatDuration((Date.now() - startTime) / 1000),
duration: formatElapsedTime((Date.now() - startTime) / 1000),
});
} catch (error) {
console.error("Error importing categories:", error);
@@ -510,7 +497,6 @@ async function importProducts(prodConnection, localConnection) {
const total = rows.length;
// Process products in batches
const BATCH_SIZE = 100;
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
let batch = rows.slice(i, i + BATCH_SIZE);
@@ -641,7 +627,7 @@ async function importProducts(prodConnection, localConnection) {
operation: "Products import completed",
current: total,
total,
duration: formatDuration((Date.now() - startTime) / 1000),
duration: formatElapsedTime((Date.now() - startTime) / 1000),
});
} catch (error) {
console.error("Error importing products:", error);
@@ -1384,7 +1370,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date(endTime).toISOString(),
elapsed_time: formatDuration((endTime - startTime) / 1000),
elapsed_time: formatElapsedTime((endTime - startTime) / 1000),
elapsed_seconds: Math.round((endTime - startTime) / 1000)
}
});
@@ -1459,7 +1445,7 @@ async function main() {
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date(endTime).toISOString(),
elapsed_time: formatDuration((endTime - startTime) / 1000),
elapsed_time: formatElapsedTime((endTime - startTime) / 1000),
elapsed_seconds: Math.round((endTime - startTime) / 1000)
}
});
@@ -1473,7 +1459,7 @@ async function main() {
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date(endTime).toISOString(),
elapsed_time: formatDuration((endTime - startTime) / 1000),
elapsed_time: formatElapsedTime((endTime - startTime) / 1000),
elapsed_seconds: Math.round((endTime - startTime) / 1000)
}
});

63
inventory-server/scripts/metrics/brand-metrics.js
View File

@@ -1,18 +1,32 @@
const { outputProgress } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateBrandMetrics(startTime, totalProducts, processedCount) {
async function calculateBrandMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Calculating brand metrics',
current: Math.floor(totalProducts * 0.95),
status: 'cancelled',
operation: 'Brand metrics calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.95), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.95)),
percentage: '95'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting brand metrics calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Calculate brand metrics with optimized queries
@@ -111,6 +125,20 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount) {
last_calculated_at = CURRENT_TIMESTAMP
`);
processedCount = Math.floor(totalProducts * 0.97);
outputProgress({
status: 'running',
operation: 'Brand metrics calculated, starting time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Calculate brand time-based metrics with optimized query
await connection.query(`
INSERT INTO brand_time_metrics (
@@ -170,10 +198,27 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount) {
avg_margin = VALUES(avg_margin)
`);
return Math.floor(totalProducts * 0.98);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Brand time-based metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
} catch (error) {
logError(error, 'Error calculating brand metrics');
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
module.exports = calculateBrandMetrics;

129
inventory-server/scripts/metrics/category-metrics.js
View File

@@ -1,18 +1,32 @@
const { outputProgress } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateCategoryMetrics(startTime, totalProducts, processedCount) {
async function calculateCategoryMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Calculating category metrics',
current: Math.floor(totalProducts * 0.85),
status: 'cancelled',
operation: 'Category metrics calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.85), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.85)),
percentage: '85'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting category metrics calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// First, calculate base category metrics
@@ -44,6 +58,20 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 0.90);
outputProgress({
status: 'running',
operation: 'Base category metrics calculated, updating with margin data',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Then update with margin and turnover data
await connection.query(`
WITH category_sales AS (
@@ -68,6 +96,20 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
cm.last_calculated_at = NOW()
`);
processedCount = Math.floor(totalProducts * 0.95);
outputProgress({
status: 'running',
operation: 'Margin data updated, calculating growth rates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Finally update growth rates
await connection.query(`
WITH current_period AS (
@@ -112,6 +154,20 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
WHERE cp.cat_id IS NOT NULL OR pp.cat_id IS NOT NULL
`);
processedCount = Math.floor(totalProducts * 0.97);
outputProgress({
status: 'running',
operation: 'Growth rates calculated, updating time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Calculate time-based metrics
await connection.query(`
INSERT INTO category_time_metrics (
@@ -157,50 +213,27 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
turnover_rate = VALUES(turnover_rate)
`);
// Calculate sales metrics for different time periods
const periods = [30, 90, 180, 365];
for (const days of periods) {
await connection.query(`
INSERT INTO category_sales_metrics (
category_id,
brand,
period_start,
period_end,
avg_daily_sales,
total_sold,
num_products,
avg_price,
last_calculated_at
)
SELECT
pc.cat_id as category_id,
COALESCE(p.brand, 'Unbranded') as brand,
DATE_SUB(CURDATE(), INTERVAL ? DAY) as period_start,
CURDATE() as period_end,
COALESCE(SUM(o.quantity), 0) / ? as avg_daily_sales,
COALESCE(SUM(o.quantity), 0) as total_sold,
COUNT(DISTINCT p.pid) as num_products,
COALESCE(AVG(o.price), 0) as avg_price,
NOW() as last_calculated_at
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
LEFT JOIN orders o ON p.pid = o.pid
AND o.date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
AND o.canceled = false
GROUP BY pc.cat_id, p.brand
ON DUPLICATE KEY UPDATE
avg_daily_sales = VALUES(avg_daily_sales),
total_sold = VALUES(total_sold),
num_products = VALUES(num_products),
avg_price = VALUES(avg_price),
last_calculated_at = NOW()
`, [days, days, days]);
}
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Time-based metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return Math.floor(totalProducts * 0.9);
return processedCount;
} catch (error) {
logError(error, 'Error calculating category metrics');
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
module.exports = calculateCategoryMetrics;

63
inventory-server/scripts/metrics/financial-metrics.js
View File

@@ -1,18 +1,32 @@
const { outputProgress } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateFinancialMetrics(startTime, totalProducts, processedCount) {
async function calculateFinancialMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Calculating financial metrics',
current: Math.floor(totalProducts * 0.6),
status: 'cancelled',
operation: 'Financial metrics calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.6), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.6)),
percentage: '60'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting financial metrics calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Calculate financial metrics with optimized query
@@ -48,6 +62,20 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
END
`);
processedCount = Math.floor(totalProducts * 0.65);
outputProgress({
status: 'running',
operation: 'Base financial metrics calculated, updating time aggregates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Update time-based aggregates with optimized query
await connection.query(`
WITH monthly_financials AS (
@@ -78,10 +106,27 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
END
`);
return Math.floor(totalProducts * 0.7);
processedCount = Math.floor(totalProducts * 0.70);
outputProgress({
status: 'running',
operation: 'Time-based aggregates updated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
} catch (error) {
logError(error, 'Error calculating financial metrics');
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
module.exports = calculateFinancialMetrics;

71
inventory-server/scripts/metrics/product-metrics.js
View File

@@ -1,4 +1,4 @@
const { outputProgress, logError } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
// Helper function to handle NaN and undefined values
@@ -9,24 +9,38 @@ function sanitizeValue(value) {
return value;
}
async function calculateProductMetrics(startTime, totalProducts, processedCount = 0) {
async function calculateProductMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
try {
// Skip flags are inherited from the parent scope
const SKIP_PRODUCT_BASE_METRICS = 0;
const SKIP_PRODUCT_TIME_AGGREGATES =0;
const SKIP_PRODUCT_TIME_AGGREGATES = 0;
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Product metrics calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
// Calculate base product metrics
if (!SKIP_PRODUCT_BASE_METRICS) {
outputProgress({
status: 'running',
operation: 'Calculating base product metrics',
current: Math.floor(totalProducts * 0.2),
operation: 'Starting base product metrics calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.2), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.2)),
percentage: '20'
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Calculate base metrics
@@ -72,8 +86,17 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
`);
processedCount = Math.floor(totalProducts * 0.4);
outputProgress({
status: 'running',
operation: 'Base product metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
} else {
console.log('Skipping base product metrics calculation');
processedCount = Math.floor(totalProducts * 0.4);
outputProgress({
status: 'running',
@@ -83,21 +106,23 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: '40'
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
}
if (isCancelled) return processedCount;
// Calculate product time aggregates
if (!SKIP_PRODUCT_TIME_AGGREGATES) {
outputProgress({
status: 'running',
operation: 'Calculating product time aggregates',
current: Math.floor(totalProducts * 0.4),
operation: 'Starting product time aggregates calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.4), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.4)),
percentage: '40'
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Calculate time-based aggregates
@@ -151,8 +176,17 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
`);
processedCount = Math.floor(totalProducts * 0.6);
outputProgress({
status: 'running',
operation: 'Product time aggregates calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
} else {
console.log('Skipping product time aggregates calculation');
processedCount = Math.floor(totalProducts * 0.6);
outputProgress({
status: 'running',
@@ -162,11 +196,14 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: '60'
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
}
return processedCount;
} catch (error) {
logError(error, 'Error calculating product metrics');
throw error;
} finally {
if (connection) {
connection.release();

119
inventory-server/scripts/metrics/sales-forecasts.js
View File

@@ -1,18 +1,32 @@
const { outputProgress } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateSalesForecasts(startTime, totalProducts, processedCount) {
async function calculateSalesForecasts(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Calculating sales forecasts',
current: Math.floor(totalProducts * 0.98),
status: 'cancelled',
operation: 'Sales forecasts calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.98), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.98)),
percentage: '98'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting sales forecasts calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// First, create a temporary table for forecast dates
@@ -42,6 +56,20 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
) numbers
`);
processedCount = Math.floor(totalProducts * 0.92);
outputProgress({
status: 'running',
operation: 'Forecast dates prepared, calculating daily sales stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Create temporary table for daily sales stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_daily_sales AS
@@ -57,6 +85,20 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
GROUP BY o.pid, DAYOFWEEK(o.date)
`);
processedCount = Math.floor(totalProducts * 0.94);
outputProgress({
status: 'running',
operation: 'Daily sales stats calculated, preparing product stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Create temporary table for product stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_product_stats AS
@@ -68,6 +110,20 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
GROUP BY pid
`);
processedCount = Math.floor(totalProducts * 0.96);
outputProgress({
status: 'running',
operation: 'Product stats prepared, calculating product-level forecasts',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Calculate product-level forecasts
await connection.query(`
INSERT INTO sales_forecasts (
@@ -116,6 +172,20 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
last_calculated_at = NOW()
`);
processedCount = Math.floor(totalProducts * 0.98);
outputProgress({
status: 'running',
operation: 'Product forecasts calculated, preparing category stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Create temporary table for category stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_sales AS
@@ -142,6 +212,20 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
GROUP BY cat_id
`);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Category stats prepared, calculating category-level forecasts',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Calculate category-level forecasts
await connection.query(`
INSERT INTO category_forecasts (
@@ -199,10 +283,27 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount)
DROP TEMPORARY TABLE IF EXISTS temp_category_stats;
`);
return Math.floor(totalProducts * 1.0);
processedCount = Math.floor(totalProducts * 1.0);
outputProgress({
status: 'running',
operation: 'Category forecasts calculated and temporary tables cleaned up',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
} catch (error) {
logError(error, 'Error calculating sales forecasts');
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
module.exports = calculateSalesForecasts;

61
inventory-server/scripts/metrics/time-aggregates.js
View File

@@ -1,18 +1,32 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateTimeAggregates(startTime, totalProducts, processedCount) {
async function calculateTimeAggregates(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Calculating time aggregates',
current: Math.floor(totalProducts * 0.95),
status: 'cancelled',
operation: 'Time aggregates calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.95), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.95)),
percentage: '95'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting time aggregates calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// Initial insert of time-based aggregates
@@ -109,6 +123,20 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount)
profit_margin = VALUES(profit_margin)
`);
processedCount = Math.floor(totalProducts * 0.60);
outputProgress({
status: 'running',
operation: 'Base time aggregates calculated, updating financial metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Update with financial metrics
await connection.query(`
UPDATE product_time_aggregates pta
@@ -136,7 +164,22 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount)
END
`);
return Math.floor(totalProducts * 0.65);
processedCount = Math.floor(totalProducts * 0.65);
outputProgress({
status: 'running',
operation: 'Financial metrics updated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
} catch (error) {
logError(error, 'Error calculating time aggregates');
throw error;
} finally {
if (connection) {
connection.release();

62
inventory-server/scripts/metrics/vendor-metrics.js
View File

@@ -1,18 +1,32 @@
const { outputProgress } = require('./utils/progress');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
const { getConnection } = require('./utils/db');
async function calculateVendorMetrics(startTime, totalProducts, processedCount) {
async function calculateVendorMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
const connection = await getConnection();
try {
if (isCancelled) {
outputProgress({
status: 'running',
operation: 'Ensuring vendors exist in vendor_details',
current: Math.floor(totalProducts * 0.7),
status: 'cancelled',
operation: 'Vendor metrics calculation cancelled',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.7), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.7)),
percentage: '70'
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
}
outputProgress({
status: 'running',
operation: 'Starting vendor metrics calculation',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
// First ensure all vendors exist in vendor_details
@@ -27,17 +41,20 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount)
WHERE vendor IS NOT NULL
`);
processedCount = Math.floor(totalProducts * 0.8);
outputProgress({
status: 'running',
operation: 'Calculating vendor metrics',
current: Math.floor(totalProducts * 0.8),
operation: 'Vendor details updated, calculating metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.8), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.8)),
percentage: '80'
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
if (isCancelled) return processedCount;
// Now calculate vendor metrics
await connection.query(`
INSERT INTO vendor_metrics (
@@ -130,10 +147,27 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount)
last_calculated_at = VALUES(last_calculated_at)
`);
return Math.floor(totalProducts * 0.9);
processedCount = Math.floor(totalProducts * 0.9);
outputProgress({
status: 'running',
operation: 'Vendor metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
});
return processedCount;
} catch (error) {
logError(error, 'Error calculating vendor metrics');
throw error;
} finally {
if (connection) {
connection.release();
}
}
}
module.exports = calculateVendorMetrics;

216
inventory-server/scripts/reset-metrics.js
View File

@@ -12,6 +12,12 @@ const dbConfig = {
};
function outputProgress(data) {
if (!data.status) {
data = {
status: 'running',
...data
};
}
console.log(JSON.stringify(data));
}
@@ -51,36 +57,228 @@ const REQUIRED_CORE_TABLES = [
'purchase_orders'
];
// Split SQL into individual statements
function splitSQLStatements(sql) {
sql = sql.replace(/\r\n/g, '\n');
let statements = [];
let currentStatement = '';
let inString = false;
let stringChar = '';
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const nextChar = sql[i + 1] || '';
if ((char === "'" || char === '"') && sql[i - 1] !== '\\') {
if (!inString) {
inString = true;
stringChar = char;
} else if (char === stringChar) {
inString = false;
}
}
if (!inString && char === '-' && nextChar === '-') {
while (i < sql.length && sql[i] !== '\n') i++;
continue;
}
if (!inString && char === '/' && nextChar === '*') {
i += 2;
while (i < sql.length && (sql[i] !== '*' || sql[i + 1] !== '/')) i++;
i++;
continue;
}
if (!inString && char === ';') {
if (currentStatement.trim()) {
statements.push(currentStatement.trim());
}
currentStatement = '';
} else {
currentStatement += char;
}
}
if (currentStatement.trim()) {
statements.push(currentStatement.trim());
}
return statements;
}
async function resetMetrics() {
let connection;
try {
outputProgress({
operation: 'Starting metrics reset',
message: 'Connecting to database...'
});
connection = await mysql.createConnection(dbConfig);
await connection.beginTransaction();
// Verify required core tables exist
outputProgress({
operation: 'Verifying core tables',
message: 'Checking required tables exist...'
});
const [tables] = await connection.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN (?)
`, [REQUIRED_CORE_TABLES]);
const existingCoreTables = tables.map(t => t.table_name);
const missingCoreTables = REQUIRED_CORE_TABLES.filter(t => !existingCoreTables.includes(t));
if (missingCoreTables.length > 0) {
throw new Error(`Required core tables missing: ${missingCoreTables.join(', ')}`);
}
// Verify config tables exist
outputProgress({
operation: 'Verifying config tables',
message: 'Checking configuration tables exist...'
});
const [configTables] = await connection.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN (?)
`, [CONFIG_TABLES]);
const existingConfigTables = configTables.map(t => t.table_name);
const missingConfigTables = CONFIG_TABLES.filter(t => !existingConfigTables.includes(t));
if (missingConfigTables.length > 0) {
throw new Error(`Required config tables missing: ${missingConfigTables.join(', ')}`);
}
// Drop all metrics tables
outputProgress({
operation: 'Dropping metrics tables',
message: 'Removing existing metrics tables...'
});
for (const table of METRICS_TABLES) {
console.log(`Dropping table: ${table}`);
try {
await connection.query(`DROP TABLE IF EXISTS ${table}`);
console.log(`Successfully dropped: ${table}`);
outputProgress({
operation: 'Table dropped',
message: `Successfully dropped table: ${table}`
});
} catch (err) {
console.error(`Error dropping ${table}:`, err.message);
outputProgress({
status: 'error',
operation: 'Drop table error',
message: `Error dropping table ${table}: ${err.message}`
});
throw err;
}
}
// Recreate all metrics tables from schema
const schemaSQL = fs.readFileSync(path.resolve(__dirname, '../db/metrics-schema.sql'), 'utf8');
await connection.query(schemaSQL);
console.log('All metrics tables recreated successfully');
// Read metrics schema
outputProgress({
operation: 'Reading schema',
message: 'Loading metrics schema file...'
});
const schemaPath = path.resolve(__dirname, '../db/metrics-schema.sql');
if (!fs.existsSync(schemaPath)) {
throw new Error(`Schema file not found at: ${schemaPath}`);
}
const schemaSQL = fs.readFileSync(schemaPath, 'utf8');
const statements = splitSQLStatements(schemaSQL);
outputProgress({
operation: 'Schema loaded',
message: `Found ${statements.length} SQL statements to execute`
});
// Execute schema statements
for (let i = 0; i < statements.length; i++) {
const stmt = statements[i];
try {
const [result] = await connection.query(stmt);
// Check for warnings
const [warnings] = await connection.query('SHOW WARNINGS');
if (warnings && warnings.length > 0) {
outputProgress({
status: 'warning',
operation: 'SQL Warning',
message: warnings
});
}
outputProgress({
operation: 'SQL Progress',
message: {
statement: i + 1,
total: statements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
affectedRows: result.affectedRows
}
});
} catch (sqlError) {
outputProgress({
status: 'error',
operation: 'SQL Error',
message: {
error: sqlError.message,
sqlState: sqlError.sqlState,
errno: sqlError.errno,
statement: stmt,
statementNumber: i + 1
}
});
throw sqlError;
}
}
// Verify metrics tables were created
outputProgress({
operation: 'Verifying metrics tables',
message: 'Checking all metrics tables were created...'
});
const [metricsTablesResult] = await connection.query(`
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN (?)
`, [METRICS_TABLES]);
const existingMetricsTables = metricsTablesResult.map(t => t.table_name);
const missingMetricsTables = METRICS_TABLES.filter(t => !existingMetricsTables.includes(t));
if (missingMetricsTables.length > 0) {
throw new Error(`Failed to create metrics tables: ${missingMetricsTables.join(', ')}`);
}
await connection.commit();
console.log('All metrics tables reset successfully');
outputProgress({
status: 'complete',
operation: 'Reset complete',
message: 'All metrics tables have been reset successfully'
});
} catch (error) {
outputProgress({
status: 'error',
operation: 'Reset failed',
message: error.message,
stack: error.stack
});
if (connection) {
await connection.rollback();
}
console.error('Error resetting metrics:', error);
throw error;
} finally {
if (connection) {

243
inventory-server/scripts/update-csv.js
View File

@@ -1,167 +1,180 @@
const fs = require('fs');
const path = require('path');
const https = require('https');
const fs = require('fs');
const axios = require('axios');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('./metrics/utils/progress');
// Change working directory to script directory
process.chdir(path.dirname(__filename));
require('dotenv').config({ path: path.resolve(__dirname, '..', '.env') });
// Configuration
const FILES = [
{
name: '39f2x83-products.csv',
url: 'https://feeds.acherryontop.com/39f2x83-products.csv'
url: process.env.PRODUCTS_CSV_URL
},
{
name: '39f2x83-orders.csv',
url: 'https://feeds.acherryontop.com/39f2x83-orders.csv'
url: process.env.ORDERS_CSV_URL
},
{
name: '39f2x83-purchase_orders.csv',
url: 'https://feeds.acherryontop.com/39f2x83-purchase_orders.csv'
url: process.env.PURCHASE_ORDERS_CSV_URL
}
];
const CSV_DIR = path.join(__dirname, '..', 'csv');
let isCancelled = false;
// Ensure CSV directory exists
if (!fs.existsSync(CSV_DIR)) {
fs.mkdirSync(CSV_DIR, { recursive: true });
function cancelUpdate() {
isCancelled = true;
outputProgress({
status: 'cancelled',
operation: 'CSV update cancelled',
current: 0,
total: FILES.length,
elapsed: null,
remaining: null,
rate: 0
});
}
// Function to download a file
function downloadFile(url, filePath) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(filePath);
async function downloadFile(file, index, startTime) {
if (isCancelled) return;
https.get(url, response => {
if (response.statusCode !== 200) {
reject(new Error(`Failed to download: ${response.statusCode} ${response.statusMessage}`));
const csvDir = path.join(__dirname, '../csv');
if (!fs.existsSync(csvDir)) {
fs.mkdirSync(csvDir, { recursive: true });
}
const writer = fs.createWriteStream(path.join(csvDir, file.name));
try {
const response = await axios({
url: file.url,
method: 'GET',
responseType: 'stream'
});
const totalLength = response.headers['content-length'];
let downloadedLength = 0;
let lastProgressUpdate = Date.now();
const PROGRESS_INTERVAL = 1000; // Update progress every second
response.data.on('data', (chunk) => {
if (isCancelled) {
writer.end();
return;
}
const totalSize = parseInt(response.headers['content-length'], 10);
let downloadedSize = 0;
let lastProgressUpdate = Date.now();
const startTime = Date.now();
downloadedLength += chunk.length;
response.on('data', chunk => {
downloadedSize += chunk.length;
// Update progress based on time interval
const now = Date.now();
// Update progress at most every 100ms to avoid console flooding
if (now - lastProgressUpdate > 100) {
const elapsed = (now - startTime) / 1000;
const rate = downloadedSize / elapsed;
const remaining = (totalSize - downloadedSize) / rate;
console.log(JSON.stringify({
if (now - lastProgressUpdate >= PROGRESS_INTERVAL) {
const progress = (downloadedLength / totalLength) * 100;
outputProgress({
status: 'running',
operation: `Downloading ${path.basename(filePath)}`,
current: downloadedSize,
total: totalSize,
rate: (rate / 1024 / 1024).toFixed(2), // MB/s
elapsed: formatDuration(elapsed),
remaining: formatDuration(remaining),
percentage: ((downloadedSize / totalSize) * 100).toFixed(1)
}));
operation: `Downloading ${file.name}`,
current: index + (downloadedLength / totalLength),
total: FILES.length,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, index + (downloadedLength / totalLength), FILES.length),
rate: calculateRate(startTime, index + (downloadedLength / totalLength)),
percentage: progress.toFixed(1),
file_progress: {
name: file.name,
downloaded: downloadedLength,
total: totalLength,
percentage: progress.toFixed(1)
}
});
lastProgressUpdate = now;
}
});
response.pipe(file);
response.data.pipe(writer);
file.on('finish', () => {
console.log(JSON.stringify({
status: 'running',
operation: `Completed ${path.basename(filePath)}`,
current: totalSize,
total: totalSize,
percentage: '100'
}));
file.close();
resolve();
return new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', reject);
});
}).on('error', error => {
fs.unlink(filePath, () => {}); // Delete the file if download failed
reject(error);
});
file.on('error', error => {
fs.unlink(filePath, () => {}); // Delete the file if there was an error
reject(error);
});
});
}
// Helper function to format duration
function formatDuration(seconds) {
if (seconds < 60) return `${Math.round(seconds)}s`;
const minutes = Math.floor(seconds / 60);
seconds = Math.round(seconds % 60);
return `${minutes}m ${seconds}s`;
} catch (error) {
fs.unlinkSync(path.join(csvDir, file.name));
throw error;
}
}
// Main function to update all files
async function updateFiles() {
console.log(JSON.stringify({
status: 'running',
operation: 'Starting CSV file updates',
total: FILES.length,
current: 0
}));
const startTime = Date.now();
for (let i = 0; i < FILES.length; i++) {
const file = FILES[i];
const filePath = path.join(CSV_DIR, file.name);
outputProgress({
status: 'running',
operation: 'Starting CSV update',
current: 0,
total: FILES.length,
elapsed: '0s',
remaining: null,
rate: 0,
percentage: '0'
});
try {
// Delete existing file if it exists
if (fs.existsSync(filePath)) {
console.log(JSON.stringify({
status: 'running',
operation: `Removing existing file: ${file.name}`,
current: i,
total: FILES.length,
percentage: ((i / FILES.length) * 100).toFixed(1)
}));
fs.unlinkSync(filePath);
for (let i = 0; i < FILES.length; i++) {
if (isCancelled) {
return;
}
// Download new file
console.log(JSON.stringify({
const file = FILES[i];
await downloadFile(file, i, startTime);
outputProgress({
status: 'running',
operation: `Starting download: ${file.name}`,
current: i,
total: FILES.length,
percentage: ((i / FILES.length) * 100).toFixed(1)
}));
await downloadFile(file.url, filePath);
console.log(JSON.stringify({
status: 'running',
operation: `Successfully updated ${file.name}`,
operation: 'CSV update in progress',
current: i + 1,
total: FILES.length,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, i + 1, FILES.length),
rate: calculateRate(startTime, i + 1),
percentage: (((i + 1) / FILES.length) * 100).toFixed(1)
}));
} catch (error) {
console.error(JSON.stringify({
status: 'error',
operation: `Error updating ${file.name}`,
error: error.message
}));
throw error;
}
});
}
console.log(JSON.stringify({
outputProgress({
status: 'complete',
operation: 'CSV file update complete',
operation: 'CSV update complete',
current: FILES.length,
total: FILES.length,
elapsed: formatElapsedTime(startTime),
remaining: '0s',
rate: calculateRate(startTime, FILES.length),
percentage: '100'
}));
});
} catch (error) {
outputProgress({
status: 'error',
operation: 'CSV update failed',
error: error.message,
current: 0,
total: FILES.length,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: 0
});
throw error;
}
}
// Run the update
updateFiles().catch(error => {
console.error(JSON.stringify({
error: `Update failed: ${error.message}`
}));
// Run the update only if this is the main module
if (require.main === module) {
updateFiles().catch((error) => {
console.error('Error updating CSV files:', error);
process.exit(1);
});
});
}
// Export the functions needed by the route
module.exports = {
updateFiles,
cancelUpdate
};