From 0a51328da2998477d004dc360fb649f251a08508 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 1 Feb 2025 14:46:17 -0500 Subject: [PATCH] Add a bunch of untested calculations enhancements based on import script changes --- inventory-server/scripts/calculate-metrics.js | 72 ++++-- .../scripts/metrics/brand-metrics.js | 28 ++- .../scripts/metrics/category-metrics.js | 216 ++++++++++++++++-- .../scripts/metrics/financial-metrics.js | 34 ++- .../scripts/metrics/product-metrics.js | 94 +++++++- .../scripts/metrics/sales-forecasts.js | 56 ++++- .../scripts/metrics/time-aggregates.js | 96 +++++++- .../scripts/metrics/vendor-metrics.js | 139 ++++++++++- 8 files changed, 659 insertions(+), 76 deletions(-) diff --git a/inventory-server/scripts/calculate-metrics.js b/inventory-server/scripts/calculate-metrics.js index 1a12d57..20e37e9 100644 --- a/inventory-server/scripts/calculate-metrics.js +++ b/inventory-server/scripts/calculate-metrics.js @@ -115,7 +115,12 @@ async function calculateMetrics() { elapsed: '0s', remaining: 'Calculating...', rate: 0, - percentage: '0' + percentage: '0', + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Get total number of products @@ -139,7 +144,12 @@ async function calculateMetrics() { elapsed: global.formatElapsedTime(startTime), remaining: global.estimateRemaining(startTime, processedCount, totalProducts), rate: global.calculateRate(startTime, processedCount), - percentage: '60' + percentage: '60', + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } @@ -194,7 +204,12 @@ async function calculateMetrics() { elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -223,7 +238,12 @@ async function calculateMetrics() { elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -247,6 +267,7 @@ async function calculateMetrics() { // Get total count for percentage calculation const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks'); const totalCount = rankingCount[0].total_count || 1; + const max_rank = totalCount; // Store max_rank for use in classification outputProgress({ status: 'running', @@ -256,7 +277,12 @@ async function calculateMetrics() { elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -282,8 +308,8 @@ async function calculateMetrics() { ELSE 'C' END LIMIT ? - `, [totalCount, abcThresholds.a_threshold, - totalCount, abcThresholds.b_threshold, + `, [max_rank, abcThresholds.a_threshold, + max_rank, abcThresholds.b_threshold, batchSize]); if (pids.length === 0) { @@ -303,8 +329,8 @@ async function calculateMetrics() { END, pm.last_calculated_at = NOW() WHERE pm.pid IN (?) - `, [totalCount, abcThresholds.a_threshold, - totalCount, abcThresholds.b_threshold, + `, [max_rank, abcThresholds.a_threshold, + max_rank, abcThresholds.b_threshold, pids.map(row => row.pid)]); abcProcessedCount += result.affectedRows; @@ -318,7 +344,12 @@ async function calculateMetrics() { elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Small delay between batches to allow other transactions @@ -337,7 +368,12 @@ async function calculateMetrics() { elapsed: formatElapsedTime(startTime), remaining: '0s', rate: calculateRate(startTime, totalProducts), - percentage: '100' + percentage: '100', + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Clear progress file on successful completion @@ -353,7 +389,12 @@ async function calculateMetrics() { elapsed: global.formatElapsedTime(startTime), remaining: null, rate: global.calculateRate(startTime, processedCount), - percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1) + percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } else { global.outputProgress({ @@ -364,7 +405,12 @@ async function calculateMetrics() { elapsed: global.formatElapsedTime(startTime), remaining: null, rate: global.calculateRate(startTime, processedCount), - percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1) + percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } throw error; diff --git a/inventory-server/scripts/metrics/brand-metrics.js b/inventory-server/scripts/metrics/brand-metrics.js index 5b90765..91b1b56 100644 --- a/inventory-server/scripts/metrics/brand-metrics.js +++ b/inventory-server/scripts/metrics/brand-metrics.js @@ -13,7 +13,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Calculate brand metrics with optimized queries @@ -134,7 +144,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -207,7 +222,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; diff --git a/inventory-server/scripts/metrics/category-metrics.js b/inventory-server/scripts/metrics/category-metrics.js index 9a658bb..912d77a 100644 --- a/inventory-server/scripts/metrics/category-metrics.js +++ b/inventory-server/scripts/metrics/category-metrics.js @@ -13,7 +13,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // First, calculate base category metrics @@ -67,7 +77,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -80,19 +95,35 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount SUM(o.quantity * o.price) as total_sales, SUM(o.quantity * (o.price - p.cost_price)) as total_margin, SUM(o.quantity) as units_sold, - AVG(GREATEST(p.stock_quantity, 0)) as avg_stock + AVG(GREATEST(p.stock_quantity, 0)) as avg_stock, + COUNT(DISTINCT DATE(o.date)) as active_days FROM product_categories pc JOIN products p ON pc.pid = p.pid JOIN orders o ON p.pid = o.pid + LEFT JOIN turnover_config tc ON + (tc.category_id = pc.cat_id AND tc.vendor = p.vendor) OR + (tc.category_id = pc.cat_id AND tc.vendor IS NULL) OR + (tc.category_id IS NULL AND tc.vendor = p.vendor) OR + (tc.category_id IS NULL AND tc.vendor IS NULL) WHERE o.canceled = false - AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) + AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL COALESCE(tc.calculation_period_days, 30) DAY) GROUP BY pc.cat_id ) UPDATE category_metrics cm JOIN category_sales cs ON cm.category_id = cs.cat_id + LEFT JOIN turnover_config tc ON + (tc.category_id = cm.category_id AND tc.vendor IS NULL) OR + (tc.category_id IS NULL AND tc.vendor IS NULL) SET cm.avg_margin = COALESCE(cs.total_margin * 100.0 / NULLIF(cs.total_sales, 0), 0), - cm.turnover_rate = LEAST(COALESCE(cs.units_sold / NULLIF(cs.avg_stock, 0), 0), 999.99), + cm.turnover_rate = CASE + WHEN cs.avg_stock > 0 AND cs.active_days > 0 + THEN LEAST( + (cs.units_sold / cs.avg_stock) * (365.0 / cs.active_days), + 999.99 + ) + ELSE 0 + END, cm.last_calculated_at = NOW() `); @@ -105,7 +136,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -115,10 +151,11 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount WITH current_period AS ( SELECT pc.cat_id, - SUM(o.quantity * o.price) as revenue + SUM(o.quantity * o.price) / (1 + COALESCE(ss.seasonality_factor, 0)) as revenue FROM product_categories pc JOIN products p ON pc.pid = p.pid JOIN orders o ON p.pid = o.pid + LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month WHERE o.canceled = false AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH) GROUP BY pc.cat_id @@ -126,29 +163,63 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount previous_period AS ( SELECT pc.cat_id, - SUM(o.quantity * o.price) as revenue + SUM(o.quantity * o.price) / (1 + COALESCE(ss.seasonality_factor, 0)) as revenue FROM product_categories pc JOIN products p ON pc.pid = p.pid JOIN orders o ON p.pid = o.pid + LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month WHERE o.canceled = false AND o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH) AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) GROUP BY pc.cat_id + ), + trend_data AS ( + SELECT + pc.cat_id, + MONTH(o.date) as month, + SUM(o.quantity * o.price) / (1 + COALESCE(ss.seasonality_factor, 0)) as revenue, + COUNT(DISTINCT DATE(o.date)) as days_in_month + FROM product_categories pc + JOIN products p ON pc.pid = p.pid + JOIN orders o ON p.pid = o.pid + LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month + WHERE o.canceled = false + AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH) + GROUP BY pc.cat_id, MONTH(o.date) + ), + trend_analysis AS ( + SELECT + cat_id, + REGR_SLOPE(revenue / days_in_month, MONTH) as trend_slope, + AVG(revenue / days_in_month) as avg_daily_revenue + FROM trend_data + GROUP BY cat_id + HAVING COUNT(*) >= 6 ) UPDATE category_metrics cm LEFT JOIN current_period cp ON cm.category_id = cp.cat_id LEFT JOIN previous_period pp ON cm.category_id = pp.cat_id + LEFT JOIN trend_analysis ta ON cm.category_id = ta.cat_id SET cm.growth_rate = CASE WHEN pp.revenue = 0 AND COALESCE(cp.revenue, 0) > 0 THEN 100.0 WHEN pp.revenue = 0 THEN 0.0 - ELSE LEAST( - GREATEST( - ((COALESCE(cp.revenue, 0) - pp.revenue) / pp.revenue) * 100.0, - -100.0 - ), - 999.99 - ) + WHEN ta.trend_slope IS NOT NULL THEN + LEAST( + GREATEST( + (ta.trend_slope / NULLIF(ta.avg_daily_revenue, 0)) * 365 * 100, + -100.0 + ), + 999.99 + ) + ELSE + LEAST( + GREATEST( + ((COALESCE(cp.revenue, 0) - pp.revenue) / pp.revenue) * 100.0, + -100.0 + ), + 999.99 + ) END, cm.last_calculated_at = NOW() WHERE cp.cat_id IS NOT NULL OR pp.cat_id IS NOT NULL @@ -163,7 +234,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -210,19 +286,119 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount total_value = VALUES(total_value), total_revenue = VALUES(total_revenue), avg_margin = VALUES(avg_margin), - turnover_rate = VALUES(turnover_rate) + turnover_rate = VALUES(turnover_rate), + last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.99); outputProgress({ status: 'running', - operation: 'Time-based metrics calculated', + operation: 'Time-based metrics calculated, updating category-sales metrics', current: processedCount, total: totalProducts, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } + }); + + if (isCancelled) return processedCount; + + // Calculate category-sales metrics + await connection.query(` + INSERT INTO category_sales_metrics ( + category_id, + brand, + period_start, + period_end, + avg_daily_sales, + total_sold, + num_products, + avg_price, + last_calculated_at + ) + WITH date_ranges AS ( + SELECT + DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY) as period_start, + CURRENT_DATE as period_end + UNION ALL + SELECT + DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY), + DATE_SUB(CURRENT_DATE, INTERVAL 31 DAY) + UNION ALL + SELECT + DATE_SUB(CURRENT_DATE, INTERVAL 180 DAY), + DATE_SUB(CURRENT_DATE, INTERVAL 91 DAY) + UNION ALL + SELECT + DATE_SUB(CURRENT_DATE, INTERVAL 365 DAY), + DATE_SUB(CURRENT_DATE, INTERVAL 181 DAY) + ), + sales_data AS ( + SELECT + pc.cat_id, + p.brand, + dr.period_start, + dr.period_end, + COUNT(DISTINCT p.pid) as num_products, + SUM(o.quantity) as total_sold, + SUM(o.quantity * o.price) as total_revenue, + COUNT(DISTINCT DATE(o.date)) as num_days + FROM products p + JOIN product_categories pc ON p.pid = pc.pid + JOIN orders o ON p.pid = o.pid + CROSS JOIN date_ranges dr + WHERE o.canceled = false + AND o.date BETWEEN dr.period_start AND dr.period_end + GROUP BY pc.cat_id, p.brand, dr.period_start, dr.period_end + ) + SELECT + cat_id as category_id, + brand, + period_start, + period_end, + CASE + WHEN num_days > 0 + THEN total_sold / num_days + ELSE 0 + END as avg_daily_sales, + total_sold, + num_products, + CASE + WHEN total_sold > 0 + THEN total_revenue / total_sold + ELSE 0 + END as avg_price, + NOW() as last_calculated_at + FROM sales_data + ON DUPLICATE KEY UPDATE + avg_daily_sales = VALUES(avg_daily_sales), + total_sold = VALUES(total_sold), + num_products = VALUES(num_products), + avg_price = VALUES(avg_price), + last_calculated_at = VALUES(last_calculated_at) + `); + + processedCount = Math.floor(totalProducts * 1.0); + outputProgress({ + status: 'running', + operation: 'Category-sales metrics calculated', + current: processedCount, + total: totalProducts, + elapsed: formatElapsedTime(startTime), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; diff --git a/inventory-server/scripts/metrics/financial-metrics.js b/inventory-server/scripts/metrics/financial-metrics.js index 3c85871..acb03a0 100644 --- a/inventory-server/scripts/metrics/financial-metrics.js +++ b/inventory-server/scripts/metrics/financial-metrics.js @@ -13,7 +13,12 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Calculate financial metrics with optimized query @@ -59,7 +69,8 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun WHEN COALESCE(pf.inventory_value, 0) > 0 AND pf.active_days > 0 THEN (COALESCE(pf.gross_profit, 0) * (365.0 / pf.active_days)) / COALESCE(pf.inventory_value, 0) ELSE 0 - END + END, + pm.last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.65); @@ -71,7 +82,12 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -103,7 +119,8 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun WHEN COALESCE(mf.inventory_value, 0) > 0 AND mf.active_days > 0 THEN (COALESCE(mf.gross_profit, 0) * (365.0 / mf.active_days)) / COALESCE(mf.inventory_value, 0) ELSE 0 - END + END, + pta.last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.70); @@ -115,7 +132,12 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; diff --git a/inventory-server/scripts/metrics/product-metrics.js b/inventory-server/scripts/metrics/product-metrics.js index ed177e6..2b174f6 100644 --- a/inventory-server/scripts/metrics/product-metrics.js +++ b/inventory-server/scripts/metrics/product-metrics.js @@ -25,7 +25,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -40,7 +45,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Calculate base metrics @@ -77,6 +87,16 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount END, pm.first_sale_date = stats.first_sale_date, pm.last_sale_date = stats.last_sale_date, + pm.days_of_inventory = CASE + WHEN COALESCE(stats.total_quantity / NULLIF(stats.active_days, 0), 0) > 0 + THEN FLOOR(p.stock_quantity / (stats.total_quantity / stats.active_days)) + ELSE NULL + END, + pm.weeks_of_inventory = CASE + WHEN COALESCE(stats.total_quantity / NULLIF(stats.active_days, 0), 0) > 0 + THEN FLOOR(p.stock_quantity / (stats.total_quantity / stats.active_days) / 7) + ELSE NULL + END, pm.gmroi = CASE WHEN COALESCE(stats.inventory_value, 0) > 0 THEN (stats.total_revenue - stats.cost_of_goods_sold) / stats.inventory_value @@ -85,6 +105,38 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount pm.last_calculated_at = NOW() `); + // Calculate forecast accuracy and bias + await connection.query(` + WITH forecast_accuracy AS ( + SELECT + sf.pid, + AVG(CASE + WHEN o.quantity > 0 + THEN ABS(sf.forecast_units - o.quantity) / o.quantity * 100 + ELSE 100 + END) as avg_forecast_error, + AVG(CASE + WHEN o.quantity > 0 + THEN (sf.forecast_units - o.quantity) / o.quantity * 100 + ELSE 0 + END) as avg_forecast_bias, + MAX(sf.forecast_date) as last_forecast_date + FROM sales_forecasts sf + JOIN orders o ON sf.pid = o.pid + AND DATE(o.date) = sf.forecast_date + WHERE o.canceled = false + AND sf.forecast_date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY) + GROUP BY sf.pid + ) + UPDATE product_metrics pm + JOIN forecast_accuracy fa ON pm.pid = fa.pid + SET + pm.forecast_accuracy = GREATEST(0, 100 - LEAST(fa.avg_forecast_error, 100)), + pm.forecast_bias = GREATEST(-100, LEAST(fa.avg_forecast_bias, 100)), + pm.last_forecast_date = fa.last_forecast_date, + pm.last_calculated_at = NOW() + `); + processedCount = Math.floor(totalProducts * 0.4); outputProgress({ status: 'running', @@ -94,7 +146,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } else { processedCount = Math.floor(totalProducts * 0.4); @@ -106,7 +163,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } @@ -122,7 +184,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Calculate time-based aggregates @@ -172,7 +239,8 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount avg_price = VALUES(avg_price), profit_margin = VALUES(profit_margin), inventory_value = VALUES(inventory_value), - gmroi = VALUES(gmroi) + gmroi = VALUES(gmroi), + last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.6); @@ -184,7 +252,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } else { processedCount = Math.floor(totalProducts * 0.6); @@ -196,7 +269,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); } diff --git a/inventory-server/scripts/metrics/sales-forecasts.js b/inventory-server/scripts/metrics/sales-forecasts.js index f02ddb0..5894294 100644 --- a/inventory-server/scripts/metrics/sales-forecasts.js +++ b/inventory-server/scripts/metrics/sales-forecasts.js @@ -13,7 +13,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // First, create a temporary table for forecast dates @@ -65,7 +75,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -94,7 +109,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -119,7 +139,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -181,7 +206,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -221,7 +251,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -292,7 +327,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; diff --git a/inventory-server/scripts/metrics/time-aggregates.js b/inventory-server/scripts/metrics/time-aggregates.js index 7c8e436..26806e9 100644 --- a/inventory-server/scripts/metrics/time-aggregates.js +++ b/inventory-server/scripts/metrics/time-aggregates.js @@ -13,7 +13,12 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // Initial insert of time-based aggregates @@ -71,10 +81,40 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, YEAR(date) as year, MONTH(date) as month, SUM(received) as stock_received, - SUM(ordered) as stock_ordered + SUM(ordered) as stock_ordered, + COUNT(DISTINCT CASE WHEN receiving_status = 40 THEN id END) as fulfilled_orders, + COUNT(DISTINCT id) as total_orders, + AVG(CASE + WHEN receiving_status = 40 + THEN DATEDIFF(received_date, date) + END) as avg_lead_time, + SUM(CASE + WHEN receiving_status = 40 AND received_date > expected_date + THEN 1 ELSE 0 + END) as late_deliveries FROM purchase_orders - WHERE status = 50 GROUP BY pid, YEAR(date), MONTH(date) + ), + stock_trends AS ( + SELECT + p.pid, + YEAR(po.date) as year, + MONTH(po.date) as month, + AVG(p.stock_quantity) as avg_stock_level, + STDDEV(p.stock_quantity) as stock_volatility, + SUM(CASE + WHEN p.stock_quantity <= COALESCE(pm.reorder_point, 5) + THEN 1 ELSE 0 + END) as days_below_reorder, + COUNT(*) as total_days + FROM products p + CROSS JOIN ( + SELECT DISTINCT DATE(date) as date + FROM purchase_orders + WHERE date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) + ) po + LEFT JOIN product_metrics pm ON p.pid = pm.pid + GROUP BY p.pid, YEAR(po.date), MONTH(po.date) ) SELECT s.pid, @@ -87,12 +127,24 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, COALESCE(p.stock_received, 0) as stock_received, COALESCE(p.stock_ordered, 0) as stock_ordered, s.avg_price, - s.profit_margin + s.profit_margin, + COALESCE(p.fulfilled_orders, 0) as fulfilled_orders, + COALESCE(p.total_orders, 0) as total_orders, + COALESCE(p.avg_lead_time, 0) as avg_lead_time, + COALESCE(p.late_deliveries, 0) as late_deliveries, + COALESCE(st.avg_stock_level, 0) as avg_stock_level, + COALESCE(st.stock_volatility, 0) as stock_volatility, + COALESCE(st.days_below_reorder, 0) as days_below_reorder, + COALESCE(st.total_days, 0) as total_days FROM sales_data s LEFT JOIN purchase_data p ON s.pid = p.pid AND s.year = p.year AND s.month = p.month + LEFT JOIN stock_trends st + ON s.pid = st.pid + AND s.year = st.year + AND s.month = st.month UNION SELECT p.pid, @@ -105,12 +157,24 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, p.stock_received, p.stock_ordered, 0 as avg_price, - 0 as profit_margin + 0 as profit_margin, + p.fulfilled_orders, + p.total_orders, + p.avg_lead_time, + p.late_deliveries, + st.avg_stock_level, + st.stock_volatility, + st.days_below_reorder, + st.total_days FROM purchase_data p LEFT JOIN sales_data s ON p.pid = s.pid AND p.year = s.year AND p.month = s.month + LEFT JOIN stock_trends st + ON p.pid = st.pid + AND p.year = st.year + AND p.month = st.month WHERE s.pid IS NULL ON DUPLICATE KEY UPDATE total_quantity_sold = VALUES(total_quantity_sold), @@ -120,7 +184,8 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, stock_received = VALUES(stock_received), stock_ordered = VALUES(stock_ordered), avg_price = VALUES(avg_price), - profit_margin = VALUES(profit_margin) + profit_margin = VALUES(profit_margin), + last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.60); @@ -132,7 +197,12 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -161,7 +231,8 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, WHEN COALESCE(fin.inventory_value, 0) > 0 AND fin.days_in_period > 0 THEN (COALESCE(fin.gross_profit, 0) * (365.0 / fin.days_in_period)) / COALESCE(fin.inventory_value, 0) ELSE 0 - END + END, + pta.last_calculated_at = CURRENT_TIMESTAMP `); processedCount = Math.floor(totalProducts * 0.65); @@ -173,7 +244,12 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; diff --git a/inventory-server/scripts/metrics/vendor-metrics.js b/inventory-server/scripts/metrics/vendor-metrics.js index e8be0b0..396021e 100644 --- a/inventory-server/scripts/metrics/vendor-metrics.js +++ b/inventory-server/scripts/metrics/vendor-metrics.js @@ -13,7 +13,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount; } @@ -26,7 +31,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); // First ensure all vendors exist in vendor_details @@ -50,7 +60,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); if (isCancelled) return processedCount; @@ -68,6 +83,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, avg_order_value, active_products, total_products, + total_purchase_value, + avg_margin_percent, status, last_calculated_at ) @@ -76,7 +93,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, p.vendor, SUM(o.quantity * o.price) as total_revenue, COUNT(DISTINCT o.id) as total_orders, - COUNT(DISTINCT p.pid) as active_products + COUNT(DISTINCT p.pid) as active_products, + SUM(o.quantity * (o.price - p.cost_price)) as total_margin FROM products p JOIN orders o ON p.pid = o.pid WHERE o.canceled = false @@ -91,7 +109,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, AVG(CASE WHEN po.receiving_status = 40 THEN DATEDIFF(po.received_date, po.date) - END) as avg_lead_time_days + END) as avg_lead_time_days, + SUM(po.ordered * po.po_cost_price) as total_purchase_value FROM products p JOIN purchase_orders po ON p.pid = po.pid WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) @@ -127,6 +146,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, END as avg_order_value, COALESCE(vs.active_products, 0) as active_products, COALESCE(vpr.total_products, 0) as total_products, + COALESCE(vp.total_purchase_value, 0) as total_purchase_value, + CASE + WHEN vs.total_revenue > 0 + THEN (vs.total_margin / vs.total_revenue) * 100 + ELSE 0 + END as avg_margin_percent, 'active' as status, NOW() as last_calculated_at FROM vendor_sales vs @@ -143,6 +168,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, avg_order_value = VALUES(avg_order_value), active_products = VALUES(active_products), total_products = VALUES(total_products), + total_purchase_value = VALUES(total_purchase_value), + avg_margin_percent = VALUES(avg_margin_percent), status = VALUES(status), last_calculated_at = VALUES(last_calculated_at) `); @@ -150,13 +177,111 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, processedCount = Math.floor(totalProducts * 0.9); outputProgress({ status: 'running', - operation: 'Vendor metrics calculated', + operation: 'Vendor metrics calculated, updating time-based metrics', current: processedCount, total: totalProducts, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1) + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } + }); + + if (isCancelled) return processedCount; + + // Calculate time-based metrics + await connection.query(` + INSERT INTO vendor_time_metrics ( + vendor, + year, + month, + total_orders, + late_orders, + avg_lead_time_days, + total_purchase_value, + total_revenue, + avg_margin_percent + ) + WITH monthly_orders AS ( + SELECT + p.vendor, + YEAR(o.date) as year, + MONTH(o.date) as month, + COUNT(DISTINCT o.id) as total_orders, + SUM(o.quantity * o.price) as total_revenue, + SUM(o.quantity * (o.price - p.cost_price)) as total_margin + FROM products p + JOIN orders o ON p.pid = o.pid + WHERE o.canceled = false + AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) + GROUP BY p.vendor, YEAR(o.date), MONTH(o.date) + ), + monthly_po AS ( + SELECT + p.vendor, + YEAR(po.date) as year, + MONTH(po.date) as month, + COUNT(DISTINCT po.id) as total_po, + COUNT(DISTINCT CASE + WHEN po.receiving_status = 40 AND po.received_date > po.expected_date + THEN po.id + END) as late_orders, + AVG(CASE + WHEN po.receiving_status = 40 + THEN DATEDIFF(po.received_date, po.date) + END) as avg_lead_time_days, + SUM(po.ordered * po.po_cost_price) as total_purchase_value + FROM products p + JOIN purchase_orders po ON p.pid = po.pid + WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) + GROUP BY p.vendor, YEAR(po.date), MONTH(po.date) + ) + SELECT + mo.vendor, + mo.year, + mo.month, + COALESCE(mp.total_po, 0) as total_orders, + COALESCE(mp.late_orders, 0) as late_orders, + COALESCE(mp.avg_lead_time_days, 0) as avg_lead_time_days, + COALESCE(mp.total_purchase_value, 0) as total_purchase_value, + COALESCE(mo.total_revenue, 0) as total_revenue, + CASE + WHEN mo.total_revenue > 0 + THEN (mo.total_margin / mo.total_revenue) * 100 + ELSE 0 + END as avg_margin_percent + FROM monthly_orders mo + LEFT JOIN monthly_po mp ON mo.vendor = mp.vendor + AND mo.year = mp.year + AND mo.month = mp.month + ON DUPLICATE KEY UPDATE + total_orders = VALUES(total_orders), + late_orders = VALUES(late_orders), + avg_lead_time_days = VALUES(avg_lead_time_days), + total_purchase_value = VALUES(total_purchase_value), + total_revenue = VALUES(total_revenue), + avg_margin_percent = VALUES(avg_margin_percent) + `); + + processedCount = Math.floor(totalProducts * 0.95); + outputProgress({ + status: 'running', + operation: 'Time-based vendor metrics calculated', + current: processedCount, + total: totalProducts, + elapsed: formatElapsedTime(startTime), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } }); return processedCount;