From 2a6a0d0a87e60de0af06de43f2c6b61a73b857f7 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 5 Feb 2025 00:02:06 -0500 Subject: [PATCH] Fixed calculations for frontend (likely still wrong but they display) + related regressions to calculate script --- .../scripts/metrics/product-metrics.js | 306 +++++++++++------- 1 file changed, 186 insertions(+), 120 deletions(-) diff --git a/inventory-server/scripts/metrics/product-metrics.js b/inventory-server/scripts/metrics/product-metrics.js index d1a7cd4..e08de56 100644 --- a/inventory-server/scripts/metrics/product-metrics.js +++ b/inventory-server/scripts/metrics/product-metrics.js @@ -13,6 +13,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount const connection = await getConnection(); let success = false; let processedOrders = 0; + const BATCH_SIZE = 5000; try { // Skip flags are inherited from the parent scope @@ -44,7 +45,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount return { processedProducts: processedCount, processedOrders, - processedPurchaseOrders: 0, // This module doesn't process POs + processedPurchaseOrders: 0, success }; } @@ -56,6 +57,15 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount FROM products `); + // Get threshold settings once + const [thresholds] = await connection.query(` + SELECT critical_days, reorder_days, overstock_days, low_stock_threshold + FROM stock_thresholds + WHERE category_id IS NULL AND vendor IS NULL + LIMIT 1 + `); + const defaultThresholds = thresholds[0]; + // Calculate base product metrics if (!SKIP_PRODUCT_BASE_METRICS) { outputProgress({ @@ -82,134 +92,190 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount `); processedOrders = orderCount[0].count; - // Calculate base metrics + // Clear temporary tables + await connection.query('TRUNCATE TABLE temp_sales_metrics'); + await connection.query('TRUNCATE TABLE temp_purchase_metrics'); + + // Populate temp_sales_metrics with base stats and sales averages await connection.query(` - UPDATE product_metrics pm - JOIN ( - SELECT - p.pid, - p.stock_quantity, - p.cost_price, - p.cost_price * p.stock_quantity as inventory_value, - SUM(o.quantity) as total_quantity, - COUNT(DISTINCT o.order_number) as number_of_orders, - SUM(o.quantity * o.price) as total_revenue, - SUM(o.quantity * p.cost_price) as cost_of_goods_sold, - AVG(o.price) as avg_price, - STDDEV(o.price) as price_std, - MIN(o.date) as first_sale_date, - MAX(o.date) as last_sale_date, - COUNT(DISTINCT DATE(o.date)) as active_days - FROM products p - LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false - GROUP BY p.pid, p.stock_quantity, p.cost_price - ) stats ON pm.pid = stats.pid - SET - pm.inventory_value = COALESCE(stats.inventory_value, 0), - pm.avg_quantity_per_order = COALESCE(stats.total_quantity / NULLIF(stats.number_of_orders, 0), 0), - pm.number_of_orders = COALESCE(stats.number_of_orders, 0), - pm.total_revenue = COALESCE(stats.total_revenue, 0), - pm.cost_of_goods_sold = COALESCE(stats.cost_of_goods_sold, 0), - pm.gross_profit = COALESCE(stats.total_revenue - stats.cost_of_goods_sold, 0), - pm.avg_margin_percent = CASE - WHEN COALESCE(stats.total_revenue, 0) > 0 - THEN ((stats.total_revenue - stats.cost_of_goods_sold) / stats.total_revenue) * 100 + INSERT INTO temp_sales_metrics + SELECT + p.pid, + COALESCE(SUM(o.quantity) / NULLIF(COUNT(DISTINCT DATE(o.date)), 0), 0) as daily_sales_avg, + COALESCE(SUM(o.quantity) / NULLIF(CEIL(COUNT(DISTINCT DATE(o.date)) / 7), 0), 0) as weekly_sales_avg, + COALESCE(SUM(o.quantity) / NULLIF(CEIL(COUNT(DISTINCT DATE(o.date)) / 30), 0), 0) as monthly_sales_avg, + COALESCE(SUM(o.quantity * o.price), 0) as total_revenue, + CASE + WHEN SUM(o.quantity * o.price) > 0 + THEN ((SUM(o.quantity * o.price) - SUM(o.quantity * p.cost_price)) / SUM(o.quantity * o.price)) * 100 ELSE 0 - END, - pm.first_sale_date = stats.first_sale_date, - pm.last_sale_date = stats.last_sale_date, - pm.days_of_inventory = CASE - WHEN COALESCE(stats.total_quantity / NULLIF(stats.active_days, 0), 0) > 0 - THEN FLOOR(stats.stock_quantity / (stats.total_quantity / stats.active_days)) - ELSE NULL - END, - pm.weeks_of_inventory = CASE - WHEN COALESCE(stats.total_quantity / NULLIF(stats.active_days, 0), 0) > 0 - THEN FLOOR(stats.stock_quantity / (stats.total_quantity / stats.active_days) / 7) - ELSE NULL - END, - pm.gmroi = CASE - WHEN COALESCE(stats.inventory_value, 0) > 0 - THEN (stats.total_revenue - stats.cost_of_goods_sold) / stats.inventory_value - ELSE 0 - END, - pm.last_calculated_at = NOW() + END as avg_margin_percent, + MIN(o.date) as first_sale_date, + MAX(o.date) as last_sale_date + FROM products p + LEFT JOIN orders o ON p.pid = o.pid + AND o.canceled = false + AND o.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY) + GROUP BY p.pid `); - // Calculate forecast accuracy and bias + // Populate temp_purchase_metrics await connection.query(` - WITH forecast_accuracy AS ( - SELECT - sf.pid, - AVG(CASE - WHEN o.quantity > 0 - THEN ABS(sf.forecast_units - o.quantity) / o.quantity * 100 - ELSE 100 - END) as avg_forecast_error, - AVG(CASE - WHEN o.quantity > 0 - THEN (sf.forecast_units - o.quantity) / o.quantity * 100 - ELSE 0 - END) as avg_forecast_bias, - MAX(sf.forecast_date) as last_forecast_date - FROM sales_forecasts sf - JOIN orders o ON sf.pid = o.pid - AND DATE(o.date) = sf.forecast_date - WHERE o.canceled = false - AND sf.forecast_date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY) - GROUP BY sf.pid - ) - UPDATE product_metrics pm - JOIN forecast_accuracy fa ON pm.pid = fa.pid - SET - pm.forecast_accuracy = GREATEST(0, 100 - LEAST(fa.avg_forecast_error, 100)), - pm.forecast_bias = GREATEST(-100, LEAST(fa.avg_forecast_bias, 100)), - pm.last_forecast_date = fa.last_forecast_date, - pm.last_calculated_at = NOW() + INSERT INTO temp_purchase_metrics + SELECT + p.pid, + AVG(DATEDIFF(po.received_date, po.date)) as avg_lead_time_days, + MAX(po.date) as last_purchase_date, + MIN(po.received_date) as first_received_date, + MAX(po.received_date) as last_received_date + FROM products p + LEFT JOIN purchase_orders po ON p.pid = po.pid + AND po.received_date IS NOT NULL + AND po.date >= DATE_SUB(CURDATE(), INTERVAL 365 DAY) + GROUP BY p.pid `); - processedCount = Math.floor(totalProducts * 0.4); - outputProgress({ - status: 'running', - operation: 'Base product metrics calculated', - current: processedCount || 0, - total: totalProducts || 0, - elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0), - rate: calculateRate(startTime, processedCount || 0), - percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1), - timing: { - start_time: new Date(startTime).toISOString(), - end_time: new Date().toISOString(), - elapsed_seconds: Math.round((Date.now() - startTime) / 1000) - } - }); - } else { - processedCount = Math.floor(totalProducts * 0.4); - outputProgress({ - status: 'running', - operation: 'Skipping base product metrics calculation', - current: processedCount || 0, - total: totalProducts || 0, - elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0), - rate: calculateRate(startTime, processedCount || 0), - percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1), - timing: { - start_time: new Date(startTime).toISOString(), - end_time: new Date().toISOString(), - elapsed_seconds: Math.round((Date.now() - startTime) / 1000) - } - }); + // Process updates in batches + let lastPid = 0; + while (true) { + if (isCancelled) break; + + const [batch] = await connection.query( + 'SELECT pid FROM products WHERE pid > ? ORDER BY pid LIMIT ?', + [lastPid, BATCH_SIZE] + ); + + if (batch.length === 0) break; + + await connection.query(` + UPDATE product_metrics pm + JOIN products p ON pm.pid = p.pid + LEFT JOIN temp_sales_metrics sm ON pm.pid = sm.pid + LEFT JOIN temp_purchase_metrics lm ON pm.pid = lm.pid + SET + pm.inventory_value = p.stock_quantity * p.cost_price, + pm.daily_sales_avg = COALESCE(sm.daily_sales_avg, 0), + pm.weekly_sales_avg = COALESCE(sm.weekly_sales_avg, 0), + pm.monthly_sales_avg = COALESCE(sm.monthly_sales_avg, 0), + pm.total_revenue = COALESCE(sm.total_revenue, 0), + pm.avg_margin_percent = COALESCE(sm.avg_margin_percent, 0), + pm.first_sale_date = sm.first_sale_date, + pm.last_sale_date = sm.last_sale_date, + pm.avg_lead_time_days = COALESCE(lm.avg_lead_time_days, 30), + pm.days_of_inventory = CASE + WHEN COALESCE(sm.daily_sales_avg, 0) > 0 + THEN FLOOR(p.stock_quantity / sm.daily_sales_avg) + ELSE NULL + END, + pm.weeks_of_inventory = CASE + WHEN COALESCE(sm.weekly_sales_avg, 0) > 0 + THEN FLOOR(p.stock_quantity / sm.weekly_sales_avg) + ELSE NULL + END, + pm.stock_status = CASE + WHEN p.stock_quantity <= 0 THEN 'Out of Stock' + WHEN COALESCE(sm.daily_sales_avg, 0) = 0 AND p.stock_quantity <= ? THEN 'Low Stock' + WHEN COALESCE(sm.daily_sales_avg, 0) = 0 THEN 'In Stock' + WHEN p.stock_quantity / NULLIF(sm.daily_sales_avg, 0) <= ? THEN 'Critical' + WHEN p.stock_quantity / NULLIF(sm.daily_sales_avg, 0) <= ? THEN 'Reorder' + WHEN p.stock_quantity / NULLIF(sm.daily_sales_avg, 0) > ? THEN 'Overstocked' + ELSE 'Healthy' + END, + pm.reorder_qty = CASE + WHEN COALESCE(sm.daily_sales_avg, 0) > 0 THEN + GREATEST( + CEIL(sm.daily_sales_avg * COALESCE(lm.avg_lead_time_days, 30) * 1.96), + ? + ) + ELSE ? + END, + pm.overstocked_amt = CASE + WHEN p.stock_quantity / NULLIF(sm.daily_sales_avg, 0) > ? + THEN GREATEST(0, p.stock_quantity - CEIL(sm.daily_sales_avg * ?)) + ELSE 0 + END, + pm.last_calculated_at = NOW() + WHERE p.pid IN (?) + `, [ + defaultThresholds.low_stock_threshold, + defaultThresholds.critical_days, + defaultThresholds.reorder_days, + defaultThresholds.overstock_days, + defaultThresholds.low_stock_threshold, + defaultThresholds.low_stock_threshold, + defaultThresholds.overstock_days, + defaultThresholds.overstock_days, + batch.map(row => row.pid) + ]); + + lastPid = batch[batch.length - 1].pid; + processedCount += batch.length; + + outputProgress({ + status: 'running', + operation: 'Processing base metrics batch', + current: processedCount, + total: totalProducts, + elapsed: formatElapsedTime(startTime), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: ((processedCount / totalProducts) * 100).toFixed(1), + timing: { + start_time: new Date(startTime).toISOString(), + end_time: new Date().toISOString(), + elapsed_seconds: Math.round((Date.now() - startTime) / 1000) + } + }); + } + + // Calculate forecast accuracy and bias in batches + lastPid = 0; + while (true) { + if (isCancelled) break; + + const [batch] = await connection.query( + 'SELECT pid FROM products WHERE pid > ? ORDER BY pid LIMIT ?', + [lastPid, BATCH_SIZE] + ); + + if (batch.length === 0) break; + + await connection.query(` + UPDATE product_metrics pm + JOIN ( + SELECT + sf.pid, + AVG(CASE + WHEN o.quantity > 0 + THEN ABS(sf.forecast_units - o.quantity) / o.quantity * 100 + ELSE 100 + END) as avg_forecast_error, + AVG(CASE + WHEN o.quantity > 0 + THEN (sf.forecast_units - o.quantity) / o.quantity * 100 + ELSE 0 + END) as avg_forecast_bias, + MAX(sf.forecast_date) as last_forecast_date + FROM sales_forecasts sf + JOIN orders o ON sf.pid = o.pid + AND DATE(o.date) = sf.forecast_date + WHERE o.canceled = false + AND sf.forecast_date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY) + AND sf.pid IN (?) + GROUP BY sf.pid + ) fa ON pm.pid = fa.pid + SET + pm.forecast_accuracy = GREATEST(0, 100 - LEAST(fa.avg_forecast_error, 100)), + pm.forecast_bias = GREATEST(-100, LEAST(fa.avg_forecast_bias, 100)), + pm.last_forecast_date = fa.last_forecast_date, + pm.last_calculated_at = NOW() + WHERE pm.pid IN (?) + `, [batch.map(row => row.pid), batch.map(row => row.pid)]); + + lastPid = batch[batch.length - 1].pid; + } } - if (isCancelled) return { - processedProducts: processedCount || 0, - processedOrders: processedOrders || 0, - processedPurchaseOrders: 0, // This module doesn't process POs - success - }; - // Calculate product time aggregates if (!SKIP_PRODUCT_TIME_AGGREGATES) { outputProgress({