From 90379386d688cc2805a58dc903bdf42736716cae Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 10 Feb 2025 15:20:22 -0500 Subject: [PATCH] Apply gemini suggested calculate improvements --- inventory-server/scripts/calculate-metrics.js | 220 ++++++++---------- .../scripts/metrics/brand-metrics.js | 38 +-- .../scripts/metrics/category-metrics.js | 38 +-- .../scripts/metrics/financial-metrics.js | 15 +- .../scripts/metrics/product-metrics.js | 58 ++--- .../scripts/metrics/sales-forecasts.js | 17 +- .../scripts/metrics/time-aggregates.js | 35 ++- .../scripts/metrics/vendor-metrics.js | 40 ++-- 8 files changed, 211 insertions(+), 250 deletions(-) diff --git a/inventory-server/scripts/calculate-metrics.js b/inventory-server/scripts/calculate-metrics.js index 2c66a5d..2edf071 100644 --- a/inventory-server/scripts/calculate-metrics.js +++ b/inventory-server/scripts/calculate-metrics.js @@ -83,6 +83,7 @@ process.on('SIGTERM', cancelCalculation); async function calculateMetrics() { let connection; const startTime = Date.now(); + // Initialize all counts to 0 let processedProducts = 0; let processedOrders = 0; let processedPurchaseOrders = 0; @@ -90,7 +91,7 @@ async function calculateMetrics() { let totalOrders = 0; let totalPurchaseOrders = 0; let calculateHistoryId; - + try { // Clean up any previously running calculations connection = await getConnection(); @@ -140,6 +141,7 @@ async function calculateMetrics() { totalProducts = productCount.total; totalOrders = orderCount.total; totalPurchaseOrders = poCount.total; + connection.release(); // If nothing needs updating, we can exit early if (totalProducts === 0 && totalOrders === 0 && totalPurchaseOrders === 0) { @@ -153,6 +155,7 @@ async function calculateMetrics() { } // Create history record for this calculation + connection = await getConnection(); // Re-establish connection const [historyResult] = await connection.query(` INSERT INTO calculate_history ( start_time, @@ -183,7 +186,7 @@ async function calculateMetrics() { totalPurchaseOrders, SKIP_PRODUCT_METRICS, SKIP_TIME_AGGREGATES, - SKIP_FINANCIAL_METRICS, + SKIP_FINANCIAL_METRICS, SKIP_VENDOR_METRICS, SKIP_CATEGORY_METRICS, SKIP_BRAND_METRICS, @@ -209,7 +212,7 @@ async function calculateMetrics() { } isCancelled = false; - connection = await getConnection(); + connection = await getConnection(); // Get a new connection for the main processing try { global.outputProgress({ @@ -228,14 +231,12 @@ async function calculateMetrics() { } }); - // Update progress periodically + // Update progress periodically - REFACTORED const updateProgress = async (products = null, orders = null, purchaseOrders = null) => { - // Ensure all values are valid numbers or default to previous value if (products !== null) processedProducts = Number(products) || processedProducts || 0; if (orders !== null) processedOrders = Number(orders) || processedOrders || 0; if (purchaseOrders !== null) processedPurchaseOrders = Number(purchaseOrders) || processedPurchaseOrders || 0; - // Ensure we never send NaN to the database const safeProducts = Number(processedProducts) || 0; const safeOrders = Number(processedOrders) || 0; const safePurchaseOrders = Number(processedPurchaseOrders) || 0; @@ -250,14 +251,14 @@ async function calculateMetrics() { `, [safeProducts, safeOrders, safePurchaseOrders, calculateHistoryId]); }; - // Helper function to ensure valid progress numbers + // Helper function to ensure valid progress numbers - this is fine const ensureValidProgress = (current, total) => ({ current: Number(current) || 0, total: Number(total) || 1, // Default to 1 to avoid division by zero percentage: (((Number(current) || 0) / (Number(total) || 1)) * 100).toFixed(1) }); - // Initial progress + // Initial progress - this is fine const initialProgress = ensureValidProgress(0, totalProducts); global.outputProgress({ status: 'running', @@ -275,37 +276,28 @@ async function calculateMetrics() { } }); + // --- Call each module, passing totals and accumulating processed counts --- + if (!SKIP_PRODUCT_METRICS) { - const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); // Update with accumulated values if (!result.success) { throw new Error('Product metrics calculation failed'); } } else { console.log('Skipping product metrics calculation...'); - processedProducts = Math.floor(totalProducts * 0.6); - await updateProgress(processedProducts); - global.outputProgress({ - status: 'running', - operation: 'Skipping product metrics calculation', - current: processedProducts, - total: totalProducts, - elapsed: global.formatElapsedTime(startTime), - remaining: global.estimateRemaining(startTime, processedProducts, totalProducts), - rate: global.calculateRate(startTime, processedProducts), - percentage: '60', - timing: { - start_time: new Date(startTime).toISOString(), - end_time: new Date().toISOString(), - elapsed_seconds: Math.round((Date.now() - startTime) / 1000) - } - }); + // Don't artificially inflate processedProducts if skipping } - // Calculate time-based aggregates if (!SKIP_TIME_AGGREGATES) { - const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Time aggregates calculation failed'); } @@ -313,21 +305,25 @@ async function calculateMetrics() { console.log('Skipping time aggregates calculation'); } - // Calculate financial metrics if (!SKIP_FINANCIAL_METRICS) { - const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Financial metrics calculation failed'); } } else { console.log('Skipping financial metrics calculation'); } - - // Calculate vendor metrics + if (!SKIP_VENDOR_METRICS) { - const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Vendor metrics calculation failed'); } @@ -335,10 +331,12 @@ async function calculateMetrics() { console.log('Skipping vendor metrics calculation'); } - // Calculate category metrics if (!SKIP_CATEGORY_METRICS) { - const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Category metrics calculation failed'); } @@ -346,10 +344,12 @@ async function calculateMetrics() { console.log('Skipping category metrics calculation'); } - // Calculate brand metrics if (!SKIP_BRAND_METRICS) { - const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Brand metrics calculation failed'); } @@ -357,10 +357,12 @@ async function calculateMetrics() { console.log('Skipping brand metrics calculation'); } - // Calculate sales forecasts if (!SKIP_SALES_FORECASTS) { - const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts); - await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); + const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts, isCancelled); // Pass totals + processedProducts += result.processedProducts; // Accumulate + processedOrders += result.processedOrders; + processedPurchaseOrders += result.processedPurchaseOrders; + await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); if (!result.success) { throw new Error('Sales forecasts calculation failed'); } @@ -368,23 +370,7 @@ async function calculateMetrics() { console.log('Skipping sales forecasts calculation'); } - // Calculate ABC classification - outputProgress({ - status: 'running', - operation: 'Starting ABC classification', - current: processedProducts || 0, - total: totalProducts || 0, - elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0), - rate: calculateRate(startTime, processedProducts || 0), - percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1), - timing: { - start_time: new Date(startTime).toISOString(), - end_time: new Date().toISOString(), - elapsed_seconds: Math.round((Date.now() - startTime) / 1000) - } - }); - + // --- ABC Classification (Refactored) --- if (isCancelled) return { processedProducts: processedProducts || 0, processedOrders: processedOrders || 0, @@ -402,21 +388,26 @@ async function calculateMetrics() { pid BIGINT NOT NULL, total_revenue DECIMAL(10,3), rank_num INT, + dense_rank_num INT, + percentile DECIMAL(5,2), total_count INT, PRIMARY KEY (pid), - INDEX (rank_num) + INDEX (rank_num), + INDEX (dense_rank_num), + INDEX (percentile) ) ENGINE=MEMORY `); + let processedCount = processedProducts; outputProgress({ status: 'running', operation: 'Creating revenue rankings', - current: processedProducts || 0, - total: totalProducts || 0, + current: processedCount, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0), - rate: calculateRate(startTime, processedProducts || 0), - percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: ((processedCount / totalProducts) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -431,49 +422,35 @@ async function calculateMetrics() { success: false }; + // Calculate rankings with proper tie handling and get total count in one go. await connection.query(` INSERT INTO temp_revenue_ranks + WITH revenue_data AS ( + SELECT + pid, + total_revenue, + COUNT(*) OVER () as total_count, + PERCENT_RANK() OVER (ORDER BY total_revenue DESC) * 100 as percentile, + RANK() OVER (ORDER BY total_revenue DESC) as rank_num, + DENSE_RANK() OVER (ORDER BY total_revenue DESC) as dense_rank_num + FROM product_metrics + WHERE total_revenue > 0 + ) SELECT pid, total_revenue, - @rank := @rank + 1 as rank_num, - @total_count := @rank as total_count - FROM ( - SELECT pid, total_revenue - FROM product_metrics - WHERE total_revenue > 0 - ORDER BY total_revenue DESC - ) ranked, - (SELECT @rank := 0) r + rank_num, + dense_rank_num, + percentile, + total_count + FROM revenue_data `); - // Get total count for percentage calculation - const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks'); - const totalCount = rankingCount[0].total_count || 1; - const max_rank = totalCount; // Store max_rank for use in classification - - outputProgress({ - status: 'running', - operation: 'Updating ABC classifications', - current: processedProducts || 0, - total: totalProducts || 0, - elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0), - rate: calculateRate(startTime, processedProducts || 0), - percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1), - timing: { - start_time: new Date(startTime).toISOString(), - end_time: new Date().toISOString(), - elapsed_seconds: Math.round((Date.now() - startTime) / 1000) - } - }); - - if (isCancelled) return { - processedProducts: processedProducts || 0, - processedOrders: processedOrders || 0, - processedPurchaseOrders: 0, - success: false - }; + // Get total count for percentage calculation (already done in the above query) + // No need for this separate query: + // const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks'); + // const totalCount = rankingCount[0].total_count || 1; + // const max_rank = totalCount; // Store max_rank for use in classification // ABC classification progress tracking let abcProcessedCount = 0; @@ -489,7 +466,7 @@ async function calculateMetrics() { success: false }; - // First get a batch of PIDs that need updating + // Get a batch of PIDs that need updating - REFACTORED to use percentile const [pids] = await connection.query(` SELECT pm.pid FROM product_metrics pm @@ -497,41 +474,38 @@ async function calculateMetrics() { WHERE pm.abc_class IS NULL OR pm.abc_class != CASE - WHEN tr.rank_num IS NULL THEN 'C' - WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A' - WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B' + WHEN tr.pid IS NULL THEN 'C' + WHEN tr.percentile <= ? THEN 'A' + WHEN tr.percentile <= ? THEN 'B' ELSE 'C' END LIMIT ? - `, [max_rank, abcThresholds.a_threshold, - max_rank, abcThresholds.b_threshold, - batchSize]); + `, [abcThresholds.a_threshold, abcThresholds.b_threshold, batchSize]); if (pids.length === 0) { break; } - // Then update just those PIDs - const [result] = await connection.query(` + // Update just those PIDs - REFACTORED to use percentile + await connection.query(` UPDATE product_metrics pm LEFT JOIN temp_revenue_ranks tr ON pm.pid = tr.pid SET pm.abc_class = CASE - WHEN tr.rank_num IS NULL THEN 'C' - WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A' - WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B' + WHEN tr.pid IS NULL THEN 'C' + WHEN tr.percentile <= ? THEN 'A' + WHEN tr.percentile <= ? THEN 'B' ELSE 'C' END, pm.last_calculated_at = NOW() WHERE pm.pid IN (?) - `, [max_rank, abcThresholds.a_threshold, - max_rank, abcThresholds.b_threshold, - pids.map(row => row.pid)]); + `, [abcThresholds.a_threshold, abcThresholds.b_threshold, pids.map(row => row.pid)]); + + abcProcessedCount += pids.length; // Use pids.length, more accurate + processedProducts += pids.length; // Add to the main processedProducts - abcProcessedCount += result.affectedRows; - // Calculate progress ensuring valid numbers - const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalCount || 1)) * 0.01)); + const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalProducts || 1)) * 0.01)); processedProducts = Number(currentProgress) || processedProducts || 0; // Only update progress at most once per second diff --git a/inventory-server/scripts/metrics/brand-metrics.js b/inventory-server/scripts/metrics/brand-metrics.js index d80b0cd..450f49c 100644 --- a/inventory-server/scripts/metrics/brand-metrics.js +++ b/inventory-server/scripts/metrics/brand-metrics.js @@ -5,7 +5,8 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = const connection = await getConnection(); let success = false; const BATCH_SIZE = 5000; - + let myProcessedProducts = 0; // Not *directly* processing products, tracking brands + try { // Get last calculation timestamp const [lastCalc] = await connection.query(` @@ -26,12 +27,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = OR o.id IS NOT NULL ) `, [lastCalculationTime, lastCalculationTime]); - const totalBrands = brandCount[0].count; + const totalBrands = brandCount[0].count; // Track total *brands* if (totalBrands === 0) { console.log('No brands need metric updates'); return { - processedProducts: 0, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success: true @@ -42,12 +43,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = outputProgress({ status: 'cancelled', operation: 'Brand metrics calculation cancelled', - current: processedCount, - total: totalBrands, + current: processedCount, // Use passed-in value + total: totalBrands, // Report total *brands* elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalBrands) * 100).toFixed(1), + percentage: ((processedCount / totalBrands) * 100).toFixed(1), // Base on brands timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -55,7 +56,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = } }); return { - processedProducts: processedCount, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success @@ -65,12 +66,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = outputProgress({ status: 'running', operation: 'Starting brand metrics calculation', - current: processedCount, - total: totalBrands, + current: processedCount, // Use passed-in value + total: totalBrands, // Report total *brands* elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalBrands), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalBrands) * 100).toFixed(1), + percentage: ((processedCount / totalBrands) * 100).toFixed(1), // Base on brands timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -80,6 +81,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = // Process in batches let lastBrand = ''; + let processedBrands = 0; // Track processed brands while (true) { if (isCancelled) break; @@ -243,17 +245,17 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats'); lastBrand = batch[batch.length - 1].brand; - processedCount += batch.length; + processedBrands += batch.length; // Increment processed *brands* outputProgress({ status: 'running', operation: 'Processing brand metrics batch', - current: processedCount, - total: totalBrands, + current: processedCount + processedBrands, // Use cumulative brand count + total: totalBrands, // Report total *brands* elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalBrands), - rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalBrands) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount + processedBrands, totalBrands), + rate: calculateRate(startTime, processedCount + processedBrands), + percentage: (((processedCount + processedBrands) / totalBrands) * 100).toFixed(1), // Base on brands timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -264,7 +266,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = // If we get here, everything completed successfully success = true; - + // Update calculate_status await connection.query(` INSERT INTO calculate_status (module_name, last_calculation_timestamp) @@ -273,7 +275,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount = `); return { - processedProducts: processedCount, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success diff --git a/inventory-server/scripts/metrics/category-metrics.js b/inventory-server/scripts/metrics/category-metrics.js index 4b2f926..dbc2375 100644 --- a/inventory-server/scripts/metrics/category-metrics.js +++ b/inventory-server/scripts/metrics/category-metrics.js @@ -5,7 +5,8 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount const connection = await getConnection(); let success = false; const BATCH_SIZE = 5000; - + let myProcessedProducts = 0; // Not *directly* processing products, but tracking categories + try { // Get last calculation timestamp const [lastCalc] = await connection.query(` @@ -28,12 +29,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount OR o.id IS NOT NULL ) `, [lastCalculationTime, lastCalculationTime]); - const totalCategories = categoryCount[0].count; + const totalCategories = categoryCount[0].count; // Track total *categories* if (totalCategories === 0) { console.log('No categories need metric updates'); return { - processedProducts: 0, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success: true @@ -44,12 +45,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount outputProgress({ status: 'cancelled', operation: 'Category metrics calculation cancelled', - current: processedCount, - total: totalCategories, + current: processedCount, // Use passed-in value + total: totalCategories, // Report total *categories* elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalCategories) * 100).toFixed(1), + percentage: ((processedCount / totalCategories) * 100).toFixed(1), // Base on categories timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -57,7 +58,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); return { - processedProducts: processedCount, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success @@ -67,12 +68,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount outputProgress({ status: 'running', operation: 'Starting category metrics calculation', - current: processedCount, - total: totalCategories, + current: processedCount, // Use passed-in value + total: totalCategories, // Report total *categories* elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalCategories), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalCategories) * 100).toFixed(1), + percentage: ((processedCount / totalCategories) * 100).toFixed(1), // Base on categories timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -82,6 +83,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount // Process in batches let lastCatId = 0; + let processedCategories = 0; // Track processed categories while (true) { if (isCancelled) break; @@ -247,17 +249,17 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats'); lastCatId = batch[batch.length - 1].cat_id; - processedCount += batch.length; + processedCategories += batch.length; // Increment processed *categories* outputProgress({ status: 'running', operation: 'Processing category metrics batch', - current: processedCount, - total: totalCategories, + current: processedCount + processedCategories, // Use cumulative category count + total: totalCategories, // Report total *categories* elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalCategories), - rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalCategories) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount + processedCategories, totalCategories), + rate: calculateRate(startTime, processedCount + processedCategories), + percentage: (((processedCount + processedCategories) / totalCategories) * 100).toFixed(1), // Base on categories timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -268,7 +270,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount // If we get here, everything completed successfully success = true; - + // Update calculate_status await connection.query(` INSERT INTO calculate_status (module_name, last_calculation_timestamp) @@ -277,7 +279,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount `); return { - processedProducts: processedCount, + processedProducts: 0, // Not directly processing products processedOrders: 0, processedPurchaseOrders: 0, success diff --git a/inventory-server/scripts/metrics/financial-metrics.js b/inventory-server/scripts/metrics/financial-metrics.js index a964aa6..8338c8d 100644 --- a/inventory-server/scripts/metrics/financial-metrics.js +++ b/inventory-server/scripts/metrics/financial-metrics.js @@ -5,6 +5,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun const connection = await getConnection(); let success = false; const BATCH_SIZE = 5000; + let myProcessedProducts = 0; // Track products processed *within this module* try { // Get last calculation timestamp @@ -54,7 +55,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun } }); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, processedOrders: 0, processedPurchaseOrders: 0, success @@ -133,17 +134,17 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun `, [batch.map(row => row.pid)]); lastPid = batch[batch.length - 1].pid; - processedCount += batch.length; + myProcessedProducts += batch.length; outputProgress({ status: 'running', operation: 'Processing financial metrics batch', - current: processedCount, + current: processedCount + myProcessedProducts, total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalProducts), - rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount + myProcessedProducts, totalProducts), + rate: calculateRate(startTime, processedCount + myProcessedProducts), + percentage: (((processedCount + myProcessedProducts) / totalProducts) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -163,7 +164,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun `); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, processedOrders: 0, processedPurchaseOrders: 0, success diff --git a/inventory-server/scripts/metrics/product-metrics.js b/inventory-server/scripts/metrics/product-metrics.js index 782b3cd..f38ca1e 100644 --- a/inventory-server/scripts/metrics/product-metrics.js +++ b/inventory-server/scripts/metrics/product-metrics.js @@ -13,6 +13,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount const connection = await getConnection(); let success = false; let processedOrders = 0; + let myProcessedProducts = 0; // Track products processed *within this module* const BATCH_SIZE = 5000; try { @@ -24,24 +25,10 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount `); const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01'; - // Get total product count if not provided - if (!totalProducts) { - const [productCount] = await connection.query(` - SELECT COUNT(DISTINCT p.pid) as count - FROM products p - LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ? - LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ? - WHERE p.updated > ? - OR o.pid IS NOT NULL - OR po.pid IS NOT NULL - `, [lastCalculationTime, lastCalculationTime, lastCalculationTime]); - totalProducts = productCount[0].count; - } - if (totalProducts === 0) { console.log('No products need updating'); return { - processedProducts: 0, + processedProducts: myProcessedProducts, processedOrders: 0, processedPurchaseOrders: 0, success: true @@ -69,7 +56,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount } }); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, processedOrders, processedPurchaseOrders: 0, success @@ -290,6 +277,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount lastPid = batch[batch.length - 1].pid; processedCount += batch.length; + myProcessedProducts += batch.length; // Increment the *module's* count outputProgress({ status: 'running', @@ -361,12 +349,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount outputProgress({ status: 'running', operation: 'Starting product time aggregates calculation', - current: processedCount || 0, - total: totalProducts || 0, + current: processedCount, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0), - rate: calculateRate(startTime, processedCount || 0), - percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -428,12 +416,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount outputProgress({ status: 'running', operation: 'Product time aggregates calculated', - current: processedCount || 0, - total: totalProducts || 0, + current: processedCount, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0), - rate: calculateRate(startTime, processedCount || 0), - percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -445,12 +433,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount outputProgress({ status: 'running', operation: 'Skipping product time aggregates calculation', - current: processedCount || 0, - total: totalProducts || 0, + current: processedCount, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0), - rate: calculateRate(startTime, processedCount || 0), - percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount, totalProducts), + rate: calculateRate(startTime, processedCount), + percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -479,7 +467,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount if (isCancelled) return { processedProducts: processedCount, processedOrders, - processedPurchaseOrders: 0, // This module doesn't process POs + processedPurchaseOrders: 0, success }; @@ -540,7 +528,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount if (isCancelled) return { processedProducts: processedCount, processedOrders, - processedPurchaseOrders: 0, // This module doesn't process POs + processedPurchaseOrders: 0, success }; @@ -620,9 +608,9 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount `); return { - processedProducts: processedCount || 0, + processedProducts: processedCount, processedOrders: processedOrders || 0, - processedPurchaseOrders: 0, // This module doesn't process POs + processedPurchaseOrders: 0, success }; diff --git a/inventory-server/scripts/metrics/sales-forecasts.js b/inventory-server/scripts/metrics/sales-forecasts.js index 248475c..16c78e8 100644 --- a/inventory-server/scripts/metrics/sales-forecasts.js +++ b/inventory-server/scripts/metrics/sales-forecasts.js @@ -4,6 +4,7 @@ const { getConnection } = require('./utils/db'); async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); let success = false; + let myProcessedProducts = 0; // Track products processed *within this module* const BATCH_SIZE = 5000; try { @@ -43,7 +44,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount status: 'cancelled', operation: 'Sales forecast calculation cancelled', current: processedCount, - total: totalProductsToUpdate, + total: totalProducts, elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), @@ -55,7 +56,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount } }); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, processedOrders: 0, processedPurchaseOrders: 0, success @@ -66,9 +67,9 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount status: 'running', operation: 'Starting sales forecast calculation', current: processedCount, - total: totalProductsToUpdate, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate), + remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1), timing: { @@ -255,15 +256,15 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_confidence_calc'); lastPid = batch[batch.length - 1].pid; - processedCount += batch.length; + myProcessedProducts += batch.length; outputProgress({ status: 'running', operation: 'Processing sales forecast batch', current: processedCount, - total: totalProductsToUpdate, + total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate), + remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount), percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1), timing: { @@ -285,7 +286,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount `); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, processedOrders: 0, processedPurchaseOrders: 0, success diff --git a/inventory-server/scripts/metrics/time-aggregates.js b/inventory-server/scripts/metrics/time-aggregates.js index c81333d..01633ae 100644 --- a/inventory-server/scripts/metrics/time-aggregates.js +++ b/inventory-server/scripts/metrics/time-aggregates.js @@ -5,7 +5,8 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount const connection = await getConnection(); let success = false; const BATCH_SIZE = 5000; - + let myProcessedProducts = 0; // Track products processed *within this module* + try { // Get last calculation timestamp const [lastCalc] = await connection.query(` @@ -15,17 +16,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount `); const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01'; - // Get total count of products needing updates - if (!totalProducts) { - const [productCount] = await connection.query(` - SELECT COUNT(DISTINCT p.pid) as count - FROM products p - LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ? - WHERE p.updated > ? - OR o.pid IS NOT NULL - `, [lastCalculationTime, lastCalculationTime]); - totalProducts = productCount[0].count; - } + // We now receive totalProducts as an argument, so we don't need to query for it here. if (totalProducts === 0) { console.log('No products need time aggregate updates'); @@ -41,7 +32,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount outputProgress({ status: 'cancelled', operation: 'Time aggregates calculation cancelled', - current: processedCount, + current: processedCount, // Use passed-in value total: totalProducts, elapsed: formatElapsedTime(startTime), remaining: null, @@ -54,7 +45,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount } }); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, // Return only what *this* module processed processedOrders: 0, processedPurchaseOrders: 0, success @@ -64,7 +55,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount outputProgress({ status: 'running', operation: 'Starting time aggregates calculation', - current: processedCount, + current: processedCount, // Use passed-in value total: totalProducts, elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts), @@ -253,17 +244,17 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_time_aggregates'); lastPid = batch[batch.length - 1].pid; - processedCount += batch.length; + myProcessedProducts += batch.length; // Increment *this module's* count outputProgress({ status: 'running', operation: 'Processing time aggregates batch', - current: processedCount, + current: processedCount + myProcessedProducts, // Show cumulative progress total: totalProducts, elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalProducts), - rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalProducts) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount + myProcessedProducts, totalProducts), + rate: calculateRate(startTime, processedCount + myProcessedProducts), + percentage: (((processedCount + myProcessedProducts) / totalProducts) * 100).toFixed(1), timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -274,7 +265,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount // If we get here, everything completed successfully success = true; - + // Update calculate_status await connection.query(` INSERT INTO calculate_status (module_name, last_calculation_timestamp) @@ -283,7 +274,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount `); return { - processedProducts: processedCount, + processedProducts: myProcessedProducts, // Return only what *this* module processed processedOrders: 0, processedPurchaseOrders: 0, success diff --git a/inventory-server/scripts/metrics/vendor-metrics.js b/inventory-server/scripts/metrics/vendor-metrics.js index 442ff41..d0af1d7 100644 --- a/inventory-server/scripts/metrics/vendor-metrics.js +++ b/inventory-server/scripts/metrics/vendor-metrics.js @@ -5,7 +5,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = const connection = await getConnection(); let success = false; const BATCH_SIZE = 5000; - + let myProcessedProducts = 0; // Not directly processing products, but we'll track vendors + try { // Get last calculation timestamp const [lastCalc] = await connection.query(` @@ -33,12 +34,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = ) ) `, [lastCalculationTime, lastCalculationTime]); - const totalVendors = vendorCount[0].count; + const totalVendors = vendorCount[0].count; // Track total *vendors* if (totalVendors === 0) { console.log('No vendors need metric updates'); return { - processedProducts: 0, + processedProducts: 0, // No products directly processed processedOrders: 0, processedPurchaseOrders: 0, success: true @@ -49,12 +50,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = outputProgress({ status: 'cancelled', operation: 'Vendor metrics calculation cancelled', - current: processedCount, - total: totalVendors, + current: processedCount, // Use passed-in value (for consistency) + total: totalVendors, // Report total *vendors* elapsed: formatElapsedTime(startTime), remaining: null, rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalVendors) * 100).toFixed(1), + percentage: ((processedCount / totalVendors) * 100).toFixed(1), // Base on vendors timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -62,7 +63,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = } }); return { - processedProducts: processedCount, + processedProducts: 0, // No products directly processed processedOrders: 0, processedPurchaseOrders: 0, success @@ -72,12 +73,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = outputProgress({ status: 'running', operation: 'Starting vendor metrics calculation', - current: processedCount, - total: totalVendors, + current: processedCount, // Use passed-in value + total: totalVendors, // Report total *vendors* elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalVendors), rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalVendors) * 100).toFixed(1), + percentage: ((processedCount / totalVendors) * 100).toFixed(1), // Base on vendors timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -87,6 +88,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = // Process in batches let lastVendor = ''; + let processedVendors = 0; // Track processed vendors while (true) { if (isCancelled) break; @@ -119,7 +121,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = // Create temporary tables with optimized structure and indexes await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_purchase_stats'); await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats'); - + await connection.query(` CREATE TEMPORARY TABLE temp_purchase_stats ( vendor VARCHAR(100) NOT NULL, @@ -253,17 +255,17 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats'); lastVendor = batch[batch.length - 1].vendor; - processedCount += batch.length; + processedVendors += batch.length; // Increment processed *vendors* outputProgress({ status: 'running', operation: 'Processing vendor metrics batch', - current: processedCount, - total: totalVendors, + current: processedCount + processedVendors, // Use cumulative vendor count + total: totalVendors, // Report total *vendors* elapsed: formatElapsedTime(startTime), - remaining: estimateRemaining(startTime, processedCount, totalVendors), - rate: calculateRate(startTime, processedCount), - percentage: ((processedCount / totalVendors) * 100).toFixed(1), + remaining: estimateRemaining(startTime, processedCount + processedVendors, totalVendors), + rate: calculateRate(startTime, processedCount + processedVendors), + percentage: (((processedCount + processedVendors) / totalVendors) * 100).toFixed(1), // Base on vendors timing: { start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(), @@ -274,7 +276,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = // If we get here, everything completed successfully success = true; - + // Update calculate_status await connection.query(` INSERT INTO calculate_status (module_name, last_calculation_timestamp) @@ -283,7 +285,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount = `); return { - processedProducts: processedCount, + processedProducts: 0, // No products directly processed processedOrders: 0, processedPurchaseOrders: 0, success