From 5676e9094dc5283eb336a3a1a8c622ca43195aee Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 2 Feb 2025 21:22:46 -0500 Subject: [PATCH] Add calculate time tracking --- inventory-server/db/config-schema.sql | 23 ++++-- inventory-server/scripts/calculate-metrics.js | 6 +- .../scripts/metrics/brand-metrics.js | 46 +++++++++++- .../scripts/metrics/category-metrics.js | 67 +++++++++++++++-- .../scripts/metrics/financial-metrics.js | 47 +++++++++++- .../scripts/metrics/product-metrics.js | 58 ++++++++++++-- .../scripts/metrics/sales-forecasts.js | 75 +++++++++++++++++-- .../scripts/metrics/time-aggregates.js | 46 +++++++++++- .../scripts/metrics/vendor-metrics.js | 62 +++++++++++++-- 9 files changed, 382 insertions(+), 48 deletions(-) diff --git a/inventory-server/db/config-schema.sql b/inventory-server/db/config-schema.sql index f6621b4..9aa8330 100644 --- a/inventory-server/db/config-schema.sql +++ b/inventory-server/db/config-schema.sql @@ -171,14 +171,6 @@ ORDER BY c.name, st.vendor; --- Update calculate_history table to track all record types -ALTER TABLE calculate_history - ADD COLUMN total_orders INT DEFAULT 0 AFTER total_products, - ADD COLUMN total_purchase_orders INT DEFAULT 0 AFTER total_orders, - CHANGE COLUMN products_processed processed_products INT DEFAULT 0, - ADD COLUMN processed_orders INT DEFAULT 0 AFTER processed_products, - ADD COLUMN processed_purchase_orders INT DEFAULT 0 AFTER processed_orders; - CREATE TABLE IF NOT EXISTS calculate_history ( id BIGINT AUTO_INCREMENT PRIMARY KEY, start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -197,6 +189,21 @@ CREATE TABLE IF NOT EXISTS calculate_history ( INDEX idx_status_time (status, start_time) ); +CREATE TABLE IF NOT EXISTS calculate_status ( + module_name ENUM( + 'product_metrics', + 'time_aggregates', + 'financial_metrics', + 'vendor_metrics', + 'category_metrics', + 'brand_metrics', + 'sales_forecasts', + 'abc_classification' + ) PRIMARY KEY, + last_calculation_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + INDEX idx_last_calc (last_calculation_timestamp) +); + CREATE TABLE IF NOT EXISTS sync_status ( table_name VARCHAR(50) PRIMARY KEY, last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/inventory-server/scripts/calculate-metrics.js b/inventory-server/scripts/calculate-metrics.js index 854b7ea..c33d8b8 100644 --- a/inventory-server/scripts/calculate-metrics.js +++ b/inventory-server/scripts/calculate-metrics.js @@ -497,9 +497,9 @@ async function calculateMetrics() { WHERE id = ? `, [ totalElapsedSeconds, - processedProducts, - processedOrders, - processedPurchaseOrders, + processedProducts || 0, // Ensure we have a valid number + processedOrders || 0, // Ensure we have a valid number + processedPurchaseOrders || 0, // Ensure we have a valid number isCancelled ? 'cancelled' : 'failed', error.message, calculateHistoryId diff --git a/inventory-server/scripts/metrics/brand-metrics.js b/inventory-server/scripts/metrics/brand-metrics.js index 6013a57..0d54365 100644 --- a/inventory-server/scripts/metrics/brand-metrics.js +++ b/inventory-server/scripts/metrics/brand-metrics.js @@ -1,8 +1,11 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateBrandMetrics(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateBrandMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +23,22 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders: 0, + processedPurchaseOrders: 0, + success + }; } + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + `); + processedOrders = orderCount[0].count; + outputProgress({ status: 'running', operation: 'Starting brand metrics calculation', @@ -178,7 +194,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Calculate brand time-based metrics with optimized query await connection.query(` @@ -266,8 +287,25 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount, i } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('brand_metrics', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating brand metrics'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/category-metrics.js b/inventory-server/scripts/metrics/category-metrics.js index 21bad8a..8ceaca5 100644 --- a/inventory-server/scripts/metrics/category-metrics.js +++ b/inventory-server/scripts/metrics/category-metrics.js @@ -1,8 +1,11 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateCategoryMetrics(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateCategoryMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +23,22 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders: 0, + processedPurchaseOrders: 0, + success + }; } + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + `); + processedOrders = orderCount[0].count; + outputProgress({ status: 'running', operation: 'Starting category metrics calculation', @@ -85,7 +101,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Then update with margin and turnover data await connection.query(` @@ -144,7 +165,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Finally update growth rates await connection.query(` @@ -287,7 +313,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Calculate time-based metrics await connection.query(` @@ -361,7 +392,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Calculate category-sales metrics await connection.query(` @@ -455,8 +491,25 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('category_metrics', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating category metrics'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/financial-metrics.js b/inventory-server/scripts/metrics/financial-metrics.js index 79faac2..e00c37c 100644 --- a/inventory-server/scripts/metrics/financial-metrics.js +++ b/inventory-server/scripts/metrics/financial-metrics.js @@ -1,8 +1,11 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateFinancialMetrics(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateFinancialMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +23,23 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders: 0, + processedPurchaseOrders: 0, + success + }; } + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH) + `); + processedOrders = orderCount[0].count; + outputProgress({ status: 'running', operation: 'Starting financial metrics calculation', @@ -90,7 +107,12 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Update time-based aggregates with optimized query await connection.query(` @@ -139,8 +161,25 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('financial_metrics', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating financial metrics'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/product-metrics.js b/inventory-server/scripts/metrics/product-metrics.js index 0ea253f..777c546 100644 --- a/inventory-server/scripts/metrics/product-metrics.js +++ b/inventory-server/scripts/metrics/product-metrics.js @@ -11,6 +11,9 @@ function sanitizeValue(value) { async function calculateProductMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { // Skip flags are inherited from the parent scope const SKIP_PRODUCT_BASE_METRICS = 0; @@ -32,7 +35,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, // This module doesn't process POs + success + }; } // First ensure all products have a metrics record @@ -60,6 +68,14 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount } }); + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + `); + processedOrders = orderCount[0].count; + // Calculate base metrics await connection.query(` UPDATE product_metrics pm @@ -181,7 +197,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount }); } - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, // This module doesn't process POs + success + }; // Calculate product time aggregates if (!SKIP_PRODUCT_TIME_AGGREGATES) { @@ -303,7 +324,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, // This module doesn't process POs + success + }; const [abcConfig] = await connection.query('SELECT a_threshold, b_threshold FROM abc_classification_config WHERE id = 1'); const abcThresholds = abcConfig[0] || { a_threshold: 20, b_threshold: 50 }; @@ -359,7 +385,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount const batchSize = 5000; while (true) { - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, // This module doesn't process POs + success + }; // Get a batch of PIDs that need updating const [pids] = await connection.query(` @@ -426,8 +457,25 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount `, [pids.map(row => row.pid), pids.map(row => row.pid)]); } - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('product_metrics', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, // This module doesn't process POs + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating product metrics'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/sales-forecasts.js b/inventory-server/scripts/metrics/sales-forecasts.js index fa99c81..94dda99 100644 --- a/inventory-server/scripts/metrics/sales-forecasts.js +++ b/inventory-server/scripts/metrics/sales-forecasts.js @@ -1,8 +1,11 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateSalesForecasts(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +23,23 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders: 0, + processedPurchaseOrders: 0, + success + }; } + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY) + `); + processedOrders = orderCount[0].count; + outputProgress({ status: 'running', operation: 'Starting sales forecasts calculation', @@ -83,7 +100,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Create temporary table for daily sales stats await connection.query(` @@ -117,7 +139,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Create temporary table for product stats await connection.query(` @@ -147,7 +174,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Calculate product-level forecasts await connection.query(` @@ -253,7 +285,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Create temporary table for category stats await connection.query(` @@ -298,7 +335,12 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Calculate category-level forecasts await connection.query(` @@ -374,8 +416,25 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount, } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('sales_forecasts', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating sales forecasts'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/time-aggregates.js b/inventory-server/scripts/metrics/time-aggregates.js index 58911ec..9930a98 100644 --- a/inventory-server/scripts/metrics/time-aggregates.js +++ b/inventory-server/scripts/metrics/time-aggregates.js @@ -1,8 +1,11 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateTimeAggregates(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateTimeAggregates(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +23,22 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders: 0, + processedPurchaseOrders: 0, + success + }; } + // Get order count that will be processed + const [orderCount] = await connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + `); + processedOrders = orderCount[0].count; + outputProgress({ status: 'running', operation: 'Starting time aggregates calculation', @@ -163,7 +179,12 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; // Update with financial metrics await connection.query(` @@ -209,8 +230,25 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount, } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('time_aggregates', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders: 0, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating time aggregates'); throw error; } finally { diff --git a/inventory-server/scripts/metrics/vendor-metrics.js b/inventory-server/scripts/metrics/vendor-metrics.js index a988581..5444c75 100644 --- a/inventory-server/scripts/metrics/vendor-metrics.js +++ b/inventory-server/scripts/metrics/vendor-metrics.js @@ -1,8 +1,12 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress'); const { getConnection } = require('./utils/db'); -async function calculateVendorMetrics(startTime, totalProducts, processedCount, isCancelled = false) { +async function calculateVendorMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { const connection = await getConnection(); + let success = false; + let processedOrders = 0; + let processedPurchaseOrders = 0; + try { if (isCancelled) { outputProgress({ @@ -20,9 +24,30 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, elapsed_seconds: Math.round((Date.now() - startTime) / 1000) } }); - return processedCount; + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders, + success + }; } + // Get counts of records that will be processed + const [[orderCount], [poCount]] = await Promise.all([ + connection.query(` + SELECT COUNT(*) as count + FROM orders o + WHERE o.canceled = false + `), + connection.query(` + SELECT COUNT(*) as count + FROM purchase_orders po + WHERE po.status != 0 + `) + ]); + processedOrders = orderCount.count; + processedPurchaseOrders = poCount.count; + outputProgress({ status: 'running', operation: 'Starting vendor metrics calculation', @@ -68,7 +93,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders, + success + }; // Now calculate vendor metrics await connection.query(` @@ -191,7 +221,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, } }); - if (isCancelled) return processedCount; + if (isCancelled) return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders, + success + }; // Calculate time-based metrics await connection.query(` @@ -302,8 +337,25 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount, } }); - return processedCount; + // If we get here, everything completed successfully + success = true; + + // Update calculate_status + await connection.query(` + INSERT INTO calculate_status (module_name, last_calculation_timestamp) + VALUES ('vendor_metrics', NOW()) + ON DUPLICATE KEY UPDATE last_calculation_timestamp = NOW() + `); + + return { + processedProducts: processedCount, + processedOrders, + processedPurchaseOrders, + success + }; + } catch (error) { + success = false; logError(error, 'Error calculating vendor metrics'); throw error; } finally {