Clean up old historical data calcs/scripts, optimize calculations to not update every row every time

inventory-server/old/historical-data.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
+const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../scripts/metrics-new/utils/progress');
 const fs = require('fs');
 const path = require('path');
 const { pipeline } = require('stream');

inventory-server/old/populate-initial-metrics.js

@@ -24,7 +24,7 @@ process.on('unhandledRejection', (reason, promise) => {
 });
 
 // Load progress module
-const progress = require('../utils/progress');
+const progress = require('../scripts/metrics-new/utils/progress');
 
 // Store progress functions in global scope to ensure availability
 global.formatElapsedTime = progress.formatElapsedTime;
@@ -36,7 +36,7 @@ global.getProgress = progress.getProgress;
 global.logError = progress.logError;
 
 // Load database module
-const { getConnection, closePool } = require('../utils/db');
+const { getConnection, closePool } = require('../scripts/metrics-new/utils/db');
 
 // Add cancel handler
 let isCancelled = false;

inventory-server/scripts/import-from-prod.js

@@ -6,7 +6,6 @@ const importCategories = require('./import/categories');
 const { importProducts } = require('./import/products');
 const importOrders = require('./import/orders');
 const importPurchaseOrders = require('./import/purchase-orders');
-const importHistoricalData = require('./import/historical-data');
 
 dotenv.config({ path: path.join(__dirname, "../.env") });
 
@@ -15,7 +14,6 @@ const IMPORT_CATEGORIES = true;
 const IMPORT_PRODUCTS = true;
 const IMPORT_ORDERS = true;
 const IMPORT_PURCHASE_ORDERS = true;
-const IMPORT_HISTORICAL_DATA = false;
 
 // Add flag for incremental updates
 const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false';  // Default to true unless explicitly set to false
@@ -80,8 +78,7 @@ async function main() {
     IMPORT_CATEGORIES,
     IMPORT_PRODUCTS,
     IMPORT_ORDERS,
-    IMPORT_PURCHASE_ORDERS,
+    IMPORT_PURCHASE_ORDERS
-    IMPORT_HISTORICAL_DATA
   ].filter(Boolean).length;
 
   try {
@@ -129,11 +126,10 @@ async function main() {
             'categories_enabled', $2::boolean,
             'products_enabled', $3::boolean,
             'orders_enabled', $4::boolean,
-            'purchase_orders_enabled', $5::boolean,
+            'purchase_orders_enabled', $5::boolean
-            'historical_data_enabled', $6::boolean
           )
         ) RETURNING id
-      `, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS, IMPORT_HISTORICAL_DATA]);
+      `, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
       importHistoryId = historyResult.rows[0].id;
     } catch (error) {
       console.error("Error creating import history record:", error);
@@ -150,8 +146,7 @@ async function main() {
       categories: null,
       products: null,
       orders: null,
-      purchaseOrders: null,
+      purchaseOrders: null
-      historicalData: null
     };
 
     let totalRecordsAdded = 0;
@@ -211,32 +206,6 @@ async function main() {
       }
     }
 
-    if (IMPORT_HISTORICAL_DATA) {
-      try {
-        results.historicalData = await importHistoricalData(prodConnection, localConnection, INCREMENTAL_UPDATE);
-        if (isImportCancelled) throw new Error("Import cancelled");
-        completedSteps++;
-        console.log('Historical data import result:', results.historicalData);
-        
-        // Handle potential error status
-        if (results.historicalData?.status === 'error') {
-          console.error('Historical data import had an error:', results.historicalData.error);
-        } else {
-          totalRecordsAdded += parseInt(results.historicalData?.recordsAdded || 0);
-          totalRecordsUpdated += parseInt(results.historicalData?.recordsUpdated || 0);
-        }
-      } catch (error) {
-        console.error('Error during historical data import:', error);
-        // Continue with other imports, don't fail the whole process
-        results.historicalData = {
-          status: 'error',
-          error: error.message,
-          recordsAdded: 0,
-          recordsUpdated: 0
-        };
-      }
-    }
-
     const endTime = Date.now();
     const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
 
@@ -254,14 +223,12 @@ async function main() {
           'products_enabled', $5::boolean,
           'orders_enabled', $6::boolean,
           'purchase_orders_enabled', $7::boolean,
-          'historical_data_enabled', $8::boolean,
+          'categories_result', COALESCE($8::jsonb, 'null'::jsonb),
-          'categories_result', COALESCE($9::jsonb, 'null'::jsonb),
+          'products_result', COALESCE($9::jsonb, 'null'::jsonb),
-          'products_result', COALESCE($10::jsonb, 'null'::jsonb),
+          'orders_result', COALESCE($10::jsonb, 'null'::jsonb),
-          'orders_result', COALESCE($11::jsonb, 'null'::jsonb),
+          'purchase_orders_result', COALESCE($11::jsonb, 'null'::jsonb)
-          'purchase_orders_result', COALESCE($12::jsonb, 'null'::jsonb),
-          'historical_data_result', COALESCE($13::jsonb, 'null'::jsonb)
         )
-      WHERE id = $14
+      WHERE id = $12
     `, [
       totalElapsedSeconds,
       parseInt(totalRecordsAdded),
@@ -270,12 +237,10 @@ async function main() {
       IMPORT_PRODUCTS,
       IMPORT_ORDERS,
       IMPORT_PURCHASE_ORDERS,
-      IMPORT_HISTORICAL_DATA,
       JSON.stringify(results.categories),
       JSON.stringify(results.products),
       JSON.stringify(results.orders),
       JSON.stringify(results.purchaseOrders),
-      JSON.stringify(results.historicalData),
       importHistoryId
     ]);
 

inventory-server/scripts/metrics-new/calculate_brand_metrics.sql

@@ -95,7 +95,14 @@ BEGIN
         profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
         sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
         lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
-        avg_margin_30d = EXCLUDED.avg_margin_30d;
+        avg_margin_30d = EXCLUDED.avg_margin_30d
+    WHERE -- Only update if at least one value has changed
+        brand_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
+        brand_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
+        brand_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
+        brand_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
+        brand_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
+        brand_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
 
     -- Update calculate_status
     INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)

inventory-server/scripts/metrics-new/calculate_category_metrics.sql

@@ -238,7 +238,8 @@ BEGIN
         category_type = EXCLUDED.category_type,
         parent_id = EXCLUDED.parent_id,
         last_calculated = EXCLUDED.last_calculated,
-        -- Update rolled-up metrics
+        
+        -- ROLLED-UP METRICS (includes this category + all descendants)
         product_count = EXCLUDED.product_count,
         active_product_count = EXCLUDED.active_product_count,
         replenishable_product_count = EXCLUDED.replenishable_product_count,
@@ -250,7 +251,8 @@ BEGIN
         profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
         sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
         lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
-        -- Update direct metrics
+        
+        -- DIRECT METRICS (only products directly in this category)
         direct_product_count = EXCLUDED.direct_product_count,
         direct_active_product_count = EXCLUDED.direct_active_product_count,
         direct_replenishable_product_count = EXCLUDED.direct_replenishable_product_count,
@@ -262,9 +264,19 @@ BEGIN
         direct_profit_30d = EXCLUDED.direct_profit_30d, direct_cogs_30d = EXCLUDED.direct_cogs_30d,
         direct_sales_365d = EXCLUDED.direct_sales_365d, direct_revenue_365d = EXCLUDED.direct_revenue_365d,
         direct_lifetime_sales = EXCLUDED.direct_lifetime_sales, direct_lifetime_revenue = EXCLUDED.direct_lifetime_revenue,
-        -- Update KPIs
+        
+        -- Calculated KPIs
         avg_margin_30d = EXCLUDED.avg_margin_30d,
-        stock_turn_30d = EXCLUDED.stock_turn_30d;
+        stock_turn_30d = EXCLUDED.stock_turn_30d
+    WHERE -- Only update if at least one value has changed
+        category_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
+        category_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
+        category_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
+        category_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
+        category_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
+        category_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
+        category_metrics.direct_product_count IS DISTINCT FROM EXCLUDED.direct_product_count OR
+        category_metrics.direct_sales_30d IS DISTINCT FROM EXCLUDED.direct_sales_30d;
 
     -- Update calculate_status
     INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)

inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql

@@ -124,7 +124,15 @@ BEGIN
         profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
         sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
         lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
-        avg_margin_30d = EXCLUDED.avg_margin_30d;
+        avg_margin_30d = EXCLUDED.avg_margin_30d
+    WHERE -- Only update if at least one value has changed
+        vendor_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
+        vendor_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
+        vendor_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
+        vendor_metrics.on_order_units IS DISTINCT FROM EXCLUDED.on_order_units OR
+        vendor_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
+        vendor_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
+        vendor_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
 
     -- Update calculate_status
     INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)

inventory-server/scripts/metrics-new/update_product_metrics.sql

@@ -735,6 +735,22 @@ BEGIN
         overstocked_units = EXCLUDED.overstocked_units, overstocked_cost = EXCLUDED.overstocked_cost, overstocked_retail = EXCLUDED.overstocked_retail, is_old_stock = EXCLUDED.is_old_stock,
         yesterday_sales = EXCLUDED.yesterday_sales,
         status = EXCLUDED.status
+        WHERE -- Only update if at least one key metric has changed
+            product_metrics.current_stock IS DISTINCT FROM EXCLUDED.current_stock OR
+            product_metrics.current_price IS DISTINCT FROM EXCLUDED.current_price OR
+            product_metrics.current_cost_price IS DISTINCT FROM EXCLUDED.current_cost_price OR
+            product_metrics.on_order_qty IS DISTINCT FROM EXCLUDED.on_order_qty OR
+            product_metrics.sales_7d IS DISTINCT FROM EXCLUDED.sales_7d OR
+            product_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
+            product_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
+            product_metrics.status IS DISTINCT FROM EXCLUDED.status OR
+            product_metrics.replenishment_units IS DISTINCT FROM EXCLUDED.replenishment_units OR
+            product_metrics.stock_cover_in_days IS DISTINCT FROM EXCLUDED.stock_cover_in_days OR
+            product_metrics.yesterday_sales IS DISTINCT FROM EXCLUDED.yesterday_sales OR
+            -- Check a few other important fields that might change
+            product_metrics.date_last_sold IS DISTINCT FROM EXCLUDED.date_last_sold OR
+            product_metrics.earliest_expected_date IS DISTINCT FROM EXCLUDED.earliest_expected_date OR
+            product_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales
         ;
 
     -- Update the status table with the timestamp from the START of this run