diff --git a/inventory-server/scripts/import/historical-data.js b/inventory-server/old/historical-data.js similarity index 99% rename from inventory-server/scripts/import/historical-data.js rename to inventory-server/old/historical-data.js index bdde8f6..ae34f24 100644 --- a/inventory-server/scripts/import/historical-data.js +++ b/inventory-server/old/historical-data.js @@ -1,4 +1,4 @@ -const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress'); +const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../scripts/metrics-new/utils/progress'); const fs = require('fs'); const path = require('path'); const { pipeline } = require('stream'); diff --git a/inventory-server/scripts/metrics-new/backfill/populate-initial-metrics.js b/inventory-server/old/populate-initial-metrics.js similarity index 99% rename from inventory-server/scripts/metrics-new/backfill/populate-initial-metrics.js rename to inventory-server/old/populate-initial-metrics.js index 89e6f7b..379cbb5 100644 --- a/inventory-server/scripts/metrics-new/backfill/populate-initial-metrics.js +++ b/inventory-server/old/populate-initial-metrics.js @@ -24,7 +24,7 @@ process.on('unhandledRejection', (reason, promise) => { }); // Load progress module -const progress = require('../utils/progress'); +const progress = require('../scripts/metrics-new/utils/progress'); // Store progress functions in global scope to ensure availability global.formatElapsedTime = progress.formatElapsedTime; @@ -36,7 +36,7 @@ global.getProgress = progress.getProgress; global.logError = progress.logError; // Load database module -const { getConnection, closePool } = require('../utils/db'); +const { getConnection, closePool } = require('../scripts/metrics-new/utils/db'); // Add cancel handler let isCancelled = false; diff --git a/inventory-server/scripts/psql-csv-import.sh b/inventory-server/old/psql-csv-import.sh similarity index 100% rename from inventory-server/scripts/psql-csv-import.sh rename to inventory-server/old/psql-csv-import.sh diff --git a/inventory-server/scripts/import-from-prod.js b/inventory-server/scripts/import-from-prod.js index a010df8..716b622 100644 --- a/inventory-server/scripts/import-from-prod.js +++ b/inventory-server/scripts/import-from-prod.js @@ -6,7 +6,6 @@ const importCategories = require('./import/categories'); const { importProducts } = require('./import/products'); const importOrders = require('./import/orders'); const importPurchaseOrders = require('./import/purchase-orders'); -const importHistoricalData = require('./import/historical-data'); dotenv.config({ path: path.join(__dirname, "../.env") }); @@ -15,7 +14,6 @@ const IMPORT_CATEGORIES = true; const IMPORT_PRODUCTS = true; const IMPORT_ORDERS = true; const IMPORT_PURCHASE_ORDERS = true; -const IMPORT_HISTORICAL_DATA = false; // Add flag for incremental updates const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false'; // Default to true unless explicitly set to false @@ -80,8 +78,7 @@ async function main() { IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, - IMPORT_PURCHASE_ORDERS, - IMPORT_HISTORICAL_DATA + IMPORT_PURCHASE_ORDERS ].filter(Boolean).length; try { @@ -129,11 +126,10 @@ async function main() { 'categories_enabled', $2::boolean, 'products_enabled', $3::boolean, 'orders_enabled', $4::boolean, - 'purchase_orders_enabled', $5::boolean, - 'historical_data_enabled', $6::boolean + 'purchase_orders_enabled', $5::boolean ) ) RETURNING id - `, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS, IMPORT_HISTORICAL_DATA]); + `, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]); importHistoryId = historyResult.rows[0].id; } catch (error) { console.error("Error creating import history record:", error); @@ -150,8 +146,7 @@ async function main() { categories: null, products: null, orders: null, - purchaseOrders: null, - historicalData: null + purchaseOrders: null }; let totalRecordsAdded = 0; @@ -211,32 +206,6 @@ async function main() { } } - if (IMPORT_HISTORICAL_DATA) { - try { - results.historicalData = await importHistoricalData(prodConnection, localConnection, INCREMENTAL_UPDATE); - if (isImportCancelled) throw new Error("Import cancelled"); - completedSteps++; - console.log('Historical data import result:', results.historicalData); - - // Handle potential error status - if (results.historicalData?.status === 'error') { - console.error('Historical data import had an error:', results.historicalData.error); - } else { - totalRecordsAdded += parseInt(results.historicalData?.recordsAdded || 0); - totalRecordsUpdated += parseInt(results.historicalData?.recordsUpdated || 0); - } - } catch (error) { - console.error('Error during historical data import:', error); - // Continue with other imports, don't fail the whole process - results.historicalData = { - status: 'error', - error: error.message, - recordsAdded: 0, - recordsUpdated: 0 - }; - } - } - const endTime = Date.now(); const totalElapsedSeconds = Math.round((endTime - startTime) / 1000); @@ -254,14 +223,12 @@ async function main() { 'products_enabled', $5::boolean, 'orders_enabled', $6::boolean, 'purchase_orders_enabled', $7::boolean, - 'historical_data_enabled', $8::boolean, - 'categories_result', COALESCE($9::jsonb, 'null'::jsonb), - 'products_result', COALESCE($10::jsonb, 'null'::jsonb), - 'orders_result', COALESCE($11::jsonb, 'null'::jsonb), - 'purchase_orders_result', COALESCE($12::jsonb, 'null'::jsonb), - 'historical_data_result', COALESCE($13::jsonb, 'null'::jsonb) + 'categories_result', COALESCE($8::jsonb, 'null'::jsonb), + 'products_result', COALESCE($9::jsonb, 'null'::jsonb), + 'orders_result', COALESCE($10::jsonb, 'null'::jsonb), + 'purchase_orders_result', COALESCE($11::jsonb, 'null'::jsonb) ) - WHERE id = $14 + WHERE id = $12 `, [ totalElapsedSeconds, parseInt(totalRecordsAdded), @@ -270,12 +237,10 @@ async function main() { IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS, - IMPORT_HISTORICAL_DATA, JSON.stringify(results.categories), JSON.stringify(results.products), JSON.stringify(results.orders), JSON.stringify(results.purchaseOrders), - JSON.stringify(results.historicalData), importHistoryId ]); diff --git a/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql b/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql index 92d61bd..b033197 100644 --- a/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql @@ -95,7 +95,14 @@ BEGIN profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d, sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d, lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue, - avg_margin_30d = EXCLUDED.avg_margin_30d; + avg_margin_30d = EXCLUDED.avg_margin_30d + WHERE -- Only update if at least one value has changed + brand_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR + brand_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR + brand_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR + brand_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR + brand_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR + brand_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales; -- Update calculate_status INSERT INTO public.calculate_status (module_name, last_calculation_timestamp) diff --git a/inventory-server/scripts/metrics-new/calculate_category_metrics.sql b/inventory-server/scripts/metrics-new/calculate_category_metrics.sql index 995278d..3dda928 100644 --- a/inventory-server/scripts/metrics-new/calculate_category_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_category_metrics.sql @@ -238,7 +238,8 @@ BEGIN category_type = EXCLUDED.category_type, parent_id = EXCLUDED.parent_id, last_calculated = EXCLUDED.last_calculated, - -- Update rolled-up metrics + + -- ROLLED-UP METRICS (includes this category + all descendants) product_count = EXCLUDED.product_count, active_product_count = EXCLUDED.active_product_count, replenishable_product_count = EXCLUDED.replenishable_product_count, @@ -250,7 +251,8 @@ BEGIN profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d, sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d, lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue, - -- Update direct metrics + + -- DIRECT METRICS (only products directly in this category) direct_product_count = EXCLUDED.direct_product_count, direct_active_product_count = EXCLUDED.direct_active_product_count, direct_replenishable_product_count = EXCLUDED.direct_replenishable_product_count, @@ -262,9 +264,19 @@ BEGIN direct_profit_30d = EXCLUDED.direct_profit_30d, direct_cogs_30d = EXCLUDED.direct_cogs_30d, direct_sales_365d = EXCLUDED.direct_sales_365d, direct_revenue_365d = EXCLUDED.direct_revenue_365d, direct_lifetime_sales = EXCLUDED.direct_lifetime_sales, direct_lifetime_revenue = EXCLUDED.direct_lifetime_revenue, - -- Update KPIs + + -- Calculated KPIs avg_margin_30d = EXCLUDED.avg_margin_30d, - stock_turn_30d = EXCLUDED.stock_turn_30d; + stock_turn_30d = EXCLUDED.stock_turn_30d + WHERE -- Only update if at least one value has changed + category_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR + category_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR + category_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR + category_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR + category_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR + category_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR + category_metrics.direct_product_count IS DISTINCT FROM EXCLUDED.direct_product_count OR + category_metrics.direct_sales_30d IS DISTINCT FROM EXCLUDED.direct_sales_30d; -- Update calculate_status INSERT INTO public.calculate_status (module_name, last_calculation_timestamp) diff --git a/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql b/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql index 3d342f4..0a71e4d 100644 --- a/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql @@ -124,7 +124,15 @@ BEGIN profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d, sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d, lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue, - avg_margin_30d = EXCLUDED.avg_margin_30d; + avg_margin_30d = EXCLUDED.avg_margin_30d + WHERE -- Only update if at least one value has changed + vendor_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR + vendor_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR + vendor_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR + vendor_metrics.on_order_units IS DISTINCT FROM EXCLUDED.on_order_units OR + vendor_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR + vendor_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR + vendor_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales; -- Update calculate_status INSERT INTO public.calculate_status (module_name, last_calculation_timestamp) diff --git a/inventory-server/scripts/metrics-new/update_product_metrics.sql b/inventory-server/scripts/metrics-new/update_product_metrics.sql index 11b4337..63ba36e 100644 --- a/inventory-server/scripts/metrics-new/update_product_metrics.sql +++ b/inventory-server/scripts/metrics-new/update_product_metrics.sql @@ -735,6 +735,22 @@ BEGIN overstocked_units = EXCLUDED.overstocked_units, overstocked_cost = EXCLUDED.overstocked_cost, overstocked_retail = EXCLUDED.overstocked_retail, is_old_stock = EXCLUDED.is_old_stock, yesterday_sales = EXCLUDED.yesterday_sales, status = EXCLUDED.status + WHERE -- Only update if at least one key metric has changed + product_metrics.current_stock IS DISTINCT FROM EXCLUDED.current_stock OR + product_metrics.current_price IS DISTINCT FROM EXCLUDED.current_price OR + product_metrics.current_cost_price IS DISTINCT FROM EXCLUDED.current_cost_price OR + product_metrics.on_order_qty IS DISTINCT FROM EXCLUDED.on_order_qty OR + product_metrics.sales_7d IS DISTINCT FROM EXCLUDED.sales_7d OR + product_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR + product_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR + product_metrics.status IS DISTINCT FROM EXCLUDED.status OR + product_metrics.replenishment_units IS DISTINCT FROM EXCLUDED.replenishment_units OR + product_metrics.stock_cover_in_days IS DISTINCT FROM EXCLUDED.stock_cover_in_days OR + product_metrics.yesterday_sales IS DISTINCT FROM EXCLUDED.yesterday_sales OR + -- Check a few other important fields that might change + product_metrics.date_last_sold IS DISTINCT FROM EXCLUDED.date_last_sold OR + product_metrics.earliest_expected_date IS DISTINCT FROM EXCLUDED.earliest_expected_date OR + product_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales ; -- Update the status table with the timestamp from the START of this run