From 12cc7a4639294d11c41e450dfe19c9ab6e90ea1d Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 7 Feb 2026 21:34:42 -0500 Subject: [PATCH] Fixes for metrics calculations --- inventory-server/scripts/import/orders.js | 38 ++++- inventory-server/scripts/import/products.js | 27 ++-- .../populate_initial_product_metrics.sql | 2 +- .../backfill/rebuild_daily_snapshots.sql | 8 +- .../metrics-new/calculate_brand_metrics.sql | 4 +- .../calculate_category_metrics.sql | 96 ++++++------ .../metrics-new/calculate_vendor_metrics.sql | 6 +- .../migrations/001_map_order_statuses.sql | 38 +++++ .../002_fix_discount_double_counting.sql | 51 +++++++ .../metrics-new/update_daily_snapshots.sql | 143 +++++++++--------- .../metrics-new/update_product_metrics.sql | 10 +- inventory-server/src/routes/metrics.js | 3 +- inventory-server/src/routes/products.js | 2 - .../analytics/CapitalEfficiency.tsx | 2 + .../src/components/products/ProductDetail.tsx | 1 - .../components/products/ProductFilters.tsx | 1 - .../components/products/columnDefinitions.ts | 1 - inventory/src/types/products.ts | 3 - 18 files changed, 267 insertions(+), 169 deletions(-) create mode 100644 inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql create mode 100644 inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql diff --git a/inventory-server/scripts/import/orders.js b/inventory-server/scripts/import/orders.js index c1a2450..eafe420 100644 --- a/inventory-server/scripts/import/orders.js +++ b/inventory-server/scripts/import/orders.js @@ -17,6 +17,33 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = const startTime = Date.now(); const skippedOrders = new Set(); const missingProducts = new Set(); + + // Map order status codes to text values (consistent with PO status mapping in purchase-orders.js) + const orderStatusMap = { + 0: 'created', + 10: 'unfinished', + 15: 'canceled', + 16: 'combined', + 20: 'placed', + 22: 'placed_incomplete', + 30: 'canceled', + 40: 'awaiting_payment', + 50: 'awaiting_products', + 55: 'shipping_later', + 56: 'shipping_together', + 60: 'ready', + 61: 'flagged', + 62: 'fix_before_pick', + 65: 'manual_picking', + 70: 'in_pt', + 80: 'picked', + 90: 'awaiting_shipment', + 91: 'remote_wait', + 92: 'awaiting_pickup', + 93: 'fix_before_ship', + 95: 'shipped_confirmed', + 100: 'shipped' + }; let recordsAdded = 0; let recordsUpdated = 0; let processedCount = 0; @@ -284,7 +311,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = new Date(order.date), // Convert to TIMESTAMP WITH TIME ZONE order.customer, toTitleCase(order.customer_name) || '', - order.status.toString(), // Convert status to TEXT + orderStatusMap[order.status] || order.status.toString(), // Map numeric status to text order.canceled, order.summary_discount || 0, order.summary_subtotal || 0, @@ -587,17 +614,14 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = oi.price, oi.quantity, ( - -- Part 1: Sale Savings for the Line - (oi.base_discount * oi.quantity) - + - -- Part 2: Prorated Points Discount (if applicable) + -- Prorated Points Discount (e.g. loyalty points applied at order level) CASE WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0) ELSE 0 END + - -- Part 3: Specific Item-Level Discount (only if parent discount affected subtotal) + -- Specific Item-Level Promo Discount (coupon codes, etc.) COALESCE(ot.promo_discount_sum, 0) )::NUMERIC(14, 4) as discount, COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax, @@ -654,7 +678,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = o.shipping, o.customer, o.customer_name, - o.status.toString(), // Convert status to TEXT + o.status, // Already mapped to text via orderStatusMap o.canceled, o.costeach ]); diff --git a/inventory-server/scripts/import/products.js b/inventory-server/scripts/import/products.js index e7c6539..ac66ccf 100644 --- a/inventory-server/scripts/import/products.js +++ b/inventory-server/scripts/import/products.js @@ -77,7 +77,6 @@ async function setupTemporaryTables(connection) { created_at TIMESTAMP WITH TIME ZONE, date_online TIMESTAMP WITH TIME ZONE, first_received TIMESTAMP WITH TIME ZONE, - landing_cost_price NUMERIC(14, 4), barcode TEXT, harmonized_tariff_code TEXT, updated_at TIMESTAMP WITH TIME ZONE, @@ -172,7 +171,6 @@ async function importMissingProducts(prodConnection, localConnection, missingPid ) ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1) END AS cost_price, - NULL as landing_cost_price, s.companyname AS vendor, CASE WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber @@ -242,8 +240,8 @@ async function importMissingProducts(prodConnection, localConnection, missingPid const batch = prodData.slice(i, i + BATCH_SIZE); const placeholders = batch.map((_, idx) => { - const base = idx * 50; // 50 columns - return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`; + const base = idx * 49; // 49 columns + return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`; }).join(','); const values = batch.flatMap(row => { @@ -270,7 +268,6 @@ async function importMissingProducts(prodConnection, localConnection, missingPid validateDate(row.date_created), validateDate(row.date_ol), validateDate(row.first_received), - row.landing_cost_price, row.barcode, row.harmonized_tariff_code, validateDate(row.updated_at), @@ -308,7 +305,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count, price, regular_price, cost_price, vendor, vendor_reference, notions_reference, brand, line, subline, artist, categories, created_at, date_online, first_received, - landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible, + barcode, harmonized_tariff_code, updated_at, visible, managing_stock, replenishable, permalink, moq, uom, rating, reviews, weight, length, width, height, country_of_origin, location, total_sold, baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags @@ -382,7 +379,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen ) ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1) END AS cost_price, - NULL as landing_cost_price, s.companyname AS vendor, CASE WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber @@ -457,8 +453,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen await withRetry(async () => { const placeholders = batch.map((_, idx) => { - const base = idx * 50; // 50 columns - return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`; + const base = idx * 49; // 49 columns + return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`; }).join(','); const values = batch.flatMap(row => { @@ -485,7 +481,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen validateDate(row.date_created), validateDate(row.date_ol), validateDate(row.first_received), - row.landing_cost_price, row.barcode, row.harmonized_tariff_code, validateDate(row.updated_at), @@ -522,7 +517,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count, price, regular_price, cost_price, vendor, vendor_reference, notions_reference, brand, line, subline, artist, categories, created_at, date_online, first_received, - landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible, + barcode, harmonized_tariff_code, updated_at, visible, managing_stock, replenishable, permalink, moq, uom, rating, reviews, weight, length, width, height, country_of_origin, location, total_sold, baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags @@ -547,7 +542,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen created_at = EXCLUDED.created_at, date_online = EXCLUDED.date_online, first_received = EXCLUDED.first_received, - landing_cost_price = EXCLUDED.landing_cost_price, barcode = EXCLUDED.barcode, harmonized_tariff_code = EXCLUDED.harmonized_tariff_code, updated_at = EXCLUDED.updated_at, @@ -702,7 +696,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate t.created_at, t.date_online, t.first_received, - t.landing_cost_price, t.barcode, t.harmonized_tariff_code, t.updated_at, @@ -742,8 +735,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate const batch = products.rows.slice(i, i + BATCH_SIZE); const placeholders = batch.map((_, idx) => { - const base = idx * 49; // 49 columns - return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`; + const base = idx * 48; // 48 columns (no primary_iid in this INSERT) + return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`; }).join(','); const values = batch.flatMap(row => { @@ -770,7 +763,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate validateDate(row.created_at), validateDate(row.date_online), validateDate(row.first_received), - row.landing_cost_price, row.barcode, row.harmonized_tariff_code, validateDate(row.updated_at), @@ -807,7 +799,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count, price, regular_price, cost_price, vendor, vendor_reference, notions_reference, brand, line, subline, artist, categories, created_at, date_online, first_received, - landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible, + barcode, harmonized_tariff_code, updated_at, visible, managing_stock, replenishable, permalink, moq, uom, rating, reviews, weight, length, width, height, country_of_origin, location, total_sold, baskets, notifies, date_last_sold, shop_score, image, image_175, image_full, options, tags @@ -833,7 +825,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate created_at = EXCLUDED.created_at, date_online = EXCLUDED.date_online, first_received = EXCLUDED.first_received, - landing_cost_price = EXCLUDED.landing_cost_price, barcode = EXCLUDED.barcode, harmonized_tariff_code = EXCLUDED.harmonized_tariff_code, updated_at = EXCLUDED.updated_at, diff --git a/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql b/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql index b78d27d..e412b3a 100644 --- a/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql +++ b/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql @@ -27,7 +27,7 @@ BEGIN p.visible as is_visible, p.replenishable, COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price, COALESCE(p.cost_price, 0.00) as current_cost_price, - COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost + COALESCE(p.cost_price, 0.00) as current_effective_cost, p.stock_quantity as current_stock, -- Use actual current stock for forecast base p.created_at, p.first_received, p.date_last_sold, p.moq, diff --git a/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql b/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql index dceb6f6..af7a90c 100644 --- a/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql +++ b/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql @@ -10,7 +10,7 @@ DECLARE _date DATE; _count INT; _total_records INT := 0; - _begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2024-01-01'); -- Starting point for data rebuild + _begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data _end_date DATE := CURRENT_DATE; BEGIN RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time; @@ -36,7 +36,7 @@ BEGIN COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold, COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts, - COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.landing_cost_price, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs, + COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs, COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Aggregate Returns (Quantity < 0 or Status = Returned) @@ -68,7 +68,7 @@ BEGIN SELECT p.pid, p.stock_quantity, - COALESCE(p.landing_cost_price, p.cost_price, 0.00) as effective_cost_price, + COALESCE(p.cost_price, 0.00) as effective_cost_price, COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price FROM public.products p @@ -111,7 +111,7 @@ BEGIN COALESCE(sd.gross_revenue_unadjusted, 0.00), COALESCE(sd.discounts, 0.00), COALESCE(sd.returns_revenue, 0.00), - COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue, + COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue, COALESCE(sd.cogs, 0.00), COALESCE(sd.gross_regular_revenue, 0.00), (COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit, diff --git a/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql b/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql index f0fd505..c3e3e27 100644 --- a/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_brand_metrics.sql @@ -28,8 +28,8 @@ BEGIN COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d, SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d, SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d, - SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d, - SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d, + SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d, + SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d, COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d, SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d, diff --git a/inventory-server/scripts/metrics-new/calculate_category_metrics.sql b/inventory-server/scripts/metrics-new/calculate_category_metrics.sql index a55d2ef..21079db 100644 --- a/inventory-server/scripts/metrics-new/calculate_category_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_category_metrics.sql @@ -28,8 +28,8 @@ BEGIN SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d, SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d, SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d, - SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d, - SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d, + SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d, + SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d, SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d, SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d, SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales, @@ -38,58 +38,56 @@ BEGIN JOIN public.product_metrics pm ON pc.pid = pm.pid GROUP BY pc.cat_id ), - -- Calculate rolled-up metrics (including all descendant categories) + -- Map each category to ALL distinct products in it or any descendant. + -- Uses the path array from category_hierarchy: for product P in category C, + -- P contributes to C and every ancestor in C's path. + -- DISTINCT ensures each (ancestor, pid) pair appears only once, preventing + -- double-counting when a product belongs to multiple categories under the same parent. + CategoryProducts AS ( + SELECT DISTINCT + ancestor_cat_id, + pc.pid + FROM public.product_categories pc + JOIN category_hierarchy ch ON pc.cat_id = ch.cat_id + CROSS JOIN LATERAL unnest(ch.path) AS ancestor_cat_id + ), + -- Calculate rolled-up metrics using deduplicated product sets RolledUpMetrics AS ( SELECT - ch.cat_id, - -- Sum metrics from this category and all its descendants - SUM(dcm.product_count) AS product_count, - SUM(dcm.active_product_count) AS active_product_count, - SUM(dcm.replenishable_product_count) AS replenishable_product_count, - SUM(dcm.current_stock_units) AS current_stock_units, - SUM(dcm.current_stock_cost) AS current_stock_cost, - SUM(dcm.current_stock_retail) AS current_stock_retail, - SUM(dcm.sales_7d) AS sales_7d, - SUM(dcm.revenue_7d) AS revenue_7d, - SUM(dcm.sales_30d) AS sales_30d, - SUM(dcm.revenue_30d) AS revenue_30d, - SUM(dcm.cogs_30d) AS cogs_30d, - SUM(dcm.profit_30d) AS profit_30d, - SUM(dcm.sales_365d) AS sales_365d, - SUM(dcm.revenue_365d) AS revenue_365d, - SUM(dcm.lifetime_sales) AS lifetime_sales, - SUM(dcm.lifetime_revenue) AS lifetime_revenue - FROM category_hierarchy ch - LEFT JOIN DirectCategoryMetrics dcm ON - dcm.cat_id = ch.cat_id OR - dcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids)) - GROUP BY ch.cat_id - ), - PreviousPeriodCategoryMetrics AS ( - -- Get previous period metrics for growth calculation - SELECT - pc.cat_id, - SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days' - AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days' - THEN dps.units_sold ELSE 0 END) AS sales_prev_30d, - SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days' - AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days' - THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d - FROM public.daily_product_snapshots dps - JOIN public.product_categories pc ON dps.pid = pc.pid - GROUP BY pc.cat_id + cp.ancestor_cat_id AS cat_id, + COUNT(DISTINCT cp.pid) AS product_count, + COUNT(DISTINCT CASE WHEN pm.is_visible THEN cp.pid END) AS active_product_count, + COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN cp.pid END) AS replenishable_product_count, + SUM(pm.current_stock) AS current_stock_units, + SUM(pm.current_stock_cost) AS current_stock_cost, + SUM(pm.current_stock_retail) AS current_stock_retail, + SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d, + SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d, + SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d, + SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d, + SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d, + SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d, + SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d, + SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d, + SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales, + SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue + FROM CategoryProducts cp + JOIN public.product_metrics pm ON cp.pid = pm.pid + GROUP BY cp.ancestor_cat_id ), + -- Previous period rolled up using same deduplicated product sets RolledUpPreviousPeriod AS ( - -- Calculate rolled-up previous period metrics SELECT - ch.cat_id, - SUM(ppcm.sales_prev_30d) AS sales_prev_30d, - SUM(ppcm.revenue_prev_30d) AS revenue_prev_30d - FROM category_hierarchy ch - LEFT JOIN PreviousPeriodCategoryMetrics ppcm ON - ppcm.cat_id = ch.cat_id OR - ppcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids)) - GROUP BY ch.cat_id + cp.ancestor_cat_id AS cat_id, + SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days' + AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days' + THEN dps.units_sold ELSE 0 END) AS sales_prev_30d, + SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days' + AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days' + THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d + FROM CategoryProducts cp + JOIN public.daily_product_snapshots dps ON cp.pid = dps.pid + GROUP BY cp.ancestor_cat_id ), AllCategories AS ( -- Ensure all categories are included diff --git a/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql b/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql index 04eb2cf..de40be8 100644 --- a/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql +++ b/inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql @@ -29,8 +29,8 @@ BEGIN COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d, SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d, SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d, - SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d, - SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d, + SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d, + SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d, COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d, SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d, @@ -72,7 +72,7 @@ BEGIN END))::int AS avg_lead_time_days_hist -- Avg lead time from HISTORICAL received POs FROM public.purchase_orders po -- Join to receivings table to find when items were received - LEFT JOIN public.receivings r ON r.pid = po.pid + LEFT JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id WHERE po.vendor IS NOT NULL AND po.vendor <> '' AND po.date >= CURRENT_DATE - INTERVAL '1 year' -- Look at POs created in the last year AND po.status = 'done' -- Only calculate lead time on completed POs diff --git a/inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql b/inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql new file mode 100644 index 0000000..defbce6 --- /dev/null +++ b/inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql @@ -0,0 +1,38 @@ +-- Migration: Map existing numeric order statuses to text values +-- Run this ONCE on the production PostgreSQL database after deploying the updated orders import. +-- This updates ~2.88M rows. On a busy system, consider running during low-traffic hours. +-- The WHERE clause ensures idempotency - only rows with numeric statuses are updated. + +UPDATE orders SET status = CASE status + WHEN '0' THEN 'created' + WHEN '10' THEN 'unfinished' + WHEN '15' THEN 'canceled' + WHEN '16' THEN 'combined' + WHEN '20' THEN 'placed' + WHEN '22' THEN 'placed_incomplete' + WHEN '30' THEN 'canceled' + WHEN '40' THEN 'awaiting_payment' + WHEN '50' THEN 'awaiting_products' + WHEN '55' THEN 'shipping_later' + WHEN '56' THEN 'shipping_together' + WHEN '60' THEN 'ready' + WHEN '61' THEN 'flagged' + WHEN '62' THEN 'fix_before_pick' + WHEN '65' THEN 'manual_picking' + WHEN '70' THEN 'in_pt' + WHEN '80' THEN 'picked' + WHEN '90' THEN 'awaiting_shipment' + WHEN '91' THEN 'remote_wait' + WHEN '92' THEN 'awaiting_pickup' + WHEN '93' THEN 'fix_before_ship' + WHEN '95' THEN 'shipped_confirmed' + WHEN '100' THEN 'shipped' + ELSE status +END +WHERE status ~ '^\d+$'; -- Only update rows that still have numeric statuses + +-- Verify the migration +SELECT status, COUNT(*) as count +FROM orders +GROUP BY status +ORDER BY count DESC; diff --git a/inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql b/inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql new file mode 100644 index 0000000..a35f70b --- /dev/null +++ b/inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql @@ -0,0 +1,51 @@ +-- Migration 002: Fix discount double-counting in orders +-- +-- PROBLEM: The orders import was calculating discount as: +-- discount = (prod_price_reg - prod_price) * quantity <-- "sale savings" (WRONG) +-- + prorated points discount +-- + item-level promo discounts +-- +-- Since `price` in the orders table already IS the sale price (prod_price, not prod_price_reg), +-- the "sale savings" component double-counted the markdown. This resulted in inflated discounts +-- and near-zero net_revenue for products sold on sale. +-- +-- Example: Product with regular_price=$30, sale_price=$15, qty=2 +-- BEFORE (buggy): discount = ($30-$15)*2 + 0 + 0 = $30.00 +-- net_revenue = $15*2 - $30 = $0.00 (WRONG!) +-- AFTER (fixed): discount = 0 + 0 + 0 = $0.00 +-- net_revenue = $15*2 - $0 = $30.00 (CORRECT!) +-- +-- FIX: This cannot be fixed with a pure SQL migration because PostgreSQL doesn't store +-- prod_price_reg. The discount column has the inflated value baked in, and we can't +-- decompose which portion was the base_discount vs actual promo discounts. +-- +-- REQUIRED ACTION: Run a FULL (non-incremental) orders re-import after deploying the +-- fixed orders.js. This will recalculate all discounts using the corrected formula. +-- +-- Steps: +-- 1. Deploy updated orders.js (base_discount removed from discount calculation) +-- 2. Run: node scripts/import/orders.js --full +-- (or trigger a full sync through whatever mechanism is used) +-- 3. After re-import, run the daily snapshots rebuild to propagate corrected revenue: +-- psql -f scripts/metrics-new/backfill/rebuild_daily_snapshots.sql +-- 4. Re-run metrics calculation: +-- node scripts/metrics-new/calculate-metrics-new.js +-- +-- VERIFICATION: After re-import, check the previously-affected products: +SELECT + o.pid, + p.title, + o.order_number, + o.price, + o.quantity, + o.discount, + (o.price * o.quantity) as gross_revenue, + (o.price * o.quantity - o.discount) as net_revenue +FROM orders o +JOIN products p ON o.pid = p.pid +WHERE o.pid IN (624756, 614513) +ORDER BY o.date DESC +LIMIT 10; + +-- Expected: discount should be 0 (or small promo amount) for regular sales, +-- and net_revenue should be close to gross_revenue. diff --git a/inventory-server/scripts/metrics-new/update_daily_snapshots.sql b/inventory-server/scripts/metrics-new/update_daily_snapshots.sql index ff05315..6040973 100644 --- a/inventory-server/scripts/metrics-new/update_daily_snapshots.sql +++ b/inventory-server/scripts/metrics-new/update_daily_snapshots.sql @@ -1,75 +1,73 @@ --- Description: Calculates and updates daily aggregated product data for recent days. --- Uses UPSERT (INSERT ON CONFLICT UPDATE) for idempotency. +-- Description: Calculates and updates daily aggregated product data. +-- Self-healing: automatically detects and fills gaps in snapshot history. +-- Always reprocesses recent days to pick up new orders and data corrections. -- Dependencies: Core import tables (products, orders, purchase_orders), calculate_status table. -- Frequency: Hourly (Run ~5-10 minutes after hourly data import completes). DO $$ DECLARE _module_name TEXT := 'daily_snapshots'; - _start_time TIMESTAMPTZ := clock_timestamp(); -- Time execution started - _last_calc_time TIMESTAMPTZ; - _target_date DATE; -- Will be set in the loop + _start_time TIMESTAMPTZ := clock_timestamp(); + _target_date DATE; _total_records INT := 0; - _has_orders BOOLEAN := FALSE; - _process_days INT := 5; -- Number of days to check/process (today plus previous 4 days) - _day_counter INT; - _missing_days INT[] := ARRAY[]::INT[]; -- Array to store days with missing or incomplete data + _days_processed INT := 0; + _max_backfill_days INT := 90; -- Safety cap: max days to backfill per run + _recent_recheck_days INT := 2; -- Always reprocess this many recent days (today + yesterday) + _latest_snapshot DATE; + _backfill_start DATE; BEGIN - -- Get the timestamp before the last successful run of this module - SELECT last_calculation_timestamp INTO _last_calc_time - FROM public.calculate_status - WHERE module_name = _module_name; - RAISE NOTICE 'Running % script. Start Time: %', _module_name, _start_time; - - -- First, check which days need processing by comparing orders data with snapshot data - FOR _day_counter IN 0..(_process_days-1) LOOP - _target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day'); - - -- Check if this date needs updating by comparing orders to snapshot data - -- If the date has orders but not enough snapshots, or if snapshots show zero sales but orders exist, it's incomplete - SELECT - CASE WHEN ( - -- We have orders for this date but not enough snapshots, or snapshots with wrong total - (EXISTS (SELECT 1 FROM public.orders WHERE date::date = _target_date) AND - ( - -- No snapshots exist for this date - NOT EXISTS (SELECT 1 FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) OR - -- Or snapshots show zero sales but orders exist - (SELECT COALESCE(SUM(units_sold), 0) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) = 0 OR - -- Or the count of snapshot records is significantly less than distinct products in orders - (SELECT COUNT(*) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) < - (SELECT COUNT(DISTINCT pid) FROM public.orders WHERE date::date = _target_date) * 0.8 - ) - ) - ) THEN TRUE ELSE FALSE END - INTO _has_orders; - - IF _has_orders THEN - -- This day needs processing - add to our array - _missing_days := _missing_days || _day_counter; - RAISE NOTICE 'Day % needs updating (incomplete or missing data)', _target_date; - END IF; - END LOOP; - - -- If no days need updating, exit early - IF array_length(_missing_days, 1) IS NULL THEN - RAISE NOTICE 'No days need updating - all snapshot data appears complete'; - - -- Still update the calculate_status to record this run - UPDATE public.calculate_status - SET last_calculation_timestamp = _start_time - WHERE module_name = _module_name; - - RETURN; - END IF; - - RAISE NOTICE 'Need to update % days with missing or incomplete data', array_length(_missing_days, 1); - -- Process only the days that need updating - FOREACH _day_counter IN ARRAY _missing_days LOOP - _target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day'); - RAISE NOTICE 'Processing date: %', _target_date; + -- Find the latest existing snapshot date to determine where gaps begin + SELECT MAX(snapshot_date) INTO _latest_snapshot + FROM public.daily_product_snapshots; + + -- Determine how far back to look for gaps, capped at _max_backfill_days + _backfill_start := GREATEST( + COALESCE(_latest_snapshot + 1, CURRENT_DATE - _max_backfill_days), + CURRENT_DATE - _max_backfill_days + ); + + IF _latest_snapshot IS NULL THEN + RAISE NOTICE 'No existing snapshots found. Backfilling up to % days.', _max_backfill_days; + ELSIF _backfill_start > _latest_snapshot + 1 THEN + RAISE NOTICE 'Latest snapshot: %. Gap exceeds % day cap — backfilling from %. Use rebuild script for full history.', + _latest_snapshot, _max_backfill_days, _backfill_start; + ELSE + RAISE NOTICE 'Latest snapshot: %. Checking for gaps from %.', _latest_snapshot, _backfill_start; + END IF; + + -- Process all dates that need snapshots: + -- 1. Gap fill: dates with orders/receivings but no snapshots (older than recent window) + -- 2. Recent recheck: last N days always reprocessed (picks up new orders, corrections) + FOR _target_date IN + SELECT d FROM ( + -- Gap fill: find dates with activity but missing snapshots + SELECT activity_dates.d + FROM ( + SELECT DISTINCT date::date AS d FROM public.orders + WHERE date::date >= _backfill_start AND date::date < CURRENT_DATE - _recent_recheck_days + UNION + SELECT DISTINCT received_date::date AS d FROM public.receivings + WHERE received_date::date >= _backfill_start AND received_date::date < CURRENT_DATE - _recent_recheck_days + ) activity_dates + WHERE NOT EXISTS ( + SELECT 1 FROM public.daily_product_snapshots dps WHERE dps.snapshot_date = activity_dates.d + ) + UNION + -- Recent days: always reprocess + SELECT d::date + FROM generate_series( + (CURRENT_DATE - _recent_recheck_days)::timestamp, + CURRENT_DATE::timestamp, + '1 day'::interval + ) d + ) dates_to_process + ORDER BY d + LOOP + _days_processed := _days_processed + 1; + RAISE NOTICE 'Processing date: % [%/%]', _target_date, _days_processed, + _days_processed; -- count not known ahead of time, but shows progress -- IMPORTANT: First delete any existing data for this date to prevent duplication DELETE FROM public.daily_product_snapshots @@ -90,7 +88,6 @@ BEGIN COALESCE( o.costeach, -- First use order-specific cost if available get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost - p.landing_cost_price, -- Fallback to landing cost p.cost_price -- Final fallback to current cost ) * o.quantity ELSE 0 END), 0.00) AS cogs, @@ -128,7 +125,7 @@ BEGIN SELECT pid, stock_quantity, - COALESCE(landing_cost_price, cost_price, 0.00) as effective_cost_price, + COALESCE(cost_price, 0.00) as effective_cost_price, COALESCE(price, 0.00) as current_price, COALESCE(regular_price, 0.00) as current_regular_price FROM public.products @@ -181,7 +178,7 @@ BEGIN COALESCE(sd.gross_revenue_unadjusted, 0.00), COALESCE(sd.discounts, 0.00), COALESCE(sd.returns_revenue, 0.00), - COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue, + COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue, COALESCE(sd.cogs, 0.00), COALESCE(sd.gross_regular_revenue, 0.00), (COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit, -- Basic profit: Net Revenue - COGS @@ -201,12 +198,18 @@ BEGIN RAISE NOTICE 'Created % daily snapshot records for % with sales/receiving activity', _total_records, _target_date; END LOOP; - -- Update the status table with the timestamp from the START of this run - UPDATE public.calculate_status - SET last_calculation_timestamp = _start_time - WHERE module_name = _module_name; + IF _days_processed = 0 THEN + RAISE NOTICE 'No days need updating — all snapshot data is current.'; + ELSE + RAISE NOTICE 'Processed % days total.', _days_processed; + END IF; - RAISE NOTICE 'Finished % processing for multiple dates. Duration: %', _module_name, clock_timestamp() - _start_time; + -- Update the status table with the timestamp from the START of this run + INSERT INTO public.calculate_status (module_name, last_calculation_timestamp) + VALUES (_module_name, _start_time) + ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time; + + RAISE NOTICE 'Finished % script. Duration: %', _module_name, clock_timestamp() - _start_time; END $$; diff --git a/inventory-server/scripts/metrics-new/update_product_metrics.sql b/inventory-server/scripts/metrics-new/update_product_metrics.sql index 2554c55..050eb73 100644 --- a/inventory-server/scripts/metrics-new/update_product_metrics.sql +++ b/inventory-server/scripts/metrics-new/update_product_metrics.sql @@ -52,7 +52,7 @@ BEGIN COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price, COALESCE(p.cost_price, 0.00) as current_cost_price, - COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost + COALESCE(p.cost_price, 0.00) as current_effective_cost, p.stock_quantity as current_stock, p.created_at, p.first_received, @@ -321,10 +321,10 @@ BEGIN (GREATEST(0, ci.historical_total_sold - COALESCE(lr.lifetime_units_from_orders, 0)) * COALESCE( -- Use oldest known price from snapshots as proxy - (SELECT revenue_7d / NULLIF(sales_7d, 0) - FROM daily_product_snapshots - WHERE pid = ci.pid AND sales_7d > 0 - ORDER BY snapshot_date ASC + (SELECT net_revenue / NULLIF(units_sold, 0) + FROM daily_product_snapshots + WHERE pid = ci.pid AND units_sold > 0 + ORDER BY snapshot_date ASC LIMIT 1), ci.current_price )) diff --git a/inventory-server/src/routes/metrics.js b/inventory-server/src/routes/metrics.js index 9122d21..6fe13ba 100644 --- a/inventory-server/src/routes/metrics.js +++ b/inventory-server/src/routes/metrics.js @@ -43,7 +43,6 @@ const COLUMN_MAP = { currentPrice: 'pm.current_price', currentRegularPrice: 'pm.current_regular_price', currentCostPrice: 'pm.current_cost_price', - currentLandingCostPrice: 'pm.current_landing_cost_price', currentStock: 'pm.current_stock', currentStockCost: 'pm.current_stock_cost', currentStockRetail: 'pm.current_stock_retail', @@ -176,7 +175,7 @@ const COLUMN_MAP = { const COLUMN_TYPES = { // Numeric columns (use numeric operators and sorting) numeric: [ - 'pid', 'currentPrice', 'currentRegularPrice', 'currentCostPrice', 'currentLandingCostPrice', + 'pid', 'currentPrice', 'currentRegularPrice', 'currentCostPrice', 'currentStock', 'currentStockCost', 'currentStockRetail', 'currentStockGross', 'onOrderQty', 'onOrderCost', 'onOrderRetail', 'ageDays', 'sales7d', 'revenue7d', 'sales14d', 'revenue14d', 'sales30d', 'revenue30d', diff --git a/inventory-server/src/routes/products.js b/inventory-server/src/routes/products.js index 01a862e..6397c2c 100644 --- a/inventory-server/src/routes/products.js +++ b/inventory-server/src/routes/products.js @@ -145,7 +145,6 @@ router.get('/', async (req, res) => { stock: 'p.stock_quantity', price: 'p.price', costPrice: 'p.cost_price', - landingCost: 'p.landing_cost_price', dailySalesAvg: 'pm.daily_sales_avg', weeklySalesAvg: 'pm.weekly_sales_avg', monthlySalesAvg: 'pm.monthly_sales_avg', @@ -621,7 +620,6 @@ router.get('/:id', async (req, res) => { price: parseFloat(productRows[0].price), regular_price: parseFloat(productRows[0].regular_price), cost_price: parseFloat(productRows[0].cost_price), - landing_cost_price: parseFloat(productRows[0].landing_cost_price), stock_quantity: parseInt(productRows[0].stock_quantity), moq: parseInt(productRows[0].moq), uom: parseInt(productRows[0].uom), diff --git a/inventory/src/components/analytics/CapitalEfficiency.tsx b/inventory/src/components/analytics/CapitalEfficiency.tsx index 6475048..20adb7a 100644 --- a/inventory/src/components/analytics/CapitalEfficiency.tsx +++ b/inventory/src/components/analytics/CapitalEfficiency.tsx @@ -127,6 +127,7 @@ export function CapitalEfficiency() { tickFormatter={formatCurrency} tick={{ fontSize: 11 }} type="number" + label={{ value: 'Stock Investment', position: 'insideBottom', offset: -5, fontSize: 12, fill: '#888' }} /> - diff --git a/inventory/src/components/products/ProductFilters.tsx b/inventory/src/components/products/ProductFilters.tsx index ae0dec9..f21c1de 100644 --- a/inventory/src/components/products/ProductFilters.tsx +++ b/inventory/src/components/products/ProductFilters.tsx @@ -119,7 +119,6 @@ const BASE_FILTER_OPTIONS: FilterOption[] = [ { id: "currentPrice", label: "Current Price", type: "number", group: "Pricing", operators: ["=", ">", ">=", "<", "<=", "between"] }, { id: "currentRegularPrice", label: "Regular Price", type: "number", group: "Pricing", operators: ["=", ">", ">=", "<", "<=", "between"] }, { id: "currentCostPrice", label: "Cost Price", type: "number", group: "Pricing", operators: ["=", ">", ">=", "<", "<=", "between"] }, - { id: "currentLandingCostPrice", label: "Landing Cost", type: "number", group: "Pricing", operators: ["=", ">", ">=", "<", "<=", "between"] }, // Valuation Group { id: "currentStockCost", label: "Current Stock Cost", type: "number", group: "Valuation", operators: ["=", ">", ">=", "<", "<=", "between"] }, diff --git a/inventory/src/components/products/columnDefinitions.ts b/inventory/src/components/products/columnDefinitions.ts index 443a7f3..b816e7a 100644 --- a/inventory/src/components/products/columnDefinitions.ts +++ b/inventory/src/components/products/columnDefinitions.ts @@ -95,7 +95,6 @@ export const AVAILABLE_COLUMNS: ColumnDef[] = [ { key: 'currentPrice', label: 'Price', group: 'Pricing', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, { key: 'currentRegularPrice', label: 'Regular Price', group: 'Pricing', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, { key: 'currentCostPrice', label: 'Cost', group: 'Pricing', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, - { key: 'currentLandingCostPrice', label: 'Landing Cost', group: 'Pricing', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, { key: 'currentStockCost', label: 'Stock Cost', group: 'Valuation', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, { key: 'currentStockRetail', label: 'Stock Retail', group: 'Valuation', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, { key: 'currentStockGross', label: 'Stock Gross', group: 'Valuation', format: (v) => v === 0 ? '0' : v ? v.toFixed(2) : '-' }, diff --git a/inventory/src/types/products.ts b/inventory/src/types/products.ts index 3e2ccf1..5fdcf4e 100644 --- a/inventory/src/types/products.ts +++ b/inventory/src/types/products.ts @@ -6,7 +6,6 @@ export interface Product { price: string; // numeric(15,3) regular_price: string; // numeric(15,3) cost_price: string; // numeric(15,3) - landing_cost_price: string | null; // numeric(15,3) barcode: string; vendor: string; vendor_reference: string; @@ -126,7 +125,6 @@ export interface ProductMetric { currentPrice: number | null; currentRegularPrice: number | null; currentCostPrice: number | null; - currentLandingCostPrice: number | null; currentStock: number; currentStockCost: number | null; currentStockRetail: number | null; @@ -310,7 +308,6 @@ export type ProductMetricColumnKey = | 'currentPrice' | 'currentRegularPrice' | 'currentCostPrice' - | 'currentLandingCostPrice' | 'configSafetyStock' | 'replenishmentUnits' | 'stockCoverInDays'