Import/metrics calc fixes

2026-02-08 22:44:57 -05:00
parent 12cc7a4639
commit 7c41a7f799
11 changed files with 828 additions and 55 deletions
--- a/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql
+++ b/inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql
@@ -214,7 +214,7 @@ BEGIN
    -- Final INSERT/UPDATE statement using all the prepared CTEs
    INSERT INTO public.product_metrics (
        pid, last_calculated, sku, title, brand, vendor, image_url, is_visible, is_replenishable,
-        current_price, current_regular_price, current_cost_price, current_landing_cost_price,
+        current_price, current_regular_price, current_cost_price,
        current_stock, current_stock_cost, current_stock_retail, current_stock_gross,
        on_order_qty, on_order_cost, on_order_retail, earliest_expected_date,
        date_created, date_first_received, date_last_received, date_first_sold, date_last_sold, age_days,
@@ -242,7 +242,7 @@ BEGIN
    SELECT
        -- Select columns in order, joining all CTEs by pid
        ci.pid, _start_time, ci.sku, ci.title, ci.brand, ci.vendor, ci.image_url, ci.is_visible, ci.replenishable,
-        ci.current_price, ci.current_regular_price, ci.current_cost_price, ci.current_effective_cost,
+        ci.current_price, ci.current_regular_price, ci.current_cost_price,
        ci.current_stock, (ci.current_stock * COALESCE(ci.current_effective_cost, 0.00))::numeric(12,2), (ci.current_stock * COALESCE(ci.current_price, 0.00))::numeric(12,2), (ci.current_stock * COALESCE(ci.current_regular_price, 0.00))::numeric(12,2),
        COALESCE(ooi.on_order_qty, 0), COALESCE(ooi.on_order_cost, 0.00)::numeric(12,2), (COALESCE(ooi.on_order_qty, 0) * COALESCE(ci.current_price, 0.00))::numeric(12,2), ooi.earliest_expected_date,
        
@@ -415,7 +415,7 @@ BEGIN
        -- *** IMPORTANT: List ALL columns here, ensuring order matches INSERT list ***
        -- Update ALL columns to ensure entire row is refreshed
        last_calculated = EXCLUDED.last_calculated, sku = EXCLUDED.sku, title = EXCLUDED.title, brand = EXCLUDED.brand, vendor = EXCLUDED.vendor, image_url = EXCLUDED.image_url, is_visible = EXCLUDED.is_visible, is_replenishable = EXCLUDED.is_replenishable,
-        current_price = EXCLUDED.current_price, current_regular_price = EXCLUDED.current_regular_price, current_cost_price = EXCLUDED.current_cost_price, current_landing_cost_price = EXCLUDED.current_landing_cost_price,
+        current_price = EXCLUDED.current_price, current_regular_price = EXCLUDED.current_regular_price, current_cost_price = EXCLUDED.current_cost_price,
        current_stock = EXCLUDED.current_stock, current_stock_cost = EXCLUDED.current_stock_cost, current_stock_retail = EXCLUDED.current_stock_retail, current_stock_gross = EXCLUDED.current_stock_gross,
        on_order_qty = EXCLUDED.on_order_qty, on_order_cost = EXCLUDED.on_order_cost, on_order_retail = EXCLUDED.on_order_retail, earliest_expected_date = EXCLUDED.earliest_expected_date,
        date_created = EXCLUDED.date_created, date_first_received = EXCLUDED.date_first_received, date_last_received = EXCLUDED.date_last_received, date_first_sold = EXCLUDED.date_first_sold, date_last_sold = EXCLUDED.date_last_sold, age_days = EXCLUDED.age_days,
--- a/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
+++ b/inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
@@ -13,7 +13,7 @@ DECLARE
    _begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data
    _end_date DATE := CURRENT_DATE;
 BEGIN
-    RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;
+    RAISE NOTICE 'Begicnning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;

    -- First truncate the existing snapshots to ensure a clean slate
    TRUNCATE TABLE public.daily_product_snapshots;
@@ -36,7 +36,13 @@ BEGIN
                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
-                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
+                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN
+                    COALESCE(
+                        o.costeach,
+                        get_weighted_avg_cost(p.pid, o.date::date),
+                        p.cost_price
+                    ) * o.quantity
+                ELSE 0 END), 0.00) AS cogs,
                COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,

                -- Aggregate Returns (Quantity < 0 or Status = Returned)
@@ -63,15 +69,17 @@ BEGIN
            GROUP BY r.pid
            HAVING COUNT(DISTINCT r.receiving_id) > 0 OR SUM(r.qty_each) > 0
        ),
-        -- Get stock quantities for the day - note this is approximate since we're using current products data
+        -- Use historical stock from stock_snapshots when available,
+        -- falling back to current stock from products table
        StockData AS (
            SELECT
                p.pid,
-                p.stock_quantity,
-                COALESCE(p.cost_price, 0.00) as effective_cost_price,
+                COALESCE(ss.stock_quantity, p.stock_quantity) AS stock_quantity,
+                COALESCE(ss.stock_value, p.stock_quantity * COALESCE(p.cost_price, 0.00)) AS stock_value,
                COALESCE(p.price, 0.00) as current_price,
                COALESCE(p.regular_price, 0.00) as current_regular_price
            FROM public.products p
+            LEFT JOIN stock_snapshots ss ON p.pid = ss.pid AND ss.snapshot_date = _date
        )
        INSERT INTO public.daily_product_snapshots (
            snapshot_date,
@@ -99,9 +107,9 @@ BEGIN
            _date AS snapshot_date,
            COALESCE(sd.pid, rd.pid) AS pid,
            sd.sku,
-            -- Use current stock as approximation, since historical stock data may not be available
+            -- Historical stock from stock_snapshots, falls back to current stock
            s.stock_quantity AS eod_stock_quantity,
-            s.stock_quantity * s.effective_cost_price AS eod_stock_cost,
+            s.stock_value AS eod_stock_cost,
            s.stock_quantity * s.current_price AS eod_stock_retail,
            s.stock_quantity * s.current_regular_price AS eod_stock_gross,
            (s.stock_quantity <= 0) AS stockout_flag,
@@ -114,7 +122,7 @@ BEGIN
            COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
            COALESCE(sd.cogs, 0.00),
            COALESCE(sd.gross_regular_revenue, 0.00),
-            (COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
+            (COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
            -- Receiving metrics
            COALESCE(rd.units_received, 0),
            COALESCE(rd.cost_received, 0.00),