matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes for metrics calculations

Browse Source
This commit is contained in:
Matt
2026-02-07 21:34:42 -05:00
parent 9b2f9016f6
commit 12cc7a4639
18 changed files with 267 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql
View File

@@ -27,7 +27,7 @@ BEGIN
p.visible as is_visible, p.replenishable,
COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
COALESCE(p.cost_price, 0.00) as current_effective_cost,
p.stock_quantity as current_stock, -- Use actual current stock for forecast base
p.created_at, p.first_received, p.date_last_sold,
p.moq,

8
inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
View File

@@ -10,7 +10,7 @@ DECLARE
_date DATE;
_count INT;
_total_records INT := 0;
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2024-01-01'); -- Starting point for data rebuild
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data
_end_date DATE := CURRENT_DATE;
BEGIN
RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;
@@ -36,7 +36,7 @@ BEGIN
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.landing_cost_price, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,
-- Aggregate Returns (Quantity < 0 or Status = Returned)
@@ -68,7 +68,7 @@ BEGIN
SELECT
p.pid,
p.stock_quantity,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as effective_cost_price,
COALESCE(p.cost_price, 0.00) as effective_cost_price,
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price
FROM public.products p
@@ -111,7 +111,7 @@ BEGIN
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,

4
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
View File

@@ -28,8 +28,8 @@ BEGIN
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,

96
inventory-server/scripts/metrics-new/calculate_category_metrics.sql
View File

@@ -28,8 +28,8 @@ BEGIN
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
@@ -38,58 +38,56 @@ BEGIN
JOIN public.product_metrics pm ON pc.pid = pm.pid
GROUP BY pc.cat_id
),
-- Calculate rolled-up metrics (including all descendant categories)
-- Map each category to ALL distinct products in it or any descendant.
-- Uses the path array from category_hierarchy: for product P in category C,
-- P contributes to C and every ancestor in C's path.
-- DISTINCT ensures each (ancestor, pid) pair appears only once, preventing
-- double-counting when a product belongs to multiple categories under the same parent.
CategoryProducts AS (
SELECT DISTINCT
ancestor_cat_id,
pc.pid
FROM public.product_categories pc
JOIN category_hierarchy ch ON pc.cat_id = ch.cat_id
CROSS JOIN LATERAL unnest(ch.path) AS ancestor_cat_id
),
-- Calculate rolled-up metrics using deduplicated product sets
RolledUpMetrics AS (
SELECT
ch.cat_id,
-- Sum metrics from this category and all its descendants
SUM(dcm.product_count) AS product_count,
SUM(dcm.active_product_count) AS active_product_count,
SUM(dcm.replenishable_product_count) AS replenishable_product_count,
SUM(dcm.current_stock_units) AS current_stock_units,
SUM(dcm.current_stock_cost) AS current_stock_cost,
SUM(dcm.current_stock_retail) AS current_stock_retail,
SUM(dcm.sales_7d) AS sales_7d,
SUM(dcm.revenue_7d) AS revenue_7d,
SUM(dcm.sales_30d) AS sales_30d,
SUM(dcm.revenue_30d) AS revenue_30d,
SUM(dcm.cogs_30d) AS cogs_30d,
SUM(dcm.profit_30d) AS profit_30d,
SUM(dcm.sales_365d) AS sales_365d,
SUM(dcm.revenue_365d) AS revenue_365d,
SUM(dcm.lifetime_sales) AS lifetime_sales,
SUM(dcm.lifetime_revenue) AS lifetime_revenue
FROM category_hierarchy ch
LEFT JOIN DirectCategoryMetrics dcm ON
dcm.cat_id = ch.cat_id OR
dcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
),
PreviousPeriodCategoryMetrics AS (
-- Get previous period metrics for growth calculation
SELECT
pc.cat_id,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM public.daily_product_snapshots dps
JOIN public.product_categories pc ON dps.pid = pc.pid
GROUP BY pc.cat_id
cp.ancestor_cat_id AS cat_id,
COUNT(DISTINCT cp.pid) AS product_count,
COUNT(DISTINCT CASE WHEN pm.is_visible THEN cp.pid END) AS active_product_count,
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN cp.pid END) AS replenishable_product_count,
SUM(pm.current_stock) AS current_stock_units,
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
FROM CategoryProducts cp
JOIN public.product_metrics pm ON cp.pid = pm.pid
GROUP BY cp.ancestor_cat_id
),
-- Previous period rolled up using same deduplicated product sets
RolledUpPreviousPeriod AS (
-- Calculate rolled-up previous period metrics
SELECT
ch.cat_id,
SUM(ppcm.sales_prev_30d) AS sales_prev_30d,
SUM(ppcm.revenue_prev_30d) AS revenue_prev_30d
FROM category_hierarchy ch
LEFT JOIN PreviousPeriodCategoryMetrics ppcm ON
ppcm.cat_id = ch.cat_id OR
ppcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
cp.ancestor_cat_id AS cat_id,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM CategoryProducts cp
JOIN public.daily_product_snapshots dps ON cp.pid = dps.pid
GROUP BY cp.ancestor_cat_id
),
AllCategories AS (
-- Ensure all categories are included

6
inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql
View File

@@ -29,8 +29,8 @@ BEGIN
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
@@ -72,7 +72,7 @@ BEGIN
END))::int AS avg_lead_time_days_hist -- Avg lead time from HISTORICAL received POs
FROM public.purchase_orders po
-- Join to receivings table to find when items were received
LEFT JOIN public.receivings r ON r.pid = po.pid
LEFT JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id
WHERE po.vendor IS NOT NULL AND po.vendor <> ''
AND po.date >= CURRENT_DATE - INTERVAL '1 year' -- Look at POs created in the last year
AND po.status = 'done' -- Only calculate lead time on completed POs

38
inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql Normal file
View File

@@ -0,0 +1,38 @@
-- Migration: Map existing numeric order statuses to text values
-- Run this ONCE on the production PostgreSQL database after deploying the updated orders import.
-- This updates ~2.88M rows. On a busy system, consider running during low-traffic hours.
-- The WHERE clause ensures idempotency - only rows with numeric statuses are updated.
UPDATE orders SET status = CASE status
WHEN '0' THEN 'created'
WHEN '10' THEN 'unfinished'
WHEN '15' THEN 'canceled'
WHEN '16' THEN 'combined'
WHEN '20' THEN 'placed'
WHEN '22' THEN 'placed_incomplete'
WHEN '30' THEN 'canceled'
WHEN '40' THEN 'awaiting_payment'
WHEN '50' THEN 'awaiting_products'
WHEN '55' THEN 'shipping_later'
WHEN '56' THEN 'shipping_together'
WHEN '60' THEN 'ready'
WHEN '61' THEN 'flagged'
WHEN '62' THEN 'fix_before_pick'
WHEN '65' THEN 'manual_picking'
WHEN '70' THEN 'in_pt'
WHEN '80' THEN 'picked'
WHEN '90' THEN 'awaiting_shipment'
WHEN '91' THEN 'remote_wait'
WHEN '92' THEN 'awaiting_pickup'
WHEN '93' THEN 'fix_before_ship'
WHEN '95' THEN 'shipped_confirmed'
WHEN '100' THEN 'shipped'
ELSE status
END
WHERE status ~ '^\d+$'; -- Only update rows that still have numeric statuses
-- Verify the migration
SELECT status, COUNT(*) as count
FROM orders
GROUP BY status
ORDER BY count DESC;

51
inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql Normal file
View File

@@ -0,0 +1,51 @@
-- Migration 002: Fix discount double-counting in orders
--
-- PROBLEM: The orders import was calculating discount as:
-- discount = (prod_price_reg - prod_price) * quantity <-- "sale savings" (WRONG)
-- + prorated points discount
-- + item-level promo discounts
--
-- Since `price` in the orders table already IS the sale price (prod_price, not prod_price_reg),
-- the "sale savings" component double-counted the markdown. This resulted in inflated discounts
-- and near-zero net_revenue for products sold on sale.
--
-- Example: Product with regular_price=$30, sale_price=$15, qty=2
-- BEFORE (buggy): discount = ($30-$15)*2 + 0 + 0 = $30.00
-- net_revenue = $15*2 - $30 = $0.00 (WRONG!)
-- AFTER (fixed): discount = 0 + 0 + 0 = $0.00
-- net_revenue = $15*2 - $0 = $30.00 (CORRECT!)
--
-- FIX: This cannot be fixed with a pure SQL migration because PostgreSQL doesn't store
-- prod_price_reg. The discount column has the inflated value baked in, and we can't
-- decompose which portion was the base_discount vs actual promo discounts.
--
-- REQUIRED ACTION: Run a FULL (non-incremental) orders re-import after deploying the
-- fixed orders.js. This will recalculate all discounts using the corrected formula.
--
-- Steps:
-- 1. Deploy updated orders.js (base_discount removed from discount calculation)
-- 2. Run: node scripts/import/orders.js --full
-- (or trigger a full sync through whatever mechanism is used)
-- 3. After re-import, run the daily snapshots rebuild to propagate corrected revenue:
-- psql -f scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
-- 4. Re-run metrics calculation:
-- node scripts/metrics-new/calculate-metrics-new.js
--
-- VERIFICATION: After re-import, check the previously-affected products:
SELECT
o.pid,
p.title,
o.order_number,
o.price,
o.quantity,
o.discount,
(o.price * o.quantity) as gross_revenue,
(o.price * o.quantity - o.discount) as net_revenue
FROM orders o
JOIN products p ON o.pid = p.pid
WHERE o.pid IN (624756, 614513)
ORDER BY o.date DESC
LIMIT 10;
-- Expected: discount should be 0 (or small promo amount) for regular sales,
-- and net_revenue should be close to gross_revenue.

143
inventory-server/scripts/metrics-new/update_daily_snapshots.sql
View File

@@ -1,75 +1,73 @@
-- Description: Calculates and updates daily aggregated product data for recent days.
-- Uses UPSERT (INSERT ON CONFLICT UPDATE) for idempotency.
-- Description: Calculates and updates daily aggregated product data.
-- Self-healing: automatically detects and fills gaps in snapshot history.
-- Always reprocesses recent days to pick up new orders and data corrections.
-- Dependencies: Core import tables (products, orders, purchase_orders), calculate_status table.
-- Frequency: Hourly (Run ~5-10 minutes after hourly data import completes).
DO $$
DECLARE
_module_name TEXT := 'daily_snapshots';
_start_time TIMESTAMPTZ := clock_timestamp(); -- Time execution started
_last_calc_time TIMESTAMPTZ;
_target_date DATE; -- Will be set in the loop
_start_time TIMESTAMPTZ := clock_timestamp();
_target_date DATE;
_total_records INT := 0;
_has_orders BOOLEAN := FALSE;
_process_days INT := 5; -- Number of days to check/process (today plus previous 4 days)
_day_counter INT;
_missing_days INT[] := ARRAY[]::INT[]; -- Array to store days with missing or incomplete data
_days_processed INT := 0;
_max_backfill_days INT := 90; -- Safety cap: max days to backfill per run
_recent_recheck_days INT := 2; -- Always reprocess this many recent days (today + yesterday)
_latest_snapshot DATE;
_backfill_start DATE;
BEGIN
-- Get the timestamp before the last successful run of this module
SELECT last_calculation_timestamp INTO _last_calc_time
FROM public.calculate_status
WHERE module_name = _module_name;
RAISE NOTICE 'Running % script. Start Time: %', _module_name, _start_time;
-- First, check which days need processing by comparing orders data with snapshot data
FOR _day_counter IN 0..(_process_days-1) LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
-- Check if this date needs updating by comparing orders to snapshot data
-- If the date has orders but not enough snapshots, or if snapshots show zero sales but orders exist, it's incomplete
SELECT
CASE WHEN (
-- We have orders for this date but not enough snapshots, or snapshots with wrong total
(EXISTS (SELECT 1 FROM public.orders WHERE date::date = _target_date) AND
(
-- No snapshots exist for this date
NOT EXISTS (SELECT 1 FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) OR
-- Or snapshots show zero sales but orders exist
(SELECT COALESCE(SUM(units_sold), 0) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) = 0 OR
-- Or the count of snapshot records is significantly less than distinct products in orders
(SELECT COUNT(*) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) <
(SELECT COUNT(DISTINCT pid) FROM public.orders WHERE date::date = _target_date) * 0.8
)
)
) THEN TRUE ELSE FALSE END
INTO _has_orders;
IF _has_orders THEN
-- This day needs processing - add to our array
_missing_days := _missing_days || _day_counter;
RAISE NOTICE 'Day % needs updating (incomplete or missing data)', _target_date;
END IF;
END LOOP;
-- If no days need updating, exit early
IF array_length(_missing_days, 1) IS NULL THEN
RAISE NOTICE 'No days need updating - all snapshot data appears complete';
-- Still update the calculate_status to record this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RETURN;
END IF;
RAISE NOTICE 'Need to update % days with missing or incomplete data', array_length(_missing_days, 1);
-- Process only the days that need updating
FOREACH _day_counter IN ARRAY _missing_days LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
RAISE NOTICE 'Processing date: %', _target_date;
-- Find the latest existing snapshot date to determine where gaps begin
SELECT MAX(snapshot_date) INTO _latest_snapshot
FROM public.daily_product_snapshots;
-- Determine how far back to look for gaps, capped at _max_backfill_days
_backfill_start := GREATEST(
COALESCE(_latest_snapshot + 1, CURRENT_DATE - _max_backfill_days),
CURRENT_DATE - _max_backfill_days
);
IF _latest_snapshot IS NULL THEN
RAISE NOTICE 'No existing snapshots found. Backfilling up to % days.', _max_backfill_days;
ELSIF _backfill_start > _latest_snapshot + 1 THEN
RAISE NOTICE 'Latest snapshot: %. Gap exceeds % day cap — backfilling from %. Use rebuild script for full history.',
_latest_snapshot, _max_backfill_days, _backfill_start;
ELSE
RAISE NOTICE 'Latest snapshot: %. Checking for gaps from %.', _latest_snapshot, _backfill_start;
END IF;
-- Process all dates that need snapshots:
-- 1. Gap fill: dates with orders/receivings but no snapshots (older than recent window)
-- 2. Recent recheck: last N days always reprocessed (picks up new orders, corrections)
FOR _target_date IN
SELECT d FROM (
-- Gap fill: find dates with activity but missing snapshots
SELECT activity_dates.d
FROM (
SELECT DISTINCT date::date AS d FROM public.orders
WHERE date::date >= _backfill_start AND date::date < CURRENT_DATE - _recent_recheck_days
UNION
SELECT DISTINCT received_date::date AS d FROM public.receivings
WHERE received_date::date >= _backfill_start AND received_date::date < CURRENT_DATE - _recent_recheck_days
) activity_dates
WHERE NOT EXISTS (
SELECT 1 FROM public.daily_product_snapshots dps WHERE dps.snapshot_date = activity_dates.d
)
UNION
-- Recent days: always reprocess
SELECT d::date
FROM generate_series(
(CURRENT_DATE - _recent_recheck_days)::timestamp,
CURRENT_DATE::timestamp,
'1 day'::interval
) d
) dates_to_process
ORDER BY d
LOOP
_days_processed := _days_processed + 1;
RAISE NOTICE 'Processing date: % [%/%]', _target_date, _days_processed,
_days_processed; -- count not known ahead of time, but shows progress
-- IMPORTANT: First delete any existing data for this date to prevent duplication
DELETE FROM public.daily_product_snapshots
@@ -90,7 +88,6 @@ BEGIN
COALESCE(
o.costeach, -- First use order-specific cost if available
get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost
p.landing_cost_price, -- Fallback to landing cost
p.cost_price -- Final fallback to current cost
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
@@ -128,7 +125,7 @@ BEGIN
SELECT
pid,
stock_quantity,
COALESCE(landing_cost_price, cost_price, 0.00) as effective_cost_price,
COALESCE(cost_price, 0.00) as effective_cost_price,
COALESCE(price, 0.00) as current_price,
COALESCE(regular_price, 0.00) as current_regular_price
FROM public.products
@@ -181,7 +178,7 @@ BEGIN
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit, -- Basic profit: Net Revenue - COGS
@@ -201,12 +198,18 @@ BEGIN
RAISE NOTICE 'Created % daily snapshot records for % with sales/receiving activity', _total_records, _target_date;
END LOOP;
-- Update the status table with the timestamp from the START of this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
IF _days_processed = 0 THEN
RAISE NOTICE 'No days need updating — all snapshot data is current.';
ELSE
RAISE NOTICE 'Processed % days total.', _days_processed;
END IF;
RAISE NOTICE 'Finished % processing for multiple dates. Duration: %', _module_name, clock_timestamp() - _start_time;
-- Update the status table with the timestamp from the START of this run
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
VALUES (_module_name, _start_time)
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
RAISE NOTICE 'Finished % script. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;

10
inventory-server/scripts/metrics-new/update_product_metrics.sql
View File

@@ -52,7 +52,7 @@ BEGIN
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
COALESCE(p.cost_price, 0.00) as current_effective_cost,
p.stock_quantity as current_stock,
p.created_at,
p.first_received,
@@ -321,10 +321,10 @@ BEGIN
(GREATEST(0, ci.historical_total_sold - COALESCE(lr.lifetime_units_from_orders, 0)) *
COALESCE(
-- Use oldest known price from snapshots as proxy
(SELECT revenue_7d / NULLIF(sales_7d, 0)
FROM daily_product_snapshots
WHERE pid = ci.pid AND sales_7d > 0
ORDER BY snapshot_date ASC
(SELECT net_revenue / NULLIF(units_sold, 0)
FROM daily_product_snapshots
WHERE pid = ci.pid AND units_sold > 0
ORDER BY snapshot_date ASC
LIMIT 1),
ci.current_price
))