matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import/calculations improvements

Browse Source
This commit is contained in:
Matt
2026-06-11 19:32:20 -04:00
parent 3b2f51e6b8
commit 069a44bd54
19 changed files with 1175 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inventory-server/scripts/metrics-new/update_daily_snapshots.sql
+56 -28
View File
@@ -39,50 +39,68 @@ BEGIN
-- 2. Stale detection: existing snapshots where aggregates don't match source data
-- (catches backfilled imports that arrived after snapshot was calculated)
-- 3. Recent recheck: last N days always reprocessed (picks up new orders, corrections)
-- NOTE: all order/receiving timestamps are bucketed into business days using
-- America/Chicago. The PG server timezone is Europe/Berlin, so a bare ::date
-- cast would shift every evening order onto the next day.
FOR _target_date IN
SELECT d FROM (
-- Gap fill: find dates with activity but missing snapshots
SELECT activity_dates.d
FROM (
SELECT DISTINCT date::date AS d FROM public.orders
WHERE date::date >= _backfill_start AND date::date < CURRENT_DATE - _recent_recheck_days
SELECT DISTINCT (date AT TIME ZONE 'America/Chicago')::date AS d FROM public.orders
WHERE (date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
UNION
SELECT DISTINCT received_date::date AS d FROM public.receivings
WHERE received_date::date >= _backfill_start AND received_date::date < CURRENT_DATE - _recent_recheck_days
SELECT DISTINCT (received_date AT TIME ZONE 'America/Chicago')::date AS d FROM public.receivings
WHERE (received_date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (received_date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
) activity_dates
WHERE NOT EXISTS (
SELECT 1 FROM public.daily_product_snapshots dps WHERE dps.snapshot_date = activity_dates.d
)
UNION
-- Stale detection: compare snapshot aggregates against source tables
-- (must bucket identically to SalesData/ReceivingData or every day
-- looks permanently stale)
SELECT snap_agg.snapshot_date AS d
FROM (
SELECT snapshot_date,
COALESCE(SUM(units_received), 0)::bigint AS snap_received,
COALESCE(SUM(units_sold), 0)::bigint AS snap_sold
COALESCE(SUM(units_sold), 0)::bigint AS snap_sold,
ROUND(COALESCE(SUM(net_revenue), 0), 2) AS snap_net_revenue
FROM public.daily_product_snapshots
WHERE snapshot_date >= _backfill_start
AND snapshot_date < CURRENT_DATE - _recent_recheck_days
GROUP BY snapshot_date
) snap_agg
LEFT JOIN (
SELECT received_date::date AS d, SUM(qty_each)::bigint AS actual_received
SELECT (received_date AT TIME ZONE 'America/Chicago')::date AS d, SUM(qty_each)::bigint AS actual_received
FROM public.receivings
WHERE received_date::date >= _backfill_start
AND received_date::date < CURRENT_DATE - _recent_recheck_days
GROUP BY received_date::date
WHERE (received_date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (received_date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
GROUP BY 1
) recv_agg ON snap_agg.snapshot_date = recv_agg.d
LEFT JOIN (
SELECT date::date AS d,
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned')
THEN quantity ELSE 0 END)::bigint AS actual_sold
SELECT (date AT TIME ZONE 'America/Chicago')::date AS d,
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned', 'combined')
THEN quantity ELSE 0 END)::bigint AS actual_sold,
-- Mirrors SalesData's net_revenue (gross - discounts - returns)
-- so price/discount corrections older than the recheck window
-- get repaired, not just unit-count changes.
ROUND(
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned', 'combined')
THEN price * quantity - discount ELSE 0 END)
- SUM(CASE WHEN quantity < 0 OR COALESCE(status, 'pending') = 'returned'
THEN price * ABS(quantity) ELSE 0 END)
, 2) AS actual_net_revenue
FROM public.orders
WHERE date::date >= _backfill_start
AND date::date < CURRENT_DATE - _recent_recheck_days
GROUP BY date::date
WHERE (date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
GROUP BY 1
) orders_agg ON snap_agg.snapshot_date = orders_agg.d
WHERE snap_agg.snap_received != COALESCE(recv_agg.actual_received, 0)
OR snap_agg.snap_sold != COALESCE(orders_agg.actual_sold, 0)
OR snap_agg.snap_net_revenue != ROUND(COALESCE(orders_agg.actual_net_revenue, 0), 2)
UNION
-- Recent days: always reprocess
SELECT d::date
@@ -116,26 +134,36 @@ BEGIN
p.sku,
-- Track number of orders to ensure we have real data
COUNT(o.id) as order_count,
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, -- Before discount
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned/Combined)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, -- Before discount
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN
COALESCE(
o.costeach, -- First use order-specific cost if available
get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date), -- Then use weighted average cost
p.cost_price -- Final fallback to current cost
) * o.quantity
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Use current regular price for simplicity here
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Use current regular price for simplicity here
-- Aggregate Returns (Quantity < 0 or Status = Returned)
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN ABS(o.quantity) ELSE 0 END), 0) AS units_returned,
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue,
-- Returns COGS: returned goods come back into stock, so their cost
-- offsets the sales COGS for the day (margin would otherwise be
-- understated in return-heavy periods).
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN
COALESCE(
o.costeach,
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date),
p.cost_price
) * ABS(o.quantity)
ELSE 0 END), 0.00) AS returns_cogs
FROM public.products p -- Start from products to include those with no orders today
JOIN public.orders o -- Changed to INNER JOIN to only process products with orders
ON p.pid = o.pid
AND o.date::date = _target_date -- Cast to date to ensure compatibility regardless of original type
AND (o.date AT TIME ZONE 'America/Chicago')::date = _target_date -- Bucket by business day (Central)
GROUP BY p.pid, p.sku
-- No HAVING clause here - we always want to include all orders
),
@@ -149,7 +177,7 @@ BEGIN
-- Calculate the cost received (qty * cost)
SUM(r.qty_each * r.cost_each) AS cost_received
FROM public.receivings r
WHERE r.received_date::date = _target_date
WHERE (r.received_date AT TIME ZONE 'America/Chicago')::date = _target_date
-- Optional: Filter out canceled receivings if needed
-- AND r.status <> 'canceled'
GROUP BY r.pid
@@ -217,9 +245,9 @@ BEGIN
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00) AS cogs, -- net of returned goods' cost
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - (COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00)) AS profit,
-- Receiving Metrics (From ReceivingData)
COALESCE(rd.units_received, 0),
COALESCE(rd.cost_received, 0.00),