matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes for metrics calculations

Browse Source
This commit is contained in:
Matt
2026-02-07 21:34:42 -05:00
parent 9b2f9016f6
commit 12cc7a4639
18 changed files with 267 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
inventory-server/scripts/import/orders.js
View File

@@ -17,6 +17,33 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
const startTime = Date.now();
const skippedOrders = new Set();
const missingProducts = new Set();
// Map order status codes to text values (consistent with PO status mapping in purchase-orders.js)
const orderStatusMap = {
0: 'created',
10: 'unfinished',
15: 'canceled',
16: 'combined',
20: 'placed',
22: 'placed_incomplete',
30: 'canceled',
40: 'awaiting_payment',
50: 'awaiting_products',
55: 'shipping_later',
56: 'shipping_together',
60: 'ready',
61: 'flagged',
62: 'fix_before_pick',
65: 'manual_picking',
70: 'in_pt',
80: 'picked',
90: 'awaiting_shipment',
91: 'remote_wait',
92: 'awaiting_pickup',
93: 'fix_before_ship',
95: 'shipped_confirmed',
100: 'shipped'
};
let recordsAdded = 0;
let recordsUpdated = 0;
let processedCount = 0;
@@ -284,7 +311,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
new Date(order.date), // Convert to TIMESTAMP WITH TIME ZONE
order.customer,
toTitleCase(order.customer_name) || '',
order.status.toString(), // Convert status to TEXT
orderStatusMap[order.status] || order.status.toString(), // Map numeric status to text
order.canceled,
order.summary_discount || 0,
order.summary_subtotal || 0,
@@ -587,17 +614,14 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
oi.price,
oi.quantity,
(
-- Part 1: Sale Savings for the Line
(oi.base_discount * oi.quantity)
+
-- Part 2: Prorated Points Discount (if applicable)
-- Prorated Points Discount (e.g. loyalty points applied at order level)
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
END
+
-- Part 3: Specific Item-Level Discount (only if parent discount affected subtotal)
-- Specific Item-Level Promo Discount (coupon codes, etc.)
COALESCE(ot.promo_discount_sum, 0)
)::NUMERIC(14, 4) as discount,
COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
@@ -654,7 +678,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
o.shipping,
o.customer,
o.customer_name,
o.status.toString(), // Convert status to TEXT
o.status, // Already mapped to text via orderStatusMap
o.canceled,
o.costeach
]);

27
inventory-server/scripts/import/products.js
View File

@@ -77,7 +77,6 @@ async function setupTemporaryTables(connection) {
created_at TIMESTAMP WITH TIME ZONE,
date_online TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
landing_cost_price NUMERIC(14, 4),
barcode TEXT,
harmonized_tariff_code TEXT,
updated_at TIMESTAMP WITH TIME ZONE,
@@ -172,7 +171,6 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
@@ -242,8 +240,8 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
const batch = prodData.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 49; // 49 columns
return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -270,7 +268,6 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
row.harmonized_tariff_code,
validateDate(row.updated_at),
@@ -308,7 +305,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
@@ -382,7 +379,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
@@ -457,8 +453,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen
await withRetry(async () => {
const placeholders = batch.map((_, idx) => {
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 49; // 49 columns
return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -485,7 +481,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
row.harmonized_tariff_code,
validateDate(row.updated_at),
@@ -522,7 +517,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
@@ -547,7 +542,6 @@ async function materializeCalculations(prodConnection, localConnection, incremen
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
updated_at = EXCLUDED.updated_at,
@@ -702,7 +696,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
t.created_at,
t.date_online,
t.first_received,
t.landing_cost_price,
t.barcode,
t.harmonized_tariff_code,
t.updated_at,
@@ -742,8 +735,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
const batch = products.rows.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 49; // 49 columns
return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 48; // 48 columns (no primary_iid in this INSERT)
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -770,7 +763,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
validateDate(row.created_at),
validateDate(row.date_online),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
row.harmonized_tariff_code,
validateDate(row.updated_at),
@@ -807,7 +799,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, shop_score, image, image_175, image_full, options, tags
@@ -833,7 +825,6 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
updated_at = EXCLUDED.updated_at,

2
inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql
View File

@@ -27,7 +27,7 @@ BEGIN
p.visible as is_visible, p.replenishable,
COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
COALESCE(p.cost_price, 0.00) as current_effective_cost,
p.stock_quantity as current_stock, -- Use actual current stock for forecast base
p.created_at, p.first_received, p.date_last_sold,
p.moq,

8
inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
View File

@@ -10,7 +10,7 @@ DECLARE
_date DATE;
_count INT;
_total_records INT := 0;
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2024-01-01'); -- Starting point for data rebuild
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data
_end_date DATE := CURRENT_DATE;
BEGIN
RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;
@@ -36,7 +36,7 @@ BEGIN
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.landing_cost_price, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,
-- Aggregate Returns (Quantity < 0 or Status = Returned)
@@ -68,7 +68,7 @@ BEGIN
SELECT
p.pid,
p.stock_quantity,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as effective_cost_price,
COALESCE(p.cost_price, 0.00) as effective_cost_price,
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price
FROM public.products p
@@ -111,7 +111,7 @@ BEGIN
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,

4
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
View File

@@ -28,8 +28,8 @@ BEGIN
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,

96
inventory-server/scripts/metrics-new/calculate_category_metrics.sql
View File

@@ -28,8 +28,8 @@ BEGIN
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
@@ -38,58 +38,56 @@ BEGIN
JOIN public.product_metrics pm ON pc.pid = pm.pid
GROUP BY pc.cat_id
),
-- Calculate rolled-up metrics (including all descendant categories)
-- Map each category to ALL distinct products in it or any descendant.
-- Uses the path array from category_hierarchy: for product P in category C,
-- P contributes to C and every ancestor in C's path.
-- DISTINCT ensures each (ancestor, pid) pair appears only once, preventing
-- double-counting when a product belongs to multiple categories under the same parent.
CategoryProducts AS (
SELECT DISTINCT
ancestor_cat_id,
pc.pid
FROM public.product_categories pc
JOIN category_hierarchy ch ON pc.cat_id = ch.cat_id
CROSS JOIN LATERAL unnest(ch.path) AS ancestor_cat_id
),
-- Calculate rolled-up metrics using deduplicated product sets
RolledUpMetrics AS (
SELECT
ch.cat_id,
-- Sum metrics from this category and all its descendants
SUM(dcm.product_count) AS product_count,
SUM(dcm.active_product_count) AS active_product_count,
SUM(dcm.replenishable_product_count) AS replenishable_product_count,
SUM(dcm.current_stock_units) AS current_stock_units,
SUM(dcm.current_stock_cost) AS current_stock_cost,
SUM(dcm.current_stock_retail) AS current_stock_retail,
SUM(dcm.sales_7d) AS sales_7d,
SUM(dcm.revenue_7d) AS revenue_7d,
SUM(dcm.sales_30d) AS sales_30d,
SUM(dcm.revenue_30d) AS revenue_30d,
SUM(dcm.cogs_30d) AS cogs_30d,
SUM(dcm.profit_30d) AS profit_30d,
SUM(dcm.sales_365d) AS sales_365d,
SUM(dcm.revenue_365d) AS revenue_365d,
SUM(dcm.lifetime_sales) AS lifetime_sales,
SUM(dcm.lifetime_revenue) AS lifetime_revenue
FROM category_hierarchy ch
LEFT JOIN DirectCategoryMetrics dcm ON
dcm.cat_id = ch.cat_id OR
dcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
),
PreviousPeriodCategoryMetrics AS (
-- Get previous period metrics for growth calculation
SELECT
pc.cat_id,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM public.daily_product_snapshots dps
JOIN public.product_categories pc ON dps.pid = pc.pid
GROUP BY pc.cat_id
cp.ancestor_cat_id AS cat_id,
COUNT(DISTINCT cp.pid) AS product_count,
COUNT(DISTINCT CASE WHEN pm.is_visible THEN cp.pid END) AS active_product_count,
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN cp.pid END) AS replenishable_product_count,
SUM(pm.current_stock) AS current_stock_units,
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
FROM CategoryProducts cp
JOIN public.product_metrics pm ON cp.pid = pm.pid
GROUP BY cp.ancestor_cat_id
),
-- Previous period rolled up using same deduplicated product sets
RolledUpPreviousPeriod AS (
-- Calculate rolled-up previous period metrics
SELECT
ch.cat_id,
SUM(ppcm.sales_prev_30d) AS sales_prev_30d,
SUM(ppcm.revenue_prev_30d) AS revenue_prev_30d
FROM category_hierarchy ch
LEFT JOIN PreviousPeriodCategoryMetrics ppcm ON
ppcm.cat_id = ch.cat_id OR
ppcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
cp.ancestor_cat_id AS cat_id,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM CategoryProducts cp
JOIN public.daily_product_snapshots dps ON cp.pid = dps.pid
GROUP BY cp.ancestor_cat_id
),
AllCategories AS (
-- Ensure all categories are included

6
inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql
View File

@@ -29,8 +29,8 @@ BEGIN
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
@@ -72,7 +72,7 @@ BEGIN
END))::int AS avg_lead_time_days_hist -- Avg lead time from HISTORICAL received POs
FROM public.purchase_orders po
-- Join to receivings table to find when items were received
LEFT JOIN public.receivings r ON r.pid = po.pid
LEFT JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id
WHERE po.vendor IS NOT NULL AND po.vendor <> ''
AND po.date >= CURRENT_DATE - INTERVAL '1 year' -- Look at POs created in the last year
AND po.status = 'done' -- Only calculate lead time on completed POs

38
inventory-server/scripts/metrics-new/migrations/001_map_order_statuses.sql Normal file
View File

@@ -0,0 +1,38 @@
-- Migration: Map existing numeric order statuses to text values
-- Run this ONCE on the production PostgreSQL database after deploying the updated orders import.
-- This updates ~2.88M rows. On a busy system, consider running during low-traffic hours.
-- The WHERE clause ensures idempotency - only rows with numeric statuses are updated.
UPDATE orders SET status = CASE status
WHEN '0' THEN 'created'
WHEN '10' THEN 'unfinished'
WHEN '15' THEN 'canceled'
WHEN '16' THEN 'combined'
WHEN '20' THEN 'placed'
WHEN '22' THEN 'placed_incomplete'
WHEN '30' THEN 'canceled'
WHEN '40' THEN 'awaiting_payment'
WHEN '50' THEN 'awaiting_products'
WHEN '55' THEN 'shipping_later'
WHEN '56' THEN 'shipping_together'
WHEN '60' THEN 'ready'
WHEN '61' THEN 'flagged'
WHEN '62' THEN 'fix_before_pick'
WHEN '65' THEN 'manual_picking'
WHEN '70' THEN 'in_pt'
WHEN '80' THEN 'picked'
WHEN '90' THEN 'awaiting_shipment'
WHEN '91' THEN 'remote_wait'
WHEN '92' THEN 'awaiting_pickup'
WHEN '93' THEN 'fix_before_ship'
WHEN '95' THEN 'shipped_confirmed'
WHEN '100' THEN 'shipped'
ELSE status
END
WHERE status ~ '^\d+$'; -- Only update rows that still have numeric statuses
-- Verify the migration
SELECT status, COUNT(*) as count
FROM orders
GROUP BY status
ORDER BY count DESC;

51
inventory-server/scripts/metrics-new/migrations/002_fix_discount_double_counting.sql Normal file
View File

@@ -0,0 +1,51 @@
-- Migration 002: Fix discount double-counting in orders
--
-- PROBLEM: The orders import was calculating discount as:
-- discount = (prod_price_reg - prod_price) * quantity <-- "sale savings" (WRONG)
-- + prorated points discount
-- + item-level promo discounts
--
-- Since `price` in the orders table already IS the sale price (prod_price, not prod_price_reg),
-- the "sale savings" component double-counted the markdown. This resulted in inflated discounts
-- and near-zero net_revenue for products sold on sale.
--
-- Example: Product with regular_price=$30, sale_price=$15, qty=2
-- BEFORE (buggy): discount = ($30-$15)*2 + 0 + 0 = $30.00
-- net_revenue = $15*2 - $30 = $0.00 (WRONG!)
-- AFTER (fixed): discount = 0 + 0 + 0 = $0.00
-- net_revenue = $15*2 - $0 = $30.00 (CORRECT!)
--
-- FIX: This cannot be fixed with a pure SQL migration because PostgreSQL doesn't store
-- prod_price_reg. The discount column has the inflated value baked in, and we can't
-- decompose which portion was the base_discount vs actual promo discounts.
--
-- REQUIRED ACTION: Run a FULL (non-incremental) orders re-import after deploying the
-- fixed orders.js. This will recalculate all discounts using the corrected formula.
--
-- Steps:
-- 1. Deploy updated orders.js (base_discount removed from discount calculation)
-- 2. Run: node scripts/import/orders.js --full
-- (or trigger a full sync through whatever mechanism is used)
-- 3. After re-import, run the daily snapshots rebuild to propagate corrected revenue:
-- psql -f scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
-- 4. Re-run metrics calculation:
-- node scripts/metrics-new/calculate-metrics-new.js
--
-- VERIFICATION: After re-import, check the previously-affected products:
SELECT
o.pid,
p.title,
o.order_number,
o.price,
o.quantity,
o.discount,
(o.price * o.quantity) as gross_revenue,
(o.price * o.quantity - o.discount) as net_revenue
FROM orders o
JOIN products p ON o.pid = p.pid
WHERE o.pid IN (624756, 614513)
ORDER BY o.date DESC
LIMIT 10;
-- Expected: discount should be 0 (or small promo amount) for regular sales,
-- and net_revenue should be close to gross_revenue.

143
inventory-server/scripts/metrics-new/update_daily_snapshots.sql
View File

@@ -1,75 +1,73 @@
-- Description: Calculates and updates daily aggregated product data for recent days.
-- Uses UPSERT (INSERT ON CONFLICT UPDATE) for idempotency.
-- Description: Calculates and updates daily aggregated product data.
-- Self-healing: automatically detects and fills gaps in snapshot history.
-- Always reprocesses recent days to pick up new orders and data corrections.
-- Dependencies: Core import tables (products, orders, purchase_orders), calculate_status table.
-- Frequency: Hourly (Run ~5-10 minutes after hourly data import completes).
DO $$
DECLARE
_module_name TEXT := 'daily_snapshots';
_start_time TIMESTAMPTZ := clock_timestamp(); -- Time execution started
_last_calc_time TIMESTAMPTZ;
_target_date DATE; -- Will be set in the loop
_start_time TIMESTAMPTZ := clock_timestamp();
_target_date DATE;
_total_records INT := 0;
_has_orders BOOLEAN := FALSE;
_process_days INT := 5; -- Number of days to check/process (today plus previous 4 days)
_day_counter INT;
_missing_days INT[] := ARRAY[]::INT[]; -- Array to store days with missing or incomplete data
_days_processed INT := 0;
_max_backfill_days INT := 90; -- Safety cap: max days to backfill per run
_recent_recheck_days INT := 2; -- Always reprocess this many recent days (today + yesterday)
_latest_snapshot DATE;
_backfill_start DATE;
BEGIN
-- Get the timestamp before the last successful run of this module
SELECT last_calculation_timestamp INTO _last_calc_time
FROM public.calculate_status
WHERE module_name = _module_name;
RAISE NOTICE 'Running % script. Start Time: %', _module_name, _start_time;
-- First, check which days need processing by comparing orders data with snapshot data
FOR _day_counter IN 0..(_process_days-1) LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
-- Check if this date needs updating by comparing orders to snapshot data
-- If the date has orders but not enough snapshots, or if snapshots show zero sales but orders exist, it's incomplete
SELECT
CASE WHEN (
-- We have orders for this date but not enough snapshots, or snapshots with wrong total
(EXISTS (SELECT 1 FROM public.orders WHERE date::date = _target_date) AND
(
-- No snapshots exist for this date
NOT EXISTS (SELECT 1 FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) OR
-- Or snapshots show zero sales but orders exist
(SELECT COALESCE(SUM(units_sold), 0) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) = 0 OR
-- Or the count of snapshot records is significantly less than distinct products in orders
(SELECT COUNT(*) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) <
(SELECT COUNT(DISTINCT pid) FROM public.orders WHERE date::date = _target_date) * 0.8
)
)
) THEN TRUE ELSE FALSE END
INTO _has_orders;
IF _has_orders THEN
-- This day needs processing - add to our array
_missing_days := _missing_days || _day_counter;
RAISE NOTICE 'Day % needs updating (incomplete or missing data)', _target_date;
END IF;
END LOOP;
-- If no days need updating, exit early
IF array_length(_missing_days, 1) IS NULL THEN
RAISE NOTICE 'No days need updating - all snapshot data appears complete';
-- Still update the calculate_status to record this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RETURN;
END IF;
RAISE NOTICE 'Need to update % days with missing or incomplete data', array_length(_missing_days, 1);
-- Process only the days that need updating
FOREACH _day_counter IN ARRAY _missing_days LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
RAISE NOTICE 'Processing date: %', _target_date;
-- Find the latest existing snapshot date to determine where gaps begin
SELECT MAX(snapshot_date) INTO _latest_snapshot
FROM public.daily_product_snapshots;
-- Determine how far back to look for gaps, capped at _max_backfill_days
_backfill_start := GREATEST(
COALESCE(_latest_snapshot + 1, CURRENT_DATE - _max_backfill_days),
CURRENT_DATE - _max_backfill_days
);
IF _latest_snapshot IS NULL THEN
RAISE NOTICE 'No existing snapshots found. Backfilling up to % days.', _max_backfill_days;
ELSIF _backfill_start > _latest_snapshot + 1 THEN
RAISE NOTICE 'Latest snapshot: %. Gap exceeds % day cap — backfilling from %. Use rebuild script for full history.',
_latest_snapshot, _max_backfill_days, _backfill_start;
ELSE
RAISE NOTICE 'Latest snapshot: %. Checking for gaps from %.', _latest_snapshot, _backfill_start;
END IF;
-- Process all dates that need snapshots:
-- 1. Gap fill: dates with orders/receivings but no snapshots (older than recent window)
-- 2. Recent recheck: last N days always reprocessed (picks up new orders, corrections)
FOR _target_date IN
SELECT d FROM (
-- Gap fill: find dates with activity but missing snapshots
SELECT activity_dates.d
FROM (
SELECT DISTINCT date::date AS d FROM public.orders
WHERE date::date >= _backfill_start AND date::date < CURRENT_DATE - _recent_recheck_days
UNION
SELECT DISTINCT received_date::date AS d FROM public.receivings
WHERE received_date::date >= _backfill_start AND received_date::date < CURRENT_DATE - _recent_recheck_days
) activity_dates
WHERE NOT EXISTS (
SELECT 1 FROM public.daily_product_snapshots dps WHERE dps.snapshot_date = activity_dates.d
)
UNION
-- Recent days: always reprocess
SELECT d::date
FROM generate_series(
(CURRENT_DATE - _recent_recheck_days)::timestamp,
CURRENT_DATE::timestamp,
'1 day'::interval
) d
) dates_to_process
ORDER BY d
LOOP
_days_processed := _days_processed + 1;
RAISE NOTICE 'Processing date: % [%/%]', _target_date, _days_processed,
_days_processed; -- count not known ahead of time, but shows progress
-- IMPORTANT: First delete any existing data for this date to prevent duplication
DELETE FROM public.daily_product_snapshots
@@ -90,7 +88,6 @@ BEGIN
COALESCE(
o.costeach, -- First use order-specific cost if available
get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost
p.landing_cost_price, -- Fallback to landing cost
p.cost_price -- Final fallback to current cost
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
@@ -128,7 +125,7 @@ BEGIN
SELECT
pid,
stock_quantity,
COALESCE(landing_cost_price, cost_price, 0.00) as effective_cost_price,
COALESCE(cost_price, 0.00) as effective_cost_price,
COALESCE(price, 0.00) as current_price,
COALESCE(regular_price, 0.00) as current_regular_price
FROM public.products
@@ -181,7 +178,7 @@ BEGIN
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit, -- Basic profit: Net Revenue - COGS
@@ -201,12 +198,18 @@ BEGIN
RAISE NOTICE 'Created % daily snapshot records for % with sales/receiving activity', _total_records, _target_date;
END LOOP;
-- Update the status table with the timestamp from the START of this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
IF _days_processed = 0 THEN
RAISE NOTICE 'No days need updating — all snapshot data is current.';
ELSE
RAISE NOTICE 'Processed % days total.', _days_processed;
END IF;
RAISE NOTICE 'Finished % processing for multiple dates. Duration: %', _module_name, clock_timestamp() - _start_time;
-- Update the status table with the timestamp from the START of this run
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
VALUES (_module_name, _start_time)
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
RAISE NOTICE 'Finished % script. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;

10
inventory-server/scripts/metrics-new/update_product_metrics.sql
View File

@@ -52,7 +52,7 @@ BEGIN
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
COALESCE(p.cost_price, 0.00) as current_effective_cost,
p.stock_quantity as current_stock,
p.created_at,
p.first_received,
@@ -321,10 +321,10 @@ BEGIN
(GREATEST(0, ci.historical_total_sold - COALESCE(lr.lifetime_units_from_orders, 0)) *
COALESCE(
-- Use oldest known price from snapshots as proxy
(SELECT revenue_7d / NULLIF(sales_7d, 0)
FROM daily_product_snapshots
WHERE pid = ci.pid AND sales_7d > 0
ORDER BY snapshot_date ASC
(SELECT net_revenue / NULLIF(units_sold, 0)
FROM daily_product_snapshots
WHERE pid = ci.pid AND units_sold > 0
ORDER BY snapshot_date ASC
LIMIT 1),
ci.current_price
))

3
inventory-server/src/routes/metrics.js
View File

@@ -43,7 +43,6 @@ const COLUMN_MAP = {
currentPrice: 'pm.current_price',
currentRegularPrice: 'pm.current_regular_price',
currentCostPrice: 'pm.current_cost_price',
currentLandingCostPrice: 'pm.current_landing_cost_price',
currentStock: 'pm.current_stock',
currentStockCost: 'pm.current_stock_cost',
currentStockRetail: 'pm.current_stock_retail',
@@ -176,7 +175,7 @@ const COLUMN_MAP = {
const COLUMN_TYPES = {
// Numeric columns (use numeric operators and sorting)
numeric: [
'pid', 'currentPrice', 'currentRegularPrice', 'currentCostPrice', 'currentLandingCostPrice',
'pid', 'currentPrice', 'currentRegularPrice', 'currentCostPrice',
'currentStock', 'currentStockCost', 'currentStockRetail', 'currentStockGross',
'onOrderQty', 'onOrderCost', 'onOrderRetail', 'ageDays',
'sales7d', 'revenue7d', 'sales14d', 'revenue14d', 'sales30d', 'revenue30d',

2
inventory-server/src/routes/products.js
View File

@@ -145,7 +145,6 @@ router.get('/', async (req, res) => {
stock: 'p.stock_quantity',
price: 'p.price',
costPrice: 'p.cost_price',
landingCost: 'p.landing_cost_price',
dailySalesAvg: 'pm.daily_sales_avg',
weeklySalesAvg: 'pm.weekly_sales_avg',
monthlySalesAvg: 'pm.monthly_sales_avg',
@@ -621,7 +620,6 @@ router.get('/:id', async (req, res) => {
price: parseFloat(productRows[0].price),
regular_price: parseFloat(productRows[0].regular_price),
cost_price: parseFloat(productRows[0].cost_price),
landing_cost_price: parseFloat(productRows[0].landing_cost_price),
stock_quantity: parseInt(productRows[0].stock_quantity),
moq: parseInt(productRows[0].moq),
uom: parseInt(productRows[0].uom),