matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UI tweaks for match columns step + auto hide empty columns

Browse Source
This commit is contained in:
Matt
2025-10-04 09:48:14 -04:00
parent dadcf3b6c6
commit 4953355b91
178 changed files with 580 additions and 61798 deletions
Download Patch File Download Diff File Expand all files Collapse all files

444
inventory-server/scripts/metrics-new/backfill/populate_initial_product_metrics.sql
View File

@@ -1,444 +0,0 @@
-- Description: Performs the first population OR full recalculation of the product_metrics table based on
-- historically backfilled daily_product_snapshots and current product/PO data.
-- Calculates all metrics considering the full available history up to 'yesterday'.
-- Run ONCE after backfill_historical_snapshots_final.sql completes successfully.
-- Dependencies: Core import tables (products, purchase_orders, receivings), daily_product_snapshots (historically populated),
-- configuration tables (settings_*), product_metrics table must exist.
-- Frequency: Run ONCE.
DO $$
DECLARE
_module_name VARCHAR := 'product_metrics_population'; -- Generic name
_start_time TIMESTAMPTZ := clock_timestamp();
-- Calculate metrics AS OF the end of the last fully completed day
_calculation_date DATE := CURRENT_DATE - INTERVAL '1 day';
BEGIN
RAISE NOTICE 'Running % module. Calculating AS OF: %. Start Time: %', _module_name, _calculation_date, _start_time;
-- Optional: Consider TRUNCATE if you want a completely fresh start,
-- otherwise ON CONFLICT will update existing rows if this is rerun.
-- TRUNCATE TABLE public.product_metrics;
RAISE NOTICE 'Populating product_metrics table. This may take some time...';
-- CTEs to gather necessary information AS OF _calculation_date
WITH CurrentInfo AS (
-- Fetches current product details, including costs/prices used for forecasting & fallbacks
SELECT
p.pid, p.sku, p.title, p.brand, p.vendor, COALESCE(p.image_175, p.image) as image_url,
p.visible as is_visible, p.replenishable,
COALESCE(p.price, 0.00) as current_price, COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
p.stock_quantity as current_stock, -- Use actual current stock for forecast base
p.created_at, p.first_received, p.date_last_sold,
p.moq,
p.uom,
p.total_sold as historical_total_sold -- Add historical total_sold from products table
FROM public.products p
),
OnOrderInfo AS (
-- Calculates current on-order quantities and costs
SELECT
pid,
SUM(ordered) AS on_order_qty,
SUM(ordered * po_cost_price) AS on_order_cost,
MIN(expected_date) AS earliest_expected_date
FROM public.purchase_orders
-- Use the most common statuses representing active, unfulfilled POs
WHERE status IN ('created', 'ordered', 'preordered', 'electronically_sent', 'electronically_ready_send', 'receiving_started')
AND status NOT IN ('canceled', 'done')
GROUP BY pid
),
HistoricalDates AS (
-- Determines key historical dates from orders and receivings
SELECT
p.pid,
MIN(o.date)::date AS date_first_sold,
MAX(o.date)::date AS max_order_date, -- Used as fallback for date_last_sold
MIN(r.received_date)::date AS date_first_received_calc,
MAX(r.received_date)::date AS date_last_received_calc
FROM public.products p
LEFT JOIN public.orders o ON p.pid = o.pid AND o.quantity > 0 AND o.status NOT IN ('canceled', 'returned')
LEFT JOIN public.receivings r ON p.pid = r.pid
GROUP BY p.pid
),
SnapshotAggregates AS (
-- Aggregates metrics from historical snapshots up to the _calculation_date
SELECT
pid,
-- Rolling periods relative to _calculation_date
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '6 days' AND _calculation_date THEN units_sold ELSE 0 END) AS sales_7d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '6 days' AND _calculation_date THEN net_revenue ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '13 days' AND _calculation_date THEN units_sold ELSE 0 END) AS sales_14d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '13 days' AND _calculation_date THEN net_revenue ELSE 0 END) AS revenue_14d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN units_sold ELSE 0 END) AS sales_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN net_revenue ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN cogs ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN profit ELSE 0 END) AS profit_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN units_returned ELSE 0 END) AS returns_units_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN returns_revenue ELSE 0 END) AS returns_revenue_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN discounts ELSE 0 END) AS discounts_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN gross_revenue ELSE 0 END) AS gross_revenue_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN gross_regular_revenue ELSE 0 END) AS gross_regular_revenue_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date AND stockout_flag THEN 1 ELSE 0 END) AS stockout_days_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '364 days' AND _calculation_date THEN units_sold ELSE 0 END) AS sales_365d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '364 days' AND _calculation_date THEN net_revenue ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN units_received ELSE 0 END) AS received_qty_30d,
SUM(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN cost_received ELSE 0 END) AS received_cost_30d,
-- Averages over the last 30 days ending _calculation_date
AVG(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN eod_stock_quantity END) AS avg_stock_units_30d,
AVG(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN eod_stock_cost END) AS avg_stock_cost_30d,
AVG(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN eod_stock_retail END) AS avg_stock_retail_30d,
AVG(CASE WHEN snapshot_date BETWEEN _calculation_date - INTERVAL '29 days' AND _calculation_date THEN eod_stock_gross END) AS avg_stock_gross_30d,
-- Lifetime (Using historical total from products table)
(SELECT total_sold FROM public.products WHERE public.products.pid = daily_product_snapshots.pid) AS lifetime_sales,
COALESCE(
-- Option 1: Use 30-day average price if available
CASE WHEN SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '29 days' AND snapshot_date <= _calculation_date THEN units_sold ELSE 0 END) > 0 THEN
(SELECT total_sold FROM public.products WHERE public.products.pid = daily_product_snapshots.pid) * (
SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '29 days' AND snapshot_date <= _calculation_date THEN net_revenue ELSE 0 END) /
NULLIF(SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '29 days' AND snapshot_date <= _calculation_date THEN units_sold ELSE 0 END), 0)
)
ELSE NULL END,
-- Option 2: Try 365-day average price if available
CASE WHEN SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '364 days' AND snapshot_date <= _calculation_date THEN units_sold ELSE 0 END) > 0 THEN
(SELECT total_sold FROM public.products WHERE public.products.pid = daily_product_snapshots.pid) * (
SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '364 days' AND snapshot_date <= _calculation_date THEN net_revenue ELSE 0 END) /
NULLIF(SUM(CASE WHEN snapshot_date >= _calculation_date - INTERVAL '364 days' AND snapshot_date <= _calculation_date THEN units_sold ELSE 0 END), 0)
)
ELSE NULL END,
-- Option 3: Use current price from products table
(SELECT total_sold * price FROM public.products WHERE public.products.pid = daily_product_snapshots.pid),
-- Option 4: Use regular price if current price might be zero
(SELECT total_sold * regular_price FROM public.products WHERE public.products.pid = daily_product_snapshots.pid),
-- Final fallback: Use accumulated revenue (less accurate for old products)
SUM(net_revenue)
) AS lifetime_revenue,
-- Yesterday (Sales for the specific _calculation_date)
SUM(CASE WHEN snapshot_date = _calculation_date THEN units_sold ELSE 0 END) as yesterday_sales
FROM public.daily_product_snapshots
WHERE snapshot_date <= _calculation_date -- Ensure we only use data up to the calculation point
GROUP BY pid
),
FirstPeriodMetrics AS (
-- Calculates sales/revenue for first X days after first sale date
-- Uses HistoricalDates CTE to get the first sale date
SELECT
pid, date_first_sold,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '6 days' THEN units_sold ELSE 0 END) AS first_7_days_sales,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '6 days' THEN net_revenue ELSE 0 END) AS first_7_days_revenue,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '29 days' THEN units_sold ELSE 0 END) AS first_30_days_sales,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '29 days' THEN net_revenue ELSE 0 END) AS first_30_days_revenue,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '59 days' THEN units_sold ELSE 0 END) AS first_60_days_sales,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '59 days' THEN net_revenue ELSE 0 END) AS first_60_days_revenue,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '89 days' THEN units_sold ELSE 0 END) AS first_90_days_sales,
SUM(CASE WHEN snapshot_date BETWEEN date_first_sold AND date_first_sold + INTERVAL '89 days' THEN net_revenue ELSE 0 END) AS first_90_days_revenue
FROM public.daily_product_snapshots ds
JOIN HistoricalDates hd USING(pid)
WHERE date_first_sold IS NOT NULL
AND snapshot_date >= date_first_sold -- Only consider snapshots after first sale
AND snapshot_date <= _calculation_date -- Only up to the overall calculation date
GROUP BY pid, date_first_sold
),
Settings AS (
-- Fetches effective configuration settings (Product > Vendor > Global)
SELECT
p.pid,
COALESCE(sp.lead_time_days, sv.default_lead_time_days, (SELECT setting_value FROM settings_global WHERE setting_key = 'default_lead_time_days')::int, 14) AS effective_lead_time,
COALESCE(sp.days_of_stock, sv.default_days_of_stock, (SELECT setting_value FROM settings_global WHERE setting_key = 'default_days_of_stock')::int, 30) AS effective_days_of_stock,
COALESCE(sp.safety_stock, (SELECT setting_value::int FROM settings_global WHERE setting_key = 'default_safety_stock_units'), 0) AS effective_safety_stock,
COALESCE(sp.exclude_from_forecast, FALSE) AS exclude_forecast
FROM public.products p
LEFT JOIN public.settings_product sp ON p.pid = sp.pid
LEFT JOIN public.settings_vendor sv ON p.vendor = sv.vendor
),
AvgLeadTime AS (
-- Calculate Average Lead Time by joining purchase_orders with receivings
SELECT
po.pid,
AVG(GREATEST(1,
CASE
WHEN r.received_date IS NOT NULL AND po.date IS NOT NULL
THEN (r.received_date::date - po.date::date)
ELSE 1
END
))::int AS avg_lead_time_days_calc
FROM public.purchase_orders po
JOIN public.receivings r ON r.pid = po.pid
WHERE po.status = 'done' -- Completed POs
AND r.received_date IS NOT NULL
AND po.date IS NOT NULL
AND r.received_date >= po.date
GROUP BY po.pid
),
RankedForABC AS (
-- Ranks products based on the configured ABC metric (using historical data)
SELECT
p.pid,
CASE COALESCE((SELECT setting_value FROM settings_global WHERE setting_key = 'abc_calculation_basis'), 'revenue_30d')
WHEN 'sales_30d' THEN COALESCE(sa.sales_30d, 0)
WHEN 'lifetime_revenue' THEN COALESCE(sa.lifetime_revenue, 0)::numeric
ELSE COALESCE(sa.revenue_30d, 0) -- Default to revenue_30d
END AS metric_value
FROM public.products p -- Use products as the base
JOIN SnapshotAggregates sa ON p.pid = sa.pid
WHERE p.replenishable = TRUE -- Only rank replenishable products
AND (CASE COALESCE((SELECT setting_value FROM settings_global WHERE setting_key = 'abc_calculation_basis'), 'revenue_30d')
WHEN 'sales_30d' THEN COALESCE(sa.sales_30d, 0)
WHEN 'lifetime_revenue' THEN COALESCE(sa.lifetime_revenue, 0)::numeric
ELSE COALESCE(sa.revenue_30d, 0)
END) > 0 -- Only include products with non-zero contribution
),
CumulativeABC AS (
-- Calculates cumulative metric values for ABC ranking
SELECT
pid, metric_value,
SUM(metric_value) OVER (ORDER BY metric_value DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) as cumulative_metric,
SUM(metric_value) OVER () as total_metric
FROM RankedForABC
),
FinalABC AS (
-- Assigns A, B, or C class based on thresholds
SELECT
pid,
CASE
WHEN cumulative_metric / NULLIF(total_metric, 0) <= COALESCE((SELECT setting_value::numeric FROM settings_global WHERE setting_key = 'abc_revenue_threshold_a'), 0.8) THEN 'A'::char(1)
WHEN cumulative_metric / NULLIF(total_metric, 0) <= COALESCE((SELECT setting_value::numeric FROM settings_global WHERE setting_key = 'abc_revenue_threshold_b'), 0.95) THEN 'B'::char(1)
ELSE 'C'::char(1)
END AS abc_class_calc
FROM CumulativeABC
)
-- Final INSERT/UPDATE statement using all the prepared CTEs
INSERT INTO public.product_metrics (
pid, last_calculated, sku, title, brand, vendor, image_url, is_visible, is_replenishable,
current_price, current_regular_price, current_cost_price, current_landing_cost_price,
current_stock, current_stock_cost, current_stock_retail, current_stock_gross,
on_order_qty, on_order_cost, on_order_retail, earliest_expected_date,
date_created, date_first_received, date_last_received, date_first_sold, date_last_sold, age_days,
sales_7d, revenue_7d, sales_14d, revenue_14d, sales_30d, revenue_30d, cogs_30d, profit_30d,
returns_units_30d, returns_revenue_30d, discounts_30d, gross_revenue_30d, gross_regular_revenue_30d,
stockout_days_30d, sales_365d, revenue_365d,
avg_stock_units_30d, avg_stock_cost_30d, avg_stock_retail_30d, avg_stock_gross_30d,
received_qty_30d, received_cost_30d,
lifetime_sales, lifetime_revenue,
first_7_days_sales, first_7_days_revenue, first_30_days_sales, first_30_days_revenue,
first_60_days_sales, first_60_days_revenue, first_90_days_sales, first_90_days_revenue,
asp_30d, acp_30d, avg_ros_30d, avg_sales_per_day_30d,
margin_30d, markup_30d, gmroi_30d, stockturn_30d, return_rate_30d, discount_rate_30d,
stockout_rate_30d, markdown_30d, markdown_rate_30d, sell_through_30d,
avg_lead_time_days, abc_class,
sales_velocity_daily, config_lead_time, config_days_of_stock, config_safety_stock,
planning_period_days, lead_time_forecast_units, days_of_stock_forecast_units,
planning_period_forecast_units, lead_time_closing_stock, days_of_stock_closing_stock,
replenishment_needed_raw, replenishment_units, replenishment_cost, replenishment_retail, replenishment_profit,
to_order_units, forecast_lost_sales_units, forecast_lost_revenue,
stock_cover_in_days, po_cover_in_days, sells_out_in_days, replenish_date,
overstocked_units, overstocked_cost, overstocked_retail, is_old_stock,
yesterday_sales
)
SELECT
-- Select columns in order, joining all CTEs by pid
ci.pid, _start_time, ci.sku, ci.title, ci.brand, ci.vendor, ci.image_url, ci.is_visible, ci.replenishable,
ci.current_price, ci.current_regular_price, ci.current_cost_price, ci.current_effective_cost,
ci.current_stock, (ci.current_stock * COALESCE(ci.current_effective_cost, 0.00))::numeric(12,2), (ci.current_stock * COALESCE(ci.current_price, 0.00))::numeric(12,2), (ci.current_stock * COALESCE(ci.current_regular_price, 0.00))::numeric(12,2),
COALESCE(ooi.on_order_qty, 0), COALESCE(ooi.on_order_cost, 0.00)::numeric(12,2), (COALESCE(ooi.on_order_qty, 0) * COALESCE(ci.current_price, 0.00))::numeric(12,2), ooi.earliest_expected_date,
-- Fix type issue with date calculation - properly cast timestamps to dates before arithmetic
ci.created_at::date,
COALESCE(ci.first_received::date, hd.date_first_received_calc),
hd.date_last_received_calc,
hd.date_first_sold,
COALESCE(ci.date_last_sold, hd.max_order_date),
-- Fix timestamp + integer error by ensuring we work only with dates
CASE
WHEN LEAST(ci.created_at::date, COALESCE(hd.date_first_sold, ci.created_at::date)) IS NOT NULL
THEN (_calculation_date::date - LEAST(ci.created_at::date, COALESCE(hd.date_first_sold, ci.created_at::date)))::int
ELSE NULL
END,
COALESCE(sa.sales_7d, 0), COALESCE(sa.revenue_7d, 0), COALESCE(sa.sales_14d, 0), COALESCE(sa.revenue_14d, 0), COALESCE(sa.sales_30d, 0), COALESCE(sa.revenue_30d, 0), COALESCE(sa.cogs_30d, 0), COALESCE(sa.profit_30d, 0),
COALESCE(sa.returns_units_30d, 0), COALESCE(sa.returns_revenue_30d, 0), COALESCE(sa.discounts_30d, 0), COALESCE(sa.gross_revenue_30d, 0), COALESCE(sa.gross_regular_revenue_30d, 0),
COALESCE(sa.stockout_days_30d, 0), COALESCE(sa.sales_365d, 0), COALESCE(sa.revenue_365d, 0),
sa.avg_stock_units_30d, sa.avg_stock_cost_30d, sa.avg_stock_retail_30d, sa.avg_stock_gross_30d, -- Averages can be NULL if no data
COALESCE(sa.received_qty_30d, 0), COALESCE(sa.received_cost_30d, 0),
COALESCE(sa.lifetime_sales, 0), COALESCE(sa.lifetime_revenue, 0),
fpm.first_7_days_sales, fpm.first_7_days_revenue, fpm.first_30_days_sales, fpm.first_30_days_revenue,
fpm.first_60_days_sales, fpm.first_60_days_revenue, fpm.first_90_days_sales, fpm.first_90_days_revenue,
-- Calculated KPIs (using COALESCE on inputs where appropriate)
sa.revenue_30d / NULLIF(sa.sales_30d, 0) AS asp_30d,
sa.cogs_30d / NULLIF(sa.sales_30d, 0) AS acp_30d,
sa.profit_30d / NULLIF(sa.sales_30d, 0) AS avg_ros_30d,
COALESCE(sa.sales_30d, 0) / 30.0 AS avg_sales_per_day_30d,
-- Fix for percentages - cast to numeric with appropriate precision
((sa.profit_30d / NULLIF(sa.revenue_30d, 0)) * 100)::numeric(8,2) AS margin_30d,
((sa.profit_30d / NULLIF(sa.cogs_30d, 0)) * 100)::numeric(8,2) AS markup_30d,
sa.profit_30d / NULLIF(sa.avg_stock_cost_30d, 0) AS gmroi_30d,
sa.sales_30d / NULLIF(sa.avg_stock_units_30d, 0) AS stockturn_30d,
((sa.returns_units_30d / NULLIF(COALESCE(sa.sales_30d, 0) + COALESCE(sa.returns_units_30d, 0), 0)) * 100)::numeric(8,2) AS return_rate_30d,
((sa.discounts_30d / NULLIF(sa.gross_revenue_30d, 0)) * 100)::numeric(8,2) AS discount_rate_30d,
((COALESCE(sa.stockout_days_30d, 0) / 30.0) * 100)::numeric(8,2) AS stockout_rate_30d,
GREATEST(0, sa.gross_regular_revenue_30d - sa.gross_revenue_30d) AS markdown_30d, -- Ensure markdown isn't negative
((GREATEST(0, sa.gross_regular_revenue_30d - sa.gross_revenue_30d) / NULLIF(sa.gross_regular_revenue_30d, 0)) * 100)::numeric(8,2) AS markdown_rate_30d,
-- Sell Through Rate: Sales / (Stock at end of period + Sales). This is one definition proxying for Sales / Beginning Stock.
((sa.sales_30d / NULLIF(
(SELECT eod_stock_quantity FROM daily_product_snapshots WHERE snapshot_date = _calculation_date AND pid = ci.pid LIMIT 1) + COALESCE(sa.sales_30d, 0)
, 0)) * 100)::numeric(8,2) AS sell_through_30d,
-- Use calculated periodic metrics
alt.avg_lead_time_days_calc,
CASE
WHEN ci.replenishable = FALSE THEN NULL -- Non-replenishable don't get a class
ELSE COALESCE(fa.abc_class_calc, 'C') -- Default ranked replenishable but non-contributing to C
END,
-- Forecasting intermediate values (based on historical aggregates ending _calculation_date)
(COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) AS sales_velocity_daily, -- Ensure divisor > 0
s.effective_lead_time AS config_lead_time, s.effective_days_of_stock AS config_days_of_stock, s.effective_safety_stock AS config_safety_stock,
(s.effective_lead_time + s.effective_days_of_stock) AS planning_period_days,
-- Calculate raw forecast need components (using safe velocity)
(COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time AS lead_time_forecast_units,
(COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock AS days_of_stock_forecast_units,
-- Planning period forecast units (sum of lead time and DOS units)
CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock) AS planning_period_forecast_units,
-- Closing stock calculations (using raw forecast components for accuracy before rounding)
(ci.current_stock + COALESCE(ooi.on_order_qty, 0) - ((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)) AS lead_time_closing_stock,
((ci.current_stock + COALESCE(ooi.on_order_qty, 0) - ((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)))
- ((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock) AS days_of_stock_closing_stock,
-- Raw replenishment needed
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time) -- Use rounded forecast units
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0) AS replenishment_needed_raw,
-- Final Forecasting Metrics
-- Replenishment Units (calculated need, before MOQ)
CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
))::int AS replenishment_units,
-- Replenishment Cost/Retail/Profit (based on replenishment_units)
(CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
))::int) * COALESCE(ci.current_effective_cost, 0.00)::numeric(12,2) AS replenishment_cost,
(CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
))::int) * COALESCE(ci.current_price, 0.00)::numeric(12,2) AS replenishment_retail,
(CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
))::int) * (COALESCE(ci.current_price, 0.00) - COALESCE(ci.current_effective_cost, 0.00))::numeric(12,2) AS replenishment_profit,
-- *** FIX: To Order Units (Apply MOQ rounding) ***
CASE
WHEN COALESCE(ci.moq, 0) <= 1 THEN -- Treat no/invalid MOQ or MOQ=1 as no rounding needed
CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
))::int
ELSE -- Apply MOQ rounding: Round UP to nearest multiple of MOQ
(CEILING(GREATEST(0,
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
+ s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0)
) / NULLIF(ci.moq::numeric, 0)) * COALESCE(ci.moq, 1))::int
END AS to_order_units,
-- Forecast Lost Sales (Units occurring during lead time if current+on_order is insufficient)
CEILING(GREATEST(0,
((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time) -- Demand during lead time
- (ci.current_stock + COALESCE(ooi.on_order_qty, 0)) -- Supply available before order arrives
))::int AS forecast_lost_sales_units,
-- Forecast Lost Revenue
(CEILING(GREATEST(0,
((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
- (ci.current_stock + COALESCE(ooi.on_order_qty, 0))
))::int) * COALESCE(ci.current_price, 0.00)::numeric(12,2) AS forecast_lost_revenue,
-- Stock Cover etc (using safe velocity)
ci.current_stock / NULLIF((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)), 0) AS stock_cover_in_days,
COALESCE(ooi.on_order_qty, 0) / NULLIF((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)), 0) AS po_cover_in_days,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)), 0) AS sells_out_in_days,
-- Replenish Date (Project forward from 'today', which is _calculation_date + 1 day)
CASE
WHEN (COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) > 0 -- Check for positive velocity
THEN
_calculation_date + INTERVAL '1 day' -- Today
+ FLOOR(GREATEST(0, ci.current_stock - s.effective_safety_stock) -- Stock above safety
/ (COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) -- divided by velocity
)::integer * INTERVAL '1 day' -- Gives date safety stock is hit
- s.effective_lead_time * INTERVAL '1 day' -- Subtract lead time
ELSE NULL -- Cannot calculate if no sales velocity
END AS replenish_date,
-- Overstocked Units (Stock above safety + planning period demand)
GREATEST(0, ci.current_stock - s.effective_safety_stock -
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time) -- Demand during lead time
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock)) -- Demand during DOS
)::int AS overstocked_units,
(GREATEST(0, ci.current_stock - s.effective_safety_stock -
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
)::int) * COALESCE(ci.current_effective_cost, 0.00)::numeric(12,2) AS overstocked_cost,
(GREATEST(0, ci.current_stock - s.effective_safety_stock -
(CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_lead_time)
+ CEILING((COALESCE(sa.sales_30d, 0) / NULLIF(GREATEST(1.0, 30.0 - COALESCE(sa.stockout_days_30d, 0)), 0)) * s.effective_days_of_stock))
)::int) * COALESCE(ci.current_price, 0.00)::numeric(12,2) AS overstocked_retail,
-- Old Stock Flag
(ci.created_at::date < (_calculation_date - INTERVAL '60 day')::date) AND
(COALESCE(ci.date_last_sold, hd.max_order_date) IS NULL OR COALESCE(ci.date_last_sold, hd.max_order_date) < (_calculation_date - INTERVAL '60 day')::date) AND
(hd.date_last_received_calc IS NULL OR hd.date_last_received_calc < (_calculation_date - INTERVAL '60 day')::date) AND
COALESCE(ooi.on_order_qty, 0) = 0 AS is_old_stock,
COALESCE(sa.yesterday_sales, 0) -- Sales for _calculation_date
FROM CurrentInfo ci
LEFT JOIN OnOrderInfo ooi ON ci.pid = ooi.pid
LEFT JOIN HistoricalDates hd ON ci.pid = hd.pid
LEFT JOIN SnapshotAggregates sa ON ci.pid = sa.pid
LEFT JOIN FirstPeriodMetrics fpm ON ci.pid = fpm.pid
LEFT JOIN Settings s ON ci.pid = s.pid
LEFT JOIN AvgLeadTime alt ON ci.pid = alt.pid -- Join calculated avg lead time
LEFT JOIN FinalABC fa ON ci.pid = fa.pid -- Join calculated ABC class
WHERE s.exclude_forecast IS FALSE OR s.exclude_forecast IS NULL
ON CONFLICT (pid) DO UPDATE SET
-- *** IMPORTANT: List ALL columns here, ensuring order matches INSERT list ***
-- Update ALL columns to ensure entire row is refreshed
last_calculated = EXCLUDED.last_calculated, sku = EXCLUDED.sku, title = EXCLUDED.title, brand = EXCLUDED.brand, vendor = EXCLUDED.vendor, image_url = EXCLUDED.image_url, is_visible = EXCLUDED.is_visible, is_replenishable = EXCLUDED.is_replenishable,
current_price = EXCLUDED.current_price, current_regular_price = EXCLUDED.current_regular_price, current_cost_price = EXCLUDED.current_cost_price, current_landing_cost_price = EXCLUDED.current_landing_cost_price,
current_stock = EXCLUDED.current_stock, current_stock_cost = EXCLUDED.current_stock_cost, current_stock_retail = EXCLUDED.current_stock_retail, current_stock_gross = EXCLUDED.current_stock_gross,
on_order_qty = EXCLUDED.on_order_qty, on_order_cost = EXCLUDED.on_order_cost, on_order_retail = EXCLUDED.on_order_retail, earliest_expected_date = EXCLUDED.earliest_expected_date,
date_created = EXCLUDED.date_created, date_first_received = EXCLUDED.date_first_received, date_last_received = EXCLUDED.date_last_received, date_first_sold = EXCLUDED.date_first_sold, date_last_sold = EXCLUDED.date_last_sold, age_days = EXCLUDED.age_days,
sales_7d = EXCLUDED.sales_7d, revenue_7d = EXCLUDED.revenue_7d, sales_14d = EXCLUDED.sales_14d, revenue_14d = EXCLUDED.revenue_14d, sales_30d = EXCLUDED.sales_30d, revenue_30d = EXCLUDED.revenue_30d, cogs_30d = EXCLUDED.cogs_30d, profit_30d = EXCLUDED.profit_30d,
returns_units_30d = EXCLUDED.returns_units_30d, returns_revenue_30d = EXCLUDED.returns_revenue_30d, discounts_30d = EXCLUDED.discounts_30d, gross_revenue_30d = EXCLUDED.gross_revenue_30d, gross_regular_revenue_30d = EXCLUDED.gross_regular_revenue_30d,
stockout_days_30d = EXCLUDED.stockout_days_30d, sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
avg_stock_units_30d = EXCLUDED.avg_stock_units_30d, avg_stock_cost_30d = EXCLUDED.avg_stock_cost_30d, avg_stock_retail_30d = EXCLUDED.avg_stock_retail_30d, avg_stock_gross_30d = EXCLUDED.avg_stock_gross_30d,
received_qty_30d = EXCLUDED.received_qty_30d, received_cost_30d = EXCLUDED.received_cost_30d,
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
first_7_days_sales = EXCLUDED.first_7_days_sales, first_7_days_revenue = EXCLUDED.first_7_days_revenue, first_30_days_sales = EXCLUDED.first_30_days_sales, first_30_days_revenue = EXCLUDED.first_30_days_revenue,
first_60_days_sales = EXCLUDED.first_60_days_sales, first_60_days_revenue = EXCLUDED.first_60_days_revenue, first_90_days_sales = EXCLUDED.first_90_days_sales, first_90_days_revenue = EXCLUDED.first_90_days_revenue,
asp_30d = EXCLUDED.asp_30d, acp_30d = EXCLUDED.acp_30d, avg_ros_30d = EXCLUDED.avg_ros_30d, avg_sales_per_day_30d = EXCLUDED.avg_sales_per_day_30d,
margin_30d = EXCLUDED.margin_30d, markup_30d = EXCLUDED.markup_30d, gmroi_30d = EXCLUDED.gmroi_30d, stockturn_30d = EXCLUDED.stockturn_30d, return_rate_30d = EXCLUDED.return_rate_30d, discount_rate_30d = EXCLUDED.discount_rate_30d,
stockout_rate_30d = EXCLUDED.stockout_rate_30d, markdown_30d = EXCLUDED.markdown_30d, markdown_rate_30d = EXCLUDED.markdown_rate_30d, sell_through_30d = EXCLUDED.sell_through_30d,
avg_lead_time_days = EXCLUDED.avg_lead_time_days, abc_class = EXCLUDED.abc_class,
sales_velocity_daily = EXCLUDED.sales_velocity_daily, config_lead_time = EXCLUDED.config_lead_time, config_days_of_stock = EXCLUDED.config_days_of_stock, config_safety_stock = EXCLUDED.config_safety_stock,
planning_period_days = EXCLUDED.planning_period_days, lead_time_forecast_units = EXCLUDED.lead_time_forecast_units, days_of_stock_forecast_units = EXCLUDED.days_of_stock_forecast_units,
planning_period_forecast_units = EXCLUDED.planning_period_forecast_units, lead_time_closing_stock = EXCLUDED.lead_time_closing_stock, days_of_stock_closing_stock = EXCLUDED.days_of_stock_closing_stock,
replenishment_needed_raw = EXCLUDED.replenishment_needed_raw, replenishment_units = EXCLUDED.replenishment_units, replenishment_cost = EXCLUDED.replenishment_cost, replenishment_retail = EXCLUDED.replenishment_retail, replenishment_profit = EXCLUDED.replenishment_profit,
to_order_units = EXCLUDED.to_order_units, -- *** Update to use EXCLUDED ***
forecast_lost_sales_units = EXCLUDED.forecast_lost_sales_units, forecast_lost_revenue = EXCLUDED.forecast_lost_revenue,
stock_cover_in_days = EXCLUDED.stock_cover_in_days, po_cover_in_days = EXCLUDED.po_cover_in_days, sells_out_in_days = EXCLUDED.sells_out_in_days, replenish_date = EXCLUDED.replenish_date,
overstocked_units = EXCLUDED.overstocked_units, overstocked_cost = EXCLUDED.overstocked_cost, overstocked_retail = EXCLUDED.overstocked_retail, is_old_stock = EXCLUDED.is_old_stock,
yesterday_sales = EXCLUDED.yesterday_sales;
RAISE NOTICE 'Finished % module. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;

152
inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
View File

@@ -1,152 +0,0 @@
-- Description: Rebuilds daily product snapshots from scratch using real orders data.
-- Fixes issues with duplicated/inflated metrics.
-- Dependencies: Core import tables (products, orders, receivings).
-- Frequency: One-time run to clear out problematic data.
DO $$
DECLARE
_module_name TEXT := 'rebuild_daily_snapshots';
_start_time TIMESTAMPTZ := clock_timestamp();
_date DATE;
_count INT;
_total_records INT := 0;
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2024-01-01'); -- Starting point for data rebuild
_end_date DATE := CURRENT_DATE;
BEGIN
RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;
-- First truncate the existing snapshots to ensure a clean slate
TRUNCATE TABLE public.daily_product_snapshots;
RAISE NOTICE 'Cleared existing snapshot data';
-- Now rebuild the snapshots day by day
_date := _begin_date;
WHILE _date <= _end_date LOOP
RAISE NOTICE 'Processing date %...', _date;
-- Create snapshots for this date
WITH SalesData AS (
SELECT
p.pid,
p.sku,
-- Count orders to ensure we only include products with real activity
COUNT(o.id) as order_count,
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN COALESCE(o.costeach, p.landing_cost_price, p.cost_price) * o.quantity ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,
-- Aggregate Returns (Quantity < 0 or Status = Returned)
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN ABS(o.quantity) ELSE 0 END), 0) AS units_returned,
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue
FROM public.products p
LEFT JOIN public.orders o
ON p.pid = o.pid
AND o.date::date = _date
GROUP BY p.pid, p.sku
HAVING COUNT(o.id) > 0 -- Only include products with actual orders for this date
),
ReceivingData AS (
SELECT
r.pid,
-- Count receiving documents to ensure we only include products with real activity
COUNT(DISTINCT r.receiving_id) as receiving_count,
-- Calculate received quantity for this day
SUM(r.qty_each) AS units_received,
-- Calculate received cost for this day
SUM(r.qty_each * r.cost_each) AS cost_received
FROM public.receivings r
WHERE r.received_date::date = _date
GROUP BY r.pid
HAVING COUNT(DISTINCT r.receiving_id) > 0 OR SUM(r.qty_each) > 0
),
-- Get stock quantities for the day - note this is approximate since we're using current products data
StockData AS (
SELECT
p.pid,
p.stock_quantity,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as effective_cost_price,
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price
FROM public.products p
)
INSERT INTO public.daily_product_snapshots (
snapshot_date,
pid,
sku,
eod_stock_quantity,
eod_stock_cost,
eod_stock_retail,
eod_stock_gross,
stockout_flag,
units_sold,
units_returned,
gross_revenue,
discounts,
returns_revenue,
net_revenue,
cogs,
gross_regular_revenue,
profit,
units_received,
cost_received,
calculation_timestamp
)
SELECT
_date AS snapshot_date,
COALESCE(sd.pid, rd.pid) AS pid,
sd.sku,
-- Use current stock as approximation, since historical stock data may not be available
s.stock_quantity AS eod_stock_quantity,
s.stock_quantity * s.effective_cost_price AS eod_stock_cost,
s.stock_quantity * s.current_price AS eod_stock_retail,
s.stock_quantity * s.current_regular_price AS eod_stock_gross,
(s.stock_quantity <= 0) AS stockout_flag,
-- Sales metrics
COALESCE(sd.units_sold, 0),
COALESCE(sd.units_returned, 0),
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
-- Receiving metrics
COALESCE(rd.units_received, 0),
COALESCE(rd.cost_received, 0.00),
_start_time
FROM SalesData sd
FULL OUTER JOIN ReceivingData rd ON sd.pid = rd.pid
LEFT JOIN StockData s ON COALESCE(sd.pid, rd.pid) = s.pid
WHERE (COALESCE(sd.order_count, 0) > 0 OR COALESCE(rd.receiving_count, 0) > 0);
-- Get record count for this day
GET DIAGNOSTICS _count = ROW_COUNT;
_total_records := _total_records + _count;
RAISE NOTICE 'Added % snapshot records for date %', _count, _date;
-- Move to next day
_date := _date + INTERVAL '1 day';
END LOOP;
RAISE NOTICE 'Rebuilding daily snapshots complete. Added % total records across % days.', _total_records, (_end_date - _begin_date)::integer + 1;
-- Update the status table for daily_snapshots
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
VALUES ('daily_snapshots', _start_time)
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
-- Now update product_metrics based on the rebuilt snapshots
RAISE NOTICE 'Triggering update of product_metrics table...';
-- Call the update_product_metrics procedure directly
-- Your system might use a different method to trigger this update
PERFORM pg_notify('recalculate_metrics', 'product_metrics');
RAISE NOTICE 'Rebuild complete. Duration: %', clock_timestamp() - _start_time;
END $$;

27
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
View File

@@ -42,20 +42,6 @@ BEGIN
JOIN public.products p ON pm.pid = p.pid
GROUP BY brand_group
),
PreviousPeriodBrandMetrics AS (
-- Get previous period metrics for growth calculation
SELECT
COALESCE(p.brand, 'Unbranded') AS brand_group,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM public.daily_product_snapshots dps
JOIN public.products p ON dps.pid = p.pid
GROUP BY brand_group
),
AllBrands AS (
-- Ensure all brands from products table are included, mapping NULL/empty to 'Unbranded'
SELECT DISTINCT COALESCE(brand, 'Unbranded') as brand_group
@@ -67,8 +53,7 @@ BEGIN
current_stock_units, current_stock_cost, current_stock_retail,
sales_7d, revenue_7d, sales_30d, revenue_30d, profit_30d, cogs_30d,
sales_365d, revenue_365d, lifetime_sales, lifetime_revenue,
avg_margin_30d,
sales_growth_30d_vs_prev, revenue_growth_30d_vs_prev
avg_margin_30d
)
SELECT
b.brand_group,
@@ -93,13 +78,9 @@ BEGIN
-- This is mathematically equivalent to profit/revenue but more explicit
((COALESCE(ba.revenue_30d, 0) - COALESCE(ba.cogs_30d, 0)) / COALESCE(ba.revenue_30d, 1)) * 100.0
ELSE NULL -- No margin for low/no revenue brands
END,
-- Growth metrics
std_numeric(safe_divide((ba.sales_30d - ppbm.sales_prev_30d) * 100.0, ppbm.sales_prev_30d), 2),
std_numeric(safe_divide((ba.revenue_30d - ppbm.revenue_prev_30d) * 100.0, ppbm.revenue_prev_30d), 2)
END
FROM AllBrands b
LEFT JOIN BrandAggregates ba ON b.brand_group = ba.brand_group
LEFT JOIN PreviousPeriodBrandMetrics ppbm ON b.brand_group = ppbm.brand_group
ON CONFLICT (brand_name) DO UPDATE SET
last_calculated = EXCLUDED.last_calculated,
@@ -114,9 +95,7 @@ BEGIN
profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
avg_margin_30d = EXCLUDED.avg_margin_30d,
sales_growth_30d_vs_prev = EXCLUDED.sales_growth_30d_vs_prev,
revenue_growth_30d_vs_prev = EXCLUDED.revenue_growth_30d_vs_prev
avg_margin_30d = EXCLUDED.avg_margin_30d
WHERE -- Only update if at least one value has changed
brand_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
brand_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR

328
inventory-server/scripts/metrics-new/calculate_category_metrics.sql
View File

@@ -1,5 +1,5 @@
-- Description: Calculates and updates aggregated metrics per category with hierarchy rollups.
-- Dependencies: product_metrics, products, categories, product_categories, category_hierarchy, calculate_status table.
-- Description: Calculates and updates aggregated metrics per category.
-- Dependencies: product_metrics, products, categories, product_categories, calculate_status table.
-- Frequency: Daily (after product_metrics update).
DO $$
@@ -9,21 +9,55 @@ DECLARE
_min_revenue NUMERIC := 50.00; -- Minimum revenue threshold for margin calculation
BEGIN
RAISE NOTICE 'Running % calculation...', _module_name;
-- Refresh the category hierarchy materialized view first
REFRESH MATERIALIZED VIEW CONCURRENTLY category_hierarchy;
-- First calculate direct metrics (products directly in each category)
WITH DirectCategoryMetrics AS (
WITH
-- Identify the hierarchy depth for each category
CategoryDepth AS (
WITH RECURSIVE CategoryTree AS (
-- Base case: Start with categories without parents (root categories)
SELECT cat_id, name, parent_id, 0 AS depth
FROM public.categories
WHERE parent_id IS NULL
UNION ALL
-- Recursive step: Add child categories with incremented depth
SELECT c.cat_id, c.name, c.parent_id, ct.depth + 1
FROM public.categories c
JOIN CategoryTree ct ON c.parent_id = ct.cat_id
)
SELECT cat_id, depth
FROM CategoryTree
),
-- For each product, find the most specific (deepest) category it belongs to
ProductDeepestCategory AS (
SELECT
pc.pid,
pc.cat_id
FROM public.product_categories pc
JOIN CategoryDepth cd ON pc.cat_id = cd.cat_id
-- This is the key part: for each product, select only the category with maximum depth
WHERE (pc.pid, cd.depth) IN (
SELECT pc2.pid, MAX(cd2.depth)
FROM public.product_categories pc2
JOIN CategoryDepth cd2 ON pc2.cat_id = cd2.cat_id
GROUP BY pc2.pid
)
),
-- Calculate metrics only at the most specific category level for each product
-- These are the direct metrics (only products directly in this category)
DirectCategoryMetrics AS (
SELECT
pc.cat_id,
pdc.cat_id,
-- Counts
COUNT(DISTINCT pm.pid) AS product_count,
COUNT(DISTINCT CASE WHEN pm.is_visible THEN pm.pid END) AS active_product_count,
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN pm.pid END) AS replenishable_product_count,
-- Current Stock
SUM(pm.current_stock) AS current_stock_units,
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
-- Sales metrics with proper filtering
-- Rolling Periods - Only include products with actual sales in each period
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
@@ -33,141 +67,179 @@ BEGIN
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
FROM public.product_categories pc
JOIN public.product_metrics pm ON pc.pid = pm.pid
GROUP BY pc.cat_id
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue,
-- Data for KPIs - Only average stock for products with stock
SUM(CASE WHEN pm.avg_stock_units_30d > 0 THEN pm.avg_stock_units_30d ELSE 0 END) AS total_avg_stock_units_30d
FROM public.product_metrics pm
JOIN ProductDeepestCategory pdc ON pm.pid = pdc.pid
GROUP BY pdc.cat_id
),
-- Calculate rolled-up metrics (including all descendant categories)
RolledUpMetrics AS (
-- Build a category lookup table for parent relationships
CategoryHierarchyPaths AS (
WITH RECURSIVE ParentPaths AS (
-- Base case: All categories with their immediate parents
SELECT
cat_id,
cat_id as leaf_id, -- Every category is its own leaf initially
ARRAY[cat_id] as path
FROM public.categories
UNION ALL
-- Recursive step: Walk up the parent chain
SELECT
c.parent_id as cat_id,
pp.leaf_id, -- Keep the original leaf_id
c.parent_id || pp.path as path
FROM ParentPaths pp
JOIN public.categories c ON pp.cat_id = c.cat_id
WHERE c.parent_id IS NOT NULL -- Stop at root categories
)
-- Select distinct paths to avoid duplication
SELECT DISTINCT cat_id, leaf_id
FROM ParentPaths
),
-- Aggregate metrics from leaf categories to their ancestors without duplication
-- These are the rolled-up metrics (including all child categories)
RollupMetrics AS (
SELECT
ch.cat_id,
-- Sum metrics from this category and all its descendants
SUM(dcm.product_count) AS product_count,
SUM(dcm.active_product_count) AS active_product_count,
SUM(dcm.replenishable_product_count) AS replenishable_product_count,
SUM(dcm.current_stock_units) AS current_stock_units,
SUM(dcm.current_stock_cost) AS current_stock_cost,
SUM(dcm.current_stock_retail) AS current_stock_retail,
SUM(dcm.sales_7d) AS sales_7d,
SUM(dcm.revenue_7d) AS revenue_7d,
SUM(dcm.sales_30d) AS sales_30d,
SUM(dcm.revenue_30d) AS revenue_30d,
SUM(dcm.cogs_30d) AS cogs_30d,
SUM(dcm.profit_30d) AS profit_30d,
SUM(dcm.sales_365d) AS sales_365d,
SUM(dcm.revenue_365d) AS revenue_365d,
SUM(dcm.lifetime_sales) AS lifetime_sales,
SUM(dcm.lifetime_revenue) AS lifetime_revenue
FROM category_hierarchy ch
LEFT JOIN DirectCategoryMetrics dcm ON
dcm.cat_id = ch.cat_id OR
dcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
chp.cat_id,
-- For each parent category, count distinct products to avoid duplication
COUNT(DISTINCT dcm.cat_id) AS child_categories_count,
SUM(dcm.product_count) AS rollup_product_count,
SUM(dcm.active_product_count) AS rollup_active_product_count,
SUM(dcm.replenishable_product_count) AS rollup_replenishable_product_count,
SUM(dcm.current_stock_units) AS rollup_current_stock_units,
SUM(dcm.current_stock_cost) AS rollup_current_stock_cost,
SUM(dcm.current_stock_retail) AS rollup_current_stock_retail,
SUM(dcm.sales_7d) AS rollup_sales_7d,
SUM(dcm.revenue_7d) AS rollup_revenue_7d,
SUM(dcm.sales_30d) AS rollup_sales_30d,
SUM(dcm.revenue_30d) AS rollup_revenue_30d,
SUM(dcm.cogs_30d) AS rollup_cogs_30d,
SUM(dcm.profit_30d) AS rollup_profit_30d,
SUM(dcm.sales_365d) AS rollup_sales_365d,
SUM(dcm.revenue_365d) AS rollup_revenue_365d,
SUM(dcm.lifetime_sales) AS rollup_lifetime_sales,
SUM(dcm.lifetime_revenue) AS rollup_lifetime_revenue,
SUM(dcm.total_avg_stock_units_30d) AS rollup_total_avg_stock_units_30d
FROM CategoryHierarchyPaths chp
JOIN DirectCategoryMetrics dcm ON chp.leaf_id = dcm.cat_id
GROUP BY chp.cat_id
),
PreviousPeriodCategoryMetrics AS (
-- Get previous period metrics for growth calculation
-- Combine direct and rollup metrics
CombinedMetrics AS (
SELECT
pc.cat_id,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM public.daily_product_snapshots dps
JOIN public.product_categories pc ON dps.pid = pc.pid
GROUP BY pc.cat_id
),
RolledUpPreviousPeriod AS (
-- Calculate rolled-up previous period metrics
SELECT
ch.cat_id,
SUM(ppcm.sales_prev_30d) AS sales_prev_30d,
SUM(ppcm.revenue_prev_30d) AS revenue_prev_30d
FROM category_hierarchy ch
LEFT JOIN PreviousPeriodCategoryMetrics ppcm ON
ppcm.cat_id = ch.cat_id OR
ppcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
GROUP BY ch.cat_id
),
AllCategories AS (
-- Ensure all categories are included
SELECT
c.cat_id,
c.name,
c.type,
c.parent_id
c.parent_id,
-- Direct metrics (just this category)
COALESCE(dcm.product_count, 0) AS direct_product_count,
COALESCE(dcm.active_product_count, 0) AS direct_active_product_count,
COALESCE(dcm.replenishable_product_count, 0) AS direct_replenishable_product_count,
COALESCE(dcm.current_stock_units, 0) AS direct_current_stock_units,
COALESCE(dcm.current_stock_cost, 0) AS direct_current_stock_cost,
COALESCE(dcm.current_stock_retail, 0) AS direct_current_stock_retail,
COALESCE(dcm.sales_7d, 0) AS direct_sales_7d,
COALESCE(dcm.revenue_7d, 0) AS direct_revenue_7d,
COALESCE(dcm.sales_30d, 0) AS direct_sales_30d,
COALESCE(dcm.revenue_30d, 0) AS direct_revenue_30d,
COALESCE(dcm.cogs_30d, 0) AS direct_cogs_30d,
COALESCE(dcm.profit_30d, 0) AS direct_profit_30d,
COALESCE(dcm.sales_365d, 0) AS direct_sales_365d,
COALESCE(dcm.revenue_365d, 0) AS direct_revenue_365d,
COALESCE(dcm.lifetime_sales, 0) AS direct_lifetime_sales,
COALESCE(dcm.lifetime_revenue, 0) AS direct_lifetime_revenue,
COALESCE(dcm.total_avg_stock_units_30d, 0) AS direct_avg_stock_units_30d,
-- Rolled up metrics (this category + all children)
COALESCE(rm.rollup_product_count, 0) AS product_count,
COALESCE(rm.rollup_active_product_count, 0) AS active_product_count,
COALESCE(rm.rollup_replenishable_product_count, 0) AS replenishable_product_count,
COALESCE(rm.rollup_current_stock_units, 0) AS current_stock_units,
COALESCE(rm.rollup_current_stock_cost, 0) AS current_stock_cost,
COALESCE(rm.rollup_current_stock_retail, 0) AS current_stock_retail,
COALESCE(rm.rollup_sales_7d, 0) AS sales_7d,
COALESCE(rm.rollup_revenue_7d, 0) AS revenue_7d,
COALESCE(rm.rollup_sales_30d, 0) AS sales_30d,
COALESCE(rm.rollup_revenue_30d, 0) AS revenue_30d,
COALESCE(rm.rollup_cogs_30d, 0) AS cogs_30d,
COALESCE(rm.rollup_profit_30d, 0) AS profit_30d,
COALESCE(rm.rollup_sales_365d, 0) AS sales_365d,
COALESCE(rm.rollup_revenue_365d, 0) AS revenue_365d,
COALESCE(rm.rollup_lifetime_sales, 0) AS lifetime_sales,
COALESCE(rm.rollup_lifetime_revenue, 0) AS lifetime_revenue,
COALESCE(rm.rollup_total_avg_stock_units_30d, 0) AS total_avg_stock_units_30d
FROM public.categories c
WHERE c.status = 'active'
LEFT JOIN DirectCategoryMetrics dcm ON c.cat_id = dcm.cat_id
LEFT JOIN RollupMetrics rm ON c.cat_id = rm.cat_id
)
INSERT INTO public.category_metrics (
category_id, category_name, category_type, parent_id, last_calculated,
-- Rolled-up metrics
-- Store all direct and rolled up metrics
product_count, active_product_count, replenishable_product_count,
current_stock_units, current_stock_cost, current_stock_retail,
sales_7d, revenue_7d, sales_30d, revenue_30d, profit_30d, cogs_30d,
sales_365d, revenue_365d, lifetime_sales, lifetime_revenue,
-- Direct metrics
-- Also store direct metrics with direct_ prefix
direct_product_count, direct_active_product_count, direct_replenishable_product_count,
direct_current_stock_units, direct_stock_cost, direct_stock_retail,
direct_sales_7d, direct_revenue_7d, direct_sales_30d, direct_revenue_30d,
direct_sales_7d, direct_revenue_7d, direct_sales_30d, direct_revenue_30d,
direct_profit_30d, direct_cogs_30d, direct_sales_365d, direct_revenue_365d,
direct_lifetime_sales, direct_lifetime_revenue,
-- KPIs
avg_margin_30d,
sales_growth_30d_vs_prev, revenue_growth_30d_vs_prev
avg_margin_30d, stock_turn_30d
)
SELECT
ac.cat_id,
ac.name,
ac.type,
ac.parent_id,
cm.cat_id,
cm.name,
cm.type,
cm.parent_id,
_start_time,
-- Rolled-up metrics (includes descendants)
COALESCE(rum.product_count, 0),
COALESCE(rum.active_product_count, 0),
COALESCE(rum.replenishable_product_count, 0),
COALESCE(rum.current_stock_units, 0),
COALESCE(rum.current_stock_cost, 0.00),
COALESCE(rum.current_stock_retail, 0.00),
COALESCE(rum.sales_7d, 0), COALESCE(rum.revenue_7d, 0.00),
COALESCE(rum.sales_30d, 0), COALESCE(rum.revenue_30d, 0.00),
COALESCE(rum.profit_30d, 0.00), COALESCE(rum.cogs_30d, 0.00),
COALESCE(rum.sales_365d, 0), COALESCE(rum.revenue_365d, 0.00),
COALESCE(rum.lifetime_sales, 0), COALESCE(rum.lifetime_revenue, 0.00),
-- Direct metrics (only this category)
COALESCE(dcm.product_count, 0),
COALESCE(dcm.active_product_count, 0),
COALESCE(dcm.replenishable_product_count, 0),
COALESCE(dcm.current_stock_units, 0),
COALESCE(dcm.current_stock_cost, 0.00),
COALESCE(dcm.current_stock_retail, 0.00),
COALESCE(dcm.sales_7d, 0), COALESCE(dcm.revenue_7d, 0.00),
COALESCE(dcm.sales_30d, 0), COALESCE(dcm.revenue_30d, 0.00),
COALESCE(dcm.profit_30d, 0.00), COALESCE(dcm.cogs_30d, 0.00),
COALESCE(dcm.sales_365d, 0), COALESCE(dcm.revenue_365d, 0.00),
COALESCE(dcm.lifetime_sales, 0), COALESCE(dcm.lifetime_revenue, 0.00),
-- Rolled-up metrics (total including children)
cm.product_count,
cm.active_product_count,
cm.replenishable_product_count,
cm.current_stock_units,
cm.current_stock_cost,
cm.current_stock_retail,
cm.sales_7d, cm.revenue_7d,
cm.sales_30d, cm.revenue_30d, cm.profit_30d, cm.cogs_30d,
cm.sales_365d, cm.revenue_365d,
cm.lifetime_sales, cm.lifetime_revenue,
-- Direct metrics (just this category)
cm.direct_product_count,
cm.direct_active_product_count,
cm.direct_replenishable_product_count,
cm.direct_current_stock_units,
cm.direct_current_stock_cost,
cm.direct_current_stock_retail,
cm.direct_sales_7d, cm.direct_revenue_7d,
cm.direct_sales_30d, cm.direct_revenue_30d, cm.direct_profit_30d, cm.direct_cogs_30d,
cm.direct_sales_365d, cm.direct_revenue_365d,
cm.direct_lifetime_sales, cm.direct_lifetime_revenue,
-- KPIs - Calculate margin only for categories with significant revenue
CASE
WHEN COALESCE(rum.revenue_30d, 0) >= _min_revenue THEN
((COALESCE(rum.revenue_30d, 0) - COALESCE(rum.cogs_30d, 0)) / COALESCE(rum.revenue_30d, 1)) * 100.0
ELSE NULL
WHEN cm.revenue_30d >= _min_revenue THEN
((cm.revenue_30d - cm.cogs_30d) / cm.revenue_30d) * 100.0
ELSE NULL -- No margin for low/no revenue categories
END,
-- Growth metrics for rolled-up values
std_numeric(safe_divide((rum.sales_30d - rupp.sales_prev_30d) * 100.0, rupp.sales_prev_30d), 2),
std_numeric(safe_divide((rum.revenue_30d - rupp.revenue_prev_30d) * 100.0, rupp.revenue_prev_30d), 2)
FROM AllCategories ac
LEFT JOIN DirectCategoryMetrics dcm ON ac.cat_id = dcm.cat_id
LEFT JOIN RolledUpMetrics rum ON ac.cat_id = rum.cat_id
LEFT JOIN RolledUpPreviousPeriod rupp ON ac.cat_id = rupp.cat_id
-- Stock Turn calculation
CASE
WHEN cm.total_avg_stock_units_30d > 0 THEN
cm.sales_30d / cm.total_avg_stock_units_30d
ELSE NULL -- No stock turn if no average stock
END
FROM CombinedMetrics cm
ON CONFLICT (category_id) DO UPDATE SET
last_calculated = EXCLUDED.last_calculated,
category_name = EXCLUDED.category_name,
category_type = EXCLUDED.category_type,
parent_id = EXCLUDED.parent_id,
-- Rolled-up metrics
last_calculated = EXCLUDED.last_calculated,
-- ROLLED-UP METRICS (includes this category + all descendants)
product_count = EXCLUDED.product_count,
active_product_count = EXCLUDED.active_product_count,
replenishable_product_count = EXCLUDED.replenishable_product_count,
@@ -179,7 +251,8 @@ BEGIN
profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
-- Direct metrics
-- DIRECT METRICS (only products directly in this category)
direct_product_count = EXCLUDED.direct_product_count,
direct_active_product_count = EXCLUDED.direct_active_product_count,
direct_replenishable_product_count = EXCLUDED.direct_replenishable_product_count,
@@ -191,9 +264,10 @@ BEGIN
direct_profit_30d = EXCLUDED.direct_profit_30d, direct_cogs_30d = EXCLUDED.direct_cogs_30d,
direct_sales_365d = EXCLUDED.direct_sales_365d, direct_revenue_365d = EXCLUDED.direct_revenue_365d,
direct_lifetime_sales = EXCLUDED.direct_lifetime_sales, direct_lifetime_revenue = EXCLUDED.direct_lifetime_revenue,
-- Calculated KPIs
avg_margin_30d = EXCLUDED.avg_margin_30d,
sales_growth_30d_vs_prev = EXCLUDED.sales_growth_30d_vs_prev,
revenue_growth_30d_vs_prev = EXCLUDED.revenue_growth_30d_vs_prev
stock_turn_30d = EXCLUDED.stock_turn_30d
WHERE -- Only update if at least one value has changed
category_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
category_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
@@ -217,23 +291,19 @@ WITH update_stats AS (
SELECT
COUNT(*) as total_categories,
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
COUNT(*) FILTER (WHERE category_type = 10) as sections,
COUNT(*) FILTER (WHERE category_type = 11) as categories,
COUNT(*) FILTER (WHERE category_type = 12) as subcategories,
SUM(product_count) as total_products_rolled,
SUM(direct_product_count) as total_products_direct,
SUM(sales_30d) as total_sales_30d,
SUM(revenue_30d) as total_revenue_30d
COUNT(*) FILTER (WHERE category_type = 11) as main_categories, -- 11 = category
COUNT(*) FILTER (WHERE category_type = 12) as subcategories, -- 12 = subcategory
SUM(product_count) as total_products,
SUM(active_product_count) as total_active_products,
SUM(current_stock_units) as total_stock_units
FROM public.category_metrics
)
SELECT
rows_processed,
total_categories,
sections,
categories,
main_categories,
subcategories,
total_products_rolled::int,
total_products_direct::int,
total_sales_30d::int,
ROUND(total_revenue_30d, 2) as total_revenue_30d
total_products::int,
total_active_products::int,
total_stock_units::int
FROM update_stats;

185
inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql
View File

@@ -1,185 +0,0 @@
-- Description: Calculates and updates aggregated metrics per vendor.
-- Dependencies: product_metrics, products, purchase_orders, calculate_status table.
-- Frequency: Daily (after product_metrics update).
DO $$
DECLARE
_module_name VARCHAR := 'vendor_metrics';
_start_time TIMESTAMPTZ := clock_timestamp();
BEGIN
RAISE NOTICE 'Running % calculation...', _module_name;
WITH VendorProductAggregates AS (
-- Aggregate metrics from product_metrics table per vendor
SELECT
p.vendor,
COUNT(DISTINCT pm.pid) AS product_count,
COUNT(DISTINCT CASE WHEN pm.is_visible THEN pm.pid END) AS active_product_count,
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN pm.pid END) AS replenishable_product_count,
SUM(pm.current_stock) AS current_stock_units,
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
SUM(pm.on_order_qty) AS on_order_units,
SUM(pm.on_order_cost) AS on_order_cost,
-- Only include products with valid sales data in each time period
COUNT(DISTINCT CASE WHEN pm.sales_7d > 0 THEN pm.pid END) AS products_with_sales_7d,
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
COUNT(DISTINCT CASE WHEN pm.lifetime_sales > 0 THEN pm.pid END) AS products_with_lifetime_sales,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
FROM public.product_metrics pm
JOIN public.products p ON pm.pid = p.pid
WHERE p.vendor IS NOT NULL AND p.vendor <> ''
GROUP BY p.vendor
),
PreviousPeriodVendorMetrics AS (
-- Get previous period metrics for growth calculation
SELECT
p.vendor,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN dps.snapshot_date >= CURRENT_DATE - INTERVAL '59 days'
AND dps.snapshot_date < CURRENT_DATE - INTERVAL '29 days'
THEN dps.net_revenue ELSE 0 END) AS revenue_prev_30d
FROM public.daily_product_snapshots dps
JOIN public.products p ON dps.pid = p.pid
WHERE p.vendor IS NOT NULL AND p.vendor <> ''
GROUP BY p.vendor
),
VendorPOAggregates AS (
-- Aggregate PO related stats including lead time calculated from POs to receivings
SELECT
po.vendor,
COUNT(DISTINCT po.po_id) AS po_count_365d,
-- Calculate lead time by averaging the days between PO date and receiving date
AVG(GREATEST(1, CASE
WHEN r.received_date IS NOT NULL AND po.date IS NOT NULL
THEN (r.received_date::date - po.date::date)
ELSE NULL
END))::int AS avg_lead_time_days_hist -- Avg lead time from HISTORICAL received POs
FROM public.purchase_orders po
-- Join to receivings table to find when items were received
LEFT JOIN public.receivings r ON r.pid = po.pid
WHERE po.vendor IS NOT NULL AND po.vendor <> ''
AND po.date >= CURRENT_DATE - INTERVAL '1 year' -- Look at POs created in the last year
AND po.status = 'done' -- Only calculate lead time on completed POs
AND r.received_date IS NOT NULL
AND po.date IS NOT NULL
AND r.received_date >= po.date
GROUP BY po.vendor
),
AllVendors AS (
-- Ensure all vendors from products table are included
SELECT DISTINCT vendor FROM public.products WHERE vendor IS NOT NULL AND vendor <> ''
)
INSERT INTO public.vendor_metrics (
vendor_name, last_calculated,
product_count, active_product_count, replenishable_product_count,
current_stock_units, current_stock_cost, current_stock_retail,
on_order_units, on_order_cost,
po_count_365d, avg_lead_time_days,
sales_7d, revenue_7d, sales_30d, revenue_30d, profit_30d, cogs_30d,
sales_365d, revenue_365d, lifetime_sales, lifetime_revenue,
avg_margin_30d,
sales_growth_30d_vs_prev, revenue_growth_30d_vs_prev
)
SELECT
v.vendor,
_start_time,
-- Base Aggregates
COALESCE(vpa.product_count, 0),
COALESCE(vpa.active_product_count, 0),
COALESCE(vpa.replenishable_product_count, 0),
COALESCE(vpa.current_stock_units, 0),
COALESCE(vpa.current_stock_cost, 0.00),
COALESCE(vpa.current_stock_retail, 0.00),
COALESCE(vpa.on_order_units, 0),
COALESCE(vpa.on_order_cost, 0.00),
-- PO Aggregates
COALESCE(vpoa.po_count_365d, 0),
vpoa.avg_lead_time_days_hist, -- Can be NULL if no received POs
-- Sales Aggregates
COALESCE(vpa.sales_7d, 0), COALESCE(vpa.revenue_7d, 0.00),
COALESCE(vpa.sales_30d, 0), COALESCE(vpa.revenue_30d, 0.00),
COALESCE(vpa.profit_30d, 0.00), COALESCE(vpa.cogs_30d, 0.00),
COALESCE(vpa.sales_365d, 0), COALESCE(vpa.revenue_365d, 0.00),
COALESCE(vpa.lifetime_sales, 0), COALESCE(vpa.lifetime_revenue, 0.00),
-- KPIs
(vpa.profit_30d / NULLIF(vpa.revenue_30d, 0)) * 100.0,
-- Growth metrics
std_numeric(safe_divide((vpa.sales_30d - ppvm.sales_prev_30d) * 100.0, ppvm.sales_prev_30d), 2),
std_numeric(safe_divide((vpa.revenue_30d - ppvm.revenue_prev_30d) * 100.0, ppvm.revenue_prev_30d), 2)
FROM AllVendors v
LEFT JOIN VendorProductAggregates vpa ON v.vendor = vpa.vendor
LEFT JOIN VendorPOAggregates vpoa ON v.vendor = vpoa.vendor
LEFT JOIN PreviousPeriodVendorMetrics ppvm ON v.vendor = ppvm.vendor
ON CONFLICT (vendor_name) DO UPDATE SET
last_calculated = EXCLUDED.last_calculated,
product_count = EXCLUDED.product_count,
active_product_count = EXCLUDED.active_product_count,
replenishable_product_count = EXCLUDED.replenishable_product_count,
current_stock_units = EXCLUDED.current_stock_units,
current_stock_cost = EXCLUDED.current_stock_cost,
current_stock_retail = EXCLUDED.current_stock_retail,
on_order_units = EXCLUDED.on_order_units,
on_order_cost = EXCLUDED.on_order_cost,
po_count_365d = EXCLUDED.po_count_365d,
avg_lead_time_days = EXCLUDED.avg_lead_time_days,
sales_7d = EXCLUDED.sales_7d, revenue_7d = EXCLUDED.revenue_7d,
sales_30d = EXCLUDED.sales_30d, revenue_30d = EXCLUDED.revenue_30d,
profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
avg_margin_30d = EXCLUDED.avg_margin_30d,
sales_growth_30d_vs_prev = EXCLUDED.sales_growth_30d_vs_prev,
revenue_growth_30d_vs_prev = EXCLUDED.revenue_growth_30d_vs_prev
WHERE -- Only update if at least one value has changed
vendor_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
vendor_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
vendor_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
vendor_metrics.on_order_units IS DISTINCT FROM EXCLUDED.on_order_units OR
vendor_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
vendor_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
vendor_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
-- Update calculate_status
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
VALUES (_module_name, _start_time)
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
RAISE NOTICE 'Finished % calculation. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;
-- Return metrics about the update operation for tracking
WITH update_stats AS (
SELECT
COUNT(*) as total_vendors,
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
SUM(product_count) as total_products,
SUM(active_product_count) as total_active_products,
SUM(po_count_365d) as total_pos_365d,
AVG(avg_lead_time_days) as overall_avg_lead_time
FROM public.vendor_metrics
)
SELECT
rows_processed,
total_vendors,
total_products::int,
total_active_products::int,
total_pos_365d::int,
ROUND(overall_avg_lead_time, 1) as overall_avg_lead_time
FROM update_stats;

222
inventory-server/scripts/metrics-new/update_daily_snapshots.sql
View File

@@ -1,222 +0,0 @@
-- Description: Calculates and updates daily aggregated product data for recent days.
-- Uses UPSERT (INSERT ON CONFLICT UPDATE) for idempotency.
-- Dependencies: Core import tables (products, orders, purchase_orders), calculate_status table.
-- Frequency: Hourly (Run ~5-10 minutes after hourly data import completes).
DO $$
DECLARE
_module_name TEXT := 'daily_snapshots';
_start_time TIMESTAMPTZ := clock_timestamp(); -- Time execution started
_last_calc_time TIMESTAMPTZ;
_target_date DATE; -- Will be set in the loop
_total_records INT := 0;
_has_orders BOOLEAN := FALSE;
_process_days INT := 5; -- Number of days to check/process (today plus previous 4 days)
_day_counter INT;
_missing_days INT[] := ARRAY[]::INT[]; -- Array to store days with missing or incomplete data
BEGIN
-- Get the timestamp before the last successful run of this module
SELECT last_calculation_timestamp INTO _last_calc_time
FROM public.calculate_status
WHERE module_name = _module_name;
RAISE NOTICE 'Running % script. Start Time: %', _module_name, _start_time;
-- First, check which days need processing by comparing orders data with snapshot data
FOR _day_counter IN 0..(_process_days-1) LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
-- Check if this date needs updating by comparing orders to snapshot data
-- If the date has orders but not enough snapshots, or if snapshots show zero sales but orders exist, it's incomplete
SELECT
CASE WHEN (
-- We have orders for this date but not enough snapshots, or snapshots with wrong total
(EXISTS (SELECT 1 FROM public.orders WHERE date::date = _target_date) AND
(
-- No snapshots exist for this date
NOT EXISTS (SELECT 1 FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) OR
-- Or snapshots show zero sales but orders exist
(SELECT COALESCE(SUM(units_sold), 0) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) = 0 OR
-- Or the count of snapshot records is significantly less than distinct products in orders
(SELECT COUNT(*) FROM public.daily_product_snapshots WHERE snapshot_date = _target_date) <
(SELECT COUNT(DISTINCT pid) FROM public.orders WHERE date::date = _target_date) * 0.8
)
)
) THEN TRUE ELSE FALSE END
INTO _has_orders;
IF _has_orders THEN
-- This day needs processing - add to our array
_missing_days := _missing_days || _day_counter;
RAISE NOTICE 'Day % needs updating (incomplete or missing data)', _target_date;
END IF;
END LOOP;
-- If no days need updating, exit early
IF array_length(_missing_days, 1) IS NULL THEN
RAISE NOTICE 'No days need updating - all snapshot data appears complete';
-- Still update the calculate_status to record this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RETURN;
END IF;
RAISE NOTICE 'Need to update % days with missing or incomplete data', array_length(_missing_days, 1);
-- Process only the days that need updating
FOREACH _day_counter IN ARRAY _missing_days LOOP
_target_date := CURRENT_DATE - (_day_counter * INTERVAL '1 day');
RAISE NOTICE 'Processing date: %', _target_date;
-- IMPORTANT: First delete any existing data for this date to prevent duplication
DELETE FROM public.daily_product_snapshots
WHERE snapshot_date = _target_date;
-- Proceed with calculating daily metrics only for products with actual activity
WITH SalesData AS (
SELECT
p.pid,
p.sku,
-- Track number of orders to ensure we have real data
COUNT(o.id) as order_count,
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, -- Before discount
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN
COALESCE(
o.costeach, -- First use order-specific cost if available
get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost
p.landing_cost_price, -- Fallback to landing cost
p.cost_price -- Final fallback to current cost
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Use current regular price for simplicity here
-- Aggregate Returns (Quantity < 0 or Status = Returned)
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN ABS(o.quantity) ELSE 0 END), 0) AS units_returned,
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue
FROM public.products p -- Start from products to include those with no orders today
JOIN public.orders o -- Changed to INNER JOIN to only process products with orders
ON p.pid = o.pid
AND o.date::date = _target_date -- Cast to date to ensure compatibility regardless of original type
GROUP BY p.pid, p.sku
-- No HAVING clause here - we always want to include all orders
),
ReceivingData AS (
SELECT
r.pid,
-- Track number of receiving docs to ensure we have real data
COUNT(DISTINCT r.receiving_id) as receiving_doc_count,
-- Sum the quantities received on this date
SUM(r.qty_each) AS units_received,
-- Calculate the cost received (qty * cost)
SUM(r.qty_each * r.cost_each) AS cost_received
FROM public.receivings r
WHERE r.received_date::date = _target_date
-- Optional: Filter out canceled receivings if needed
-- AND r.status <> 'canceled'
GROUP BY r.pid
-- Only include products with actual receiving activity
HAVING COUNT(DISTINCT r.receiving_id) > 0 OR SUM(r.qty_each) > 0
),
CurrentStock AS (
-- Select current stock values directly from products table
SELECT
pid,
stock_quantity,
COALESCE(landing_cost_price, cost_price, 0.00) as effective_cost_price,
COALESCE(price, 0.00) as current_price,
COALESCE(regular_price, 0.00) as current_regular_price
FROM public.products
),
ProductsWithActivity AS (
-- Quick pre-filter to only process products with activity
SELECT DISTINCT pid
FROM (
SELECT pid FROM SalesData
UNION
SELECT pid FROM ReceivingData
) a
)
-- Now insert records, but ONLY for products with actual activity
INSERT INTO public.daily_product_snapshots (
snapshot_date,
pid,
sku,
eod_stock_quantity,
eod_stock_cost,
eod_stock_retail,
eod_stock_gross,
stockout_flag,
units_sold,
units_returned,
gross_revenue,
discounts,
returns_revenue,
net_revenue,
cogs,
gross_regular_revenue,
profit,
units_received,
cost_received,
calculation_timestamp
)
SELECT
_target_date AS snapshot_date,
COALESCE(sd.pid, rd.pid) AS pid, -- Use sales or receiving PID
COALESCE(sd.sku, p.sku) AS sku, -- Get SKU from sales data or products table
-- Inventory Metrics (Using CurrentStock)
cs.stock_quantity AS eod_stock_quantity,
cs.stock_quantity * cs.effective_cost_price AS eod_stock_cost,
cs.stock_quantity * cs.current_price AS eod_stock_retail,
cs.stock_quantity * cs.current_regular_price AS eod_stock_gross,
(cs.stock_quantity <= 0) AS stockout_flag,
-- Sales Metrics (From SalesData)
COALESCE(sd.units_sold, 0),
COALESCE(sd.units_returned, 0),
COALESCE(sd.gross_revenue_unadjusted, 0.00),
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit, -- Basic profit: Net Revenue - COGS
-- Receiving Metrics (From ReceivingData)
COALESCE(rd.units_received, 0),
COALESCE(rd.cost_received, 0.00),
_start_time -- Timestamp of this calculation run
FROM SalesData sd
FULL OUTER JOIN ReceivingData rd ON sd.pid = rd.pid
JOIN ProductsWithActivity pwa ON COALESCE(sd.pid, rd.pid) = pwa.pid
LEFT JOIN public.products p ON COALESCE(sd.pid, rd.pid) = p.pid
LEFT JOIN CurrentStock cs ON COALESCE(sd.pid, rd.pid) = cs.pid
WHERE p.pid IS NOT NULL; -- Ensure we only insert for existing products
-- Get the total number of records inserted for this date
GET DIAGNOSTICS _total_records = ROW_COUNT;
RAISE NOTICE 'Created % daily snapshot records for % with sales/receiving activity', _total_records, _target_date;
END LOOP;
-- Update the status table with the timestamp from the START of this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RAISE NOTICE 'Finished % processing for multiple dates. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;
-- Return the total records processed for tracking
SELECT
COUNT(*) as rows_processed,
COUNT(DISTINCT snapshot_date) as days_processed,
MIN(snapshot_date) as earliest_date,
MAX(snapshot_date) as latest_date,
SUM(units_sold) as total_units_sold,
SUM(units_received) as total_units_received
FROM public.daily_product_snapshots
WHERE calculation_timestamp >= (NOW() - INTERVAL '5 minutes'); -- Recent updates only

139
inventory-server/scripts/metrics-new/update_periodic_metrics.sql
View File

@@ -1,139 +0,0 @@
-- Description: Calculates metrics that don't need hourly updates, like ABC class
-- and average lead time.
-- Dependencies: product_metrics, purchase_orders, settings_global, calculate_status.
-- Frequency: Daily or Weekly (e.g., run via cron job overnight).
DO $$
DECLARE
_module_name TEXT := 'periodic_metrics';
_start_time TIMESTAMPTZ := clock_timestamp();
_last_calc_time TIMESTAMPTZ;
_abc_basis VARCHAR;
_abc_period INT;
_threshold_a NUMERIC;
_threshold_b NUMERIC;
BEGIN
-- Get the timestamp before the last successful run of this module
SELECT last_calculation_timestamp INTO _last_calc_time
FROM public.calculate_status
WHERE module_name = _module_name;
RAISE NOTICE 'Running % module. Start Time: %', _module_name, _start_time;
-- 1. Calculate Average Lead Time
RAISE NOTICE 'Calculating Average Lead Time...';
WITH LeadTimes AS (
SELECT
po.pid,
-- Calculate lead time by looking at when items ordered on POs were received
AVG(GREATEST(1, (r.received_date::date - po.date::date))) AS avg_days -- Use GREATEST(1,...) to avoid 0 or negative days
FROM public.purchase_orders po
-- Join to receivings table to find actual receipts
JOIN public.receivings r ON r.pid = po.pid
WHERE po.status = 'done' -- Only include completed POs
AND r.received_date >= po.date -- Ensure received date is not before order date
-- Optional: add check to make sure receiving is related to PO if you have source_po_id
-- AND (r.source_po_id = po.po_id OR r.source_po_id IS NULL)
GROUP BY po.pid
)
UPDATE public.product_metrics pm
SET avg_lead_time_days = lt.avg_days::int
FROM LeadTimes lt
WHERE pm.pid = lt.pid
AND pm.avg_lead_time_days IS DISTINCT FROM lt.avg_days::int; -- Only update if changed
RAISE NOTICE 'Finished Average Lead Time calculation.';
-- 2. Calculate ABC Classification
RAISE NOTICE 'Calculating ABC Classification...';
-- Get ABC settings
SELECT setting_value INTO _abc_basis FROM public.settings_global WHERE setting_key = 'abc_calculation_basis' LIMIT 1;
SELECT setting_value::numeric INTO _threshold_a FROM public.settings_global WHERE setting_key = 'abc_revenue_threshold_a' LIMIT 1;
SELECT setting_value::numeric INTO _threshold_b FROM public.settings_global WHERE setting_key = 'abc_revenue_threshold_b' LIMIT 1;
_abc_basis := COALESCE(_abc_basis, 'revenue_30d'); -- Default basis
_threshold_a := COALESCE(_threshold_a, 0.80);
_threshold_b := COALESCE(_threshold_b, 0.95);
RAISE NOTICE 'Using ABC Basis: %, Threshold A: %, Threshold B: %', _abc_basis, _threshold_a, _threshold_b;
WITH RankedProducts AS (
SELECT
pid,
-- Dynamically select the metric based on setting
CASE _abc_basis
WHEN 'sales_30d' THEN COALESCE(sales_30d, 0)
WHEN 'lifetime_revenue' THEN COALESCE(lifetime_revenue, 0)::numeric -- Cast needed if different type
ELSE COALESCE(revenue_30d, 0) -- Default to revenue_30d
END AS metric_value
FROM public.product_metrics
WHERE is_replenishable = TRUE -- Typically only classify replenishable items
),
Cumulative AS (
SELECT
pid,
metric_value,
SUM(metric_value) OVER (ORDER BY metric_value DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) as cumulative_metric,
SUM(metric_value) OVER () as total_metric
FROM RankedProducts
WHERE metric_value > 0 -- Exclude items with no contribution
)
UPDATE public.product_metrics pm
SET abc_class =
CASE
WHEN c.cumulative_metric / NULLIF(c.total_metric, 0) <= _threshold_a THEN 'A'
WHEN c.cumulative_metric / NULLIF(c.total_metric, 0) <= _threshold_b THEN 'B'
ELSE 'C'
END
FROM Cumulative c
WHERE pm.pid = c.pid
AND pm.abc_class IS DISTINCT FROM ( -- Only update if changed
CASE
WHEN c.cumulative_metric / NULLIF(c.total_metric, 0) <= _threshold_a THEN 'A'
WHEN c.cumulative_metric / NULLIF(c.total_metric, 0) <= _threshold_b THEN 'B'
ELSE 'C'
END);
-- Set non-contributing or non-replenishable to 'C' or NULL if preferred
UPDATE public.product_metrics
SET abc_class = 'C' -- Or NULL
WHERE abc_class IS NULL AND is_replenishable = TRUE; -- Catch those with 0 metric value
UPDATE public.product_metrics
SET abc_class = NULL -- Or 'N/A'?
WHERE is_replenishable = FALSE AND abc_class IS NOT NULL; -- Unclassify non-replenishable items
RAISE NOTICE 'Finished ABC Classification calculation.';
-- Add other periodic calculations here if needed (e.g., recalculating first/last dates)
-- Update the status table with the timestamp from the START of this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RAISE NOTICE 'Finished % module. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;
-- Return metrics about the update operation for tracking
WITH update_stats AS (
SELECT
COUNT(*) as total_products,
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
COUNT(*) FILTER (WHERE abc_class = 'A') as abc_a_count,
COUNT(*) FILTER (WHERE abc_class = 'B') as abc_b_count,
COUNT(*) FILTER (WHERE abc_class = 'C') as abc_c_count,
COUNT(*) FILTER (WHERE avg_lead_time_days IS NOT NULL) as products_with_lead_time,
AVG(avg_lead_time_days) as overall_avg_lead_time
FROM public.product_metrics
)
SELECT
rows_processed,
total_products,
abc_a_count,
abc_b_count,
abc_c_count,
products_with_lead_time,
ROUND(overall_avg_lead_time, 1) as overall_avg_lead_time
FROM update_stats;

609
inventory-server/scripts/metrics-new/update_product_metrics.sql
View File

@@ -1,609 +0,0 @@
-- Description: Calculates and updates the main product_metrics table based on current data
-- and aggregated daily snapshots. Uses UPSERT for idempotency.
-- Dependencies: Core import tables, daily_product_snapshots, configuration tables, calculate_status.
-- Frequency: Hourly (Run AFTER update_daily_snapshots.sql completes).
DO $$
DECLARE
_module_name TEXT := 'product_metrics';
_start_time TIMESTAMPTZ := clock_timestamp();
_last_calc_time TIMESTAMPTZ;
_current_date DATE := CURRENT_DATE;
BEGIN
-- Get the timestamp before the last successful run of this module
SELECT last_calculation_timestamp INTO _last_calc_time
FROM public.calculate_status
WHERE module_name = _module_name;
RAISE NOTICE 'Running % module. Start Time: %', _module_name, _start_time;
-- Use CTEs to gather all necessary information
WITH CurrentInfo AS (
SELECT
p.pid,
p.sku,
p.title,
p.brand,
p.vendor,
COALESCE(p.image_175, p.image) as image_url,
p.visible as is_visible,
p.replenishable as is_replenishable,
-- Add new product fields
p.barcode,
p.harmonized_tariff_code,
p.vendor_reference,
p.notions_reference,
p.line,
p.subline,
p.artist,
p.moq,
p.rating,
p.reviews,
p.weight,
p.length,
p.width,
p.height,
p.country_of_origin,
p.location,
p.baskets,
p.notifies,
p.preorder_count,
p.notions_inv_count,
COALESCE(p.price, 0.00) as current_price,
COALESCE(p.regular_price, 0.00) as current_regular_price,
COALESCE(p.cost_price, 0.00) as current_cost_price,
COALESCE(p.landing_cost_price, p.cost_price, 0.00) as current_effective_cost, -- Use landing if available, else cost
p.stock_quantity as current_stock,
p.created_at,
p.first_received,
p.date_last_sold,
p.total_sold as historical_total_sold, -- Add historical total_sold from products table
p.uom -- Assuming UOM logic is handled elsewhere or simple (e.g., 1=each)
FROM public.products p
),
OnOrderInfo AS (
SELECT
pid,
SUM(ordered) AS on_order_qty,
SUM(ordered * po_cost_price) AS on_order_cost,
MIN(expected_date) AS earliest_expected_date
FROM public.purchase_orders
WHERE status IN ('created', 'ordered', 'preordered', 'electronically_sent', 'electronically_ready_send', 'receiving_started')
AND status NOT IN ('canceled', 'done')
GROUP BY pid
),
HistoricalDates AS (
-- Note: Calculating these MIN/MAX values hourly can be slow on large tables.
-- Consider calculating periodically or storing on products if import can populate them.
SELECT
p.pid,
MIN(o.date)::date AS date_first_sold,
MAX(o.date)::date AS max_order_date, -- Use MAX for potential recalc of date_last_sold
-- For first received, use the new receivings table
MIN(r.received_date)::date AS date_first_received_calc,
-- For last received, use the new receivings table
MAX(r.received_date)::date AS date_last_received_calc
FROM public.products p
LEFT JOIN public.orders o ON p.pid = o.pid AND o.quantity > 0 AND o.status NOT IN ('canceled', 'returned')
LEFT JOIN public.receivings r ON p.pid = r.pid
GROUP BY p.pid
),
SnapshotAggregates AS (
SELECT
pid,
-- Get the counts of all available data
COUNT(DISTINCT snapshot_date) AS available_days,
-- Rolling periods with no time constraint - just sum everything we have
SUM(units_sold) AS total_units_sold,
SUM(net_revenue) AS total_net_revenue,
-- Specific time windows using date range boundaries precisely
-- Use _current_date - INTERVAL '6 days' to include 7 days (today + 6 previous days)
-- This ensures we count exactly the right number of days in each period
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '6 days' AND snapshot_date <= _current_date THEN units_sold ELSE 0 END) AS sales_7d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '6 days' AND snapshot_date <= _current_date THEN net_revenue ELSE 0 END) AS revenue_7d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '13 days' AND snapshot_date <= _current_date THEN units_sold ELSE 0 END) AS sales_14d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '13 days' AND snapshot_date <= _current_date THEN net_revenue ELSE 0 END) AS revenue_14d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN units_sold ELSE 0 END) AS sales_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN net_revenue ELSE 0 END) AS revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN cogs ELSE 0 END) AS cogs_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN profit ELSE 0 END) AS profit_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN units_returned ELSE 0 END) AS returns_units_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN returns_revenue ELSE 0 END) AS returns_revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN discounts ELSE 0 END) AS discounts_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN gross_revenue ELSE 0 END) AS gross_revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN gross_regular_revenue ELSE 0 END) AS gross_regular_revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date AND stockout_flag THEN 1 ELSE 0 END) AS stockout_days_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '364 days' AND snapshot_date <= _current_date THEN units_sold ELSE 0 END) AS sales_365d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '364 days' AND snapshot_date <= _current_date THEN net_revenue ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN units_received ELSE 0 END) AS received_qty_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN cost_received ELSE 0 END) AS received_cost_30d,
-- Averages for stock levels - only include dates within the specified period
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_quantity END) AS avg_stock_units_30d,
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_cost END) AS avg_stock_cost_30d,
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_retail END) AS avg_stock_retail_30d,
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_gross END) AS avg_stock_gross_30d,
-- Lifetime - should match total values above
SUM(units_sold) AS lifetime_sales,
SUM(net_revenue) AS lifetime_revenue,
-- Yesterday
SUM(CASE WHEN snapshot_date = _current_date - INTERVAL '1 day' THEN units_sold ELSE 0 END) as yesterday_sales
FROM public.daily_product_snapshots
GROUP BY pid
),
FirstPeriodMetrics AS (
SELECT
pid,
date_first_sold,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '6 days' THEN units_sold ELSE 0 END) AS first_7_days_sales,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '6 days' THEN net_revenue ELSE 0 END) AS first_7_days_revenue,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '29 days' THEN units_sold ELSE 0 END) AS first_30_days_sales,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '29 days' THEN net_revenue ELSE 0 END) AS first_30_days_revenue,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '59 days' THEN units_sold ELSE 0 END) AS first_60_days_sales,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '59 days' THEN net_revenue ELSE 0 END) AS first_60_days_revenue,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '89 days' THEN units_sold ELSE 0 END) AS first_90_days_sales,
SUM(CASE WHEN snapshot_date >= date_first_sold AND snapshot_date <= date_first_sold + INTERVAL '89 days' THEN net_revenue ELSE 0 END) AS first_90_days_revenue
FROM public.daily_product_snapshots ds
JOIN HistoricalDates hd USING(pid)
WHERE date_first_sold IS NOT NULL
AND snapshot_date >= date_first_sold
AND snapshot_date <= date_first_sold + INTERVAL '90 days' -- Limit scan range
GROUP BY pid, date_first_sold
),
Settings AS (
SELECT
p.pid,
COALESCE(sp.lead_time_days, sv.default_lead_time_days, (SELECT setting_value FROM settings_global WHERE setting_key = 'default_lead_time_days')::int, 14) AS effective_lead_time,
COALESCE(sp.days_of_stock, sv.default_days_of_stock, (SELECT setting_value FROM settings_global WHERE setting_key = 'default_days_of_stock')::int, 30) AS effective_days_of_stock,
COALESCE(sp.safety_stock, 0) AS effective_safety_stock, -- Assuming safety stock is units, not days from global for now
COALESCE(sp.exclude_from_forecast, FALSE) AS exclude_forecast
FROM public.products p
LEFT JOIN public.settings_product sp ON p.pid = sp.pid
LEFT JOIN public.settings_vendor sv ON p.vendor = sv.vendor
),
LifetimeRevenue AS (
-- Calculate actual revenue from orders table
SELECT
o.pid,
SUM(o.price * o.quantity - COALESCE(o.discount, 0)) AS lifetime_revenue_from_orders,
SUM(o.quantity) AS lifetime_units_from_orders
FROM public.orders o
WHERE o.status NOT IN ('canceled', 'returned')
AND o.quantity > 0
GROUP BY o.pid
),
PreviousPeriodMetrics AS (
-- Calculate metrics for previous 30-day period for growth comparison
SELECT
pid,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '59 days'
AND snapshot_date < _current_date - INTERVAL '29 days'
THEN units_sold ELSE 0 END) AS sales_prev_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '59 days'
AND snapshot_date < _current_date - INTERVAL '29 days'
THEN net_revenue ELSE 0 END) AS revenue_prev_30d,
-- Year-over-year comparison
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '395 days'
AND snapshot_date < _current_date - INTERVAL '365 days'
THEN units_sold ELSE 0 END) AS sales_30d_last_year,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '395 days'
AND snapshot_date < _current_date - INTERVAL '365 days'
THEN net_revenue ELSE 0 END) AS revenue_30d_last_year
FROM public.daily_product_snapshots
GROUP BY pid
),
DemandVariability AS (
-- Calculate variance and standard deviation of daily sales
SELECT
pid,
COUNT(*) AS days_with_data,
AVG(units_sold) AS avg_daily_sales,
VARIANCE(units_sold) AS sales_variance,
STDDEV(units_sold) AS sales_std_dev,
-- Coefficient of variation
CASE
WHEN AVG(units_sold) > 0 THEN STDDEV(units_sold) / AVG(units_sold)
ELSE NULL
END AS sales_cv
FROM public.daily_product_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
GROUP BY pid
),
ServiceLevels AS (
-- Calculate service level and fill rate metrics
SELECT
pid,
COUNT(*) FILTER (WHERE stockout_flag = true) AS stockout_incidents_30d,
COUNT(*) FILTER (WHERE stockout_flag = true AND units_sold > 0) AS lost_sales_incidents_30d,
-- Service level: percentage of days without stockouts
(1.0 - (COUNT(*) FILTER (WHERE stockout_flag = true)::NUMERIC / NULLIF(COUNT(*), 0))) * 100 AS service_level_30d,
-- Fill rate: units sold / (units sold + potential lost sales)
CASE
WHEN SUM(units_sold) > 0 THEN
(SUM(units_sold)::NUMERIC /
(SUM(units_sold) + SUM(CASE WHEN stockout_flag THEN units_sold * 0.2 ELSE 0 END))) * 100
ELSE NULL
END AS fill_rate_30d
FROM public.daily_product_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
GROUP BY pid
),
SeasonalityAnalysis AS (
-- Simple seasonality detection
SELECT
p.pid,
sp.seasonal_pattern,
sp.seasonality_index,
sp.peak_season
FROM products p
CROSS JOIN LATERAL detect_seasonal_pattern(p.pid) sp
)
-- Final UPSERT into product_metrics
INSERT INTO public.product_metrics (
pid, last_calculated, sku, title, brand, vendor, image_url, is_visible, is_replenishable,
barcode, harmonized_tariff_code, vendor_reference, notions_reference, line, subline, artist,
moq, rating, reviews, weight, length, width, height, country_of_origin, location,
baskets, notifies, preorder_count, notions_inv_count,
current_price, current_regular_price, current_cost_price, current_landing_cost_price,
current_stock, current_stock_cost, current_stock_retail, current_stock_gross,
on_order_qty, on_order_cost, on_order_retail, earliest_expected_date,
date_created, date_first_received, date_last_received, date_first_sold, date_last_sold, age_days,
sales_7d, revenue_7d, sales_14d, revenue_14d, sales_30d, revenue_30d, cogs_30d, profit_30d,
returns_units_30d, returns_revenue_30d, discounts_30d, gross_revenue_30d, gross_regular_revenue_30d,
stockout_days_30d, sales_365d, revenue_365d,
avg_stock_units_30d, avg_stock_cost_30d, avg_stock_retail_30d, avg_stock_gross_30d,
received_qty_30d, received_cost_30d,
lifetime_sales, lifetime_revenue, lifetime_revenue_quality,
first_7_days_sales, first_7_days_revenue, first_30_days_sales, first_30_days_revenue,
first_60_days_sales, first_60_days_revenue, first_90_days_sales, first_90_days_revenue,
asp_30d, acp_30d, avg_ros_30d, avg_sales_per_day_30d, avg_sales_per_month_30d,
margin_30d, markup_30d, gmroi_30d, stockturn_30d, return_rate_30d, discount_rate_30d,
stockout_rate_30d, markdown_30d, markdown_rate_30d, sell_through_30d,
-- avg_lead_time_days, -- Calculated periodically
-- abc_class, -- Calculated periodically
sales_velocity_daily, config_lead_time, config_days_of_stock, config_safety_stock,
planning_period_days, lead_time_forecast_units, days_of_stock_forecast_units,
planning_period_forecast_units, lead_time_closing_stock, days_of_stock_closing_stock,
replenishment_needed_raw, replenishment_units, replenishment_cost, replenishment_retail, replenishment_profit,
to_order_units, forecast_lost_sales_units, forecast_lost_revenue,
stock_cover_in_days, po_cover_in_days, sells_out_in_days, replenish_date,
overstocked_units, overstocked_cost, overstocked_retail, is_old_stock,
yesterday_sales,
status, -- Add status field for calculated status
-- New fields
sales_growth_30d_vs_prev, revenue_growth_30d_vs_prev,
sales_growth_yoy, revenue_growth_yoy,
sales_variance_30d, sales_std_dev_30d, sales_cv_30d, demand_pattern,
fill_rate_30d, stockout_incidents_30d, service_level_30d, lost_sales_incidents_30d,
seasonality_index, seasonal_pattern, peak_season
)
SELECT
ci.pid, _start_time, ci.sku, ci.title, ci.brand, ci.vendor, ci.image_url, ci.is_visible, ci.is_replenishable,
ci.barcode, ci.harmonized_tariff_code, ci.vendor_reference, ci.notions_reference, ci.line, ci.subline, ci.artist,
ci.moq, ci.rating, ci.reviews, ci.weight, ci.length, ci.width, ci.height, ci.country_of_origin, ci.location,
ci.baskets, ci.notifies, ci.preorder_count, ci.notions_inv_count,
ci.current_price, ci.current_regular_price, ci.current_cost_price, ci.current_effective_cost,
ci.current_stock, ci.current_stock * ci.current_effective_cost, ci.current_stock * ci.current_price, ci.current_stock * ci.current_regular_price,
COALESCE(ooi.on_order_qty, 0), COALESCE(ooi.on_order_cost, 0.00), COALESCE(ooi.on_order_qty, 0) * ci.current_price, ooi.earliest_expected_date,
ci.created_at::date, COALESCE(ci.first_received::date, hd.date_first_received_calc), hd.date_last_received_calc, hd.date_first_sold, COALESCE(ci.date_last_sold, hd.max_order_date),
CASE
WHEN ci.created_at IS NULL AND hd.date_first_sold IS NULL THEN 0
WHEN ci.created_at IS NULL THEN (_current_date - hd.date_first_sold)::integer
WHEN hd.date_first_sold IS NULL THEN (_current_date - ci.created_at::date)::integer
ELSE (_current_date - LEAST(ci.created_at::date, hd.date_first_sold))::integer
END AS age_days,
sa.sales_7d, sa.revenue_7d, sa.sales_14d, sa.revenue_14d, sa.sales_30d, sa.revenue_30d, sa.cogs_30d, sa.profit_30d,
sa.returns_units_30d, sa.returns_revenue_30d, sa.discounts_30d, sa.gross_revenue_30d, sa.gross_regular_revenue_30d,
sa.stockout_days_30d, sa.sales_365d, sa.revenue_365d,
sa.avg_stock_units_30d, sa.avg_stock_cost_30d, sa.avg_stock_retail_30d, sa.avg_stock_gross_30d,
sa.received_qty_30d, sa.received_cost_30d,
-- Use total_sold from products table as the source of truth for lifetime sales
-- This includes all historical data from the production database
ci.historical_total_sold AS lifetime_sales,
-- Calculate lifetime revenue using actual historical prices where available
CASE
WHEN lr.lifetime_revenue_from_orders IS NOT NULL THEN
-- We have some order history - use it plus estimate for remaining
lr.lifetime_revenue_from_orders +
(GREATEST(0, ci.historical_total_sold - COALESCE(lr.lifetime_units_from_orders, 0)) *
COALESCE(
-- Use oldest known price from snapshots as proxy
(SELECT revenue_7d / NULLIF(sales_7d, 0)
FROM daily_product_snapshots
WHERE pid = ci.pid AND sales_7d > 0
ORDER BY snapshot_date ASC
LIMIT 1),
ci.current_price
))
ELSE
-- No order history - estimate using current price
ci.historical_total_sold * ci.current_price
END AS lifetime_revenue,
CASE
WHEN lr.lifetime_units_from_orders >= ci.historical_total_sold * 0.9 THEN 'exact'
WHEN lr.lifetime_units_from_orders >= ci.historical_total_sold * 0.5 THEN 'partial'
ELSE 'estimated'
END AS lifetime_revenue_quality,
fpm.first_7_days_sales, fpm.first_7_days_revenue, fpm.first_30_days_sales, fpm.first_30_days_revenue,
fpm.first_60_days_sales, fpm.first_60_days_revenue, fpm.first_90_days_sales, fpm.first_90_days_revenue,
sa.revenue_30d / NULLIF(sa.sales_30d, 0) AS asp_30d,
sa.cogs_30d / NULLIF(sa.sales_30d, 0) AS acp_30d,
sa.profit_30d / NULLIF(sa.sales_30d, 0) AS avg_ros_30d,
sa.sales_30d / 30.0 AS avg_sales_per_day_30d,
sa.sales_30d AS avg_sales_per_month_30d, -- Using 30d sales as proxy for month
(sa.profit_30d / NULLIF(sa.revenue_30d, 0)) * 100 AS margin_30d,
(sa.profit_30d / NULLIF(sa.cogs_30d, 0)) * 100 AS markup_30d,
sa.profit_30d / NULLIF(sa.avg_stock_cost_30d, 0) AS gmroi_30d,
sa.sales_30d / NULLIF(sa.avg_stock_units_30d, 0) AS stockturn_30d,
(sa.returns_units_30d / NULLIF(sa.sales_30d + sa.returns_units_30d, 0)) * 100 AS return_rate_30d,
(sa.discounts_30d / NULLIF(sa.gross_revenue_30d, 0)) * 100 AS discount_rate_30d,
(sa.stockout_days_30d / 30.0) * 100 AS stockout_rate_30d,
sa.gross_regular_revenue_30d - sa.gross_revenue_30d AS markdown_30d,
((sa.gross_regular_revenue_30d - sa.gross_revenue_30d) / NULLIF(sa.gross_regular_revenue_30d, 0)) * 100 AS markdown_rate_30d,
-- Fix sell-through rate: Industry standard is Units Sold / (Beginning Inventory + Units Received)
-- Approximating beginning inventory as current stock + units sold - units received
(sa.sales_30d / NULLIF(
ci.current_stock + sa.sales_30d + sa.returns_units_30d - sa.received_qty_30d,
0
)) * 100 AS sell_through_30d,
-- Forecasting intermediate values
-- Use the calculate_sales_velocity function instead of repetitive calculation
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) AS sales_velocity_daily,
s.effective_lead_time AS config_lead_time,
s.effective_days_of_stock AS config_days_of_stock,
s.effective_safety_stock AS config_safety_stock,
(s.effective_lead_time + s.effective_days_of_stock) AS planning_period_days,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time AS lead_time_forecast_units,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock AS days_of_stock_forecast_units,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * (s.effective_lead_time + s.effective_days_of_stock) AS planning_period_forecast_units,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time)) AS lead_time_closing_stock,
((ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock) AS days_of_stock_closing_stock,
((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0) AS replenishment_needed_raw,
-- Final Forecasting / Replenishment Metrics
CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS replenishment_units,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_effective_cost AS replenishment_cost,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_price AS replenishment_retail,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * (ci.current_price - ci.current_effective_cost) AS replenishment_profit,
-- To Order (Apply MOQ/UOM logic here if needed, otherwise equals replenishment)
CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS to_order_units,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) AS forecast_lost_sales_units,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) * ci.current_price AS forecast_lost_revenue,
ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS stock_cover_in_days,
COALESCE(ooi.on_order_qty, 0) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS po_cover_in_days,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS sells_out_in_days,
-- Replenish Date: Date when stock is projected to hit safety stock, minus lead time
CASE
WHEN calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) > 0
THEN _current_date + FLOOR(GREATEST(0, ci.current_stock - s.effective_safety_stock) / calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int))::int - s.effective_lead_time
ELSE NULL
END AS replenish_date,
GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))::int AS overstocked_units,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))) * ci.current_effective_cost AS overstocked_cost,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))) * ci.current_price AS overstocked_retail,
-- Old Stock Flag
(ci.created_at::date < _current_date - INTERVAL '60 day') AND
(COALESCE(ci.date_last_sold, hd.max_order_date) IS NULL OR COALESCE(ci.date_last_sold, hd.max_order_date) < _current_date - INTERVAL '60 day') AND
(hd.date_last_received_calc IS NULL OR hd.date_last_received_calc < _current_date - INTERVAL '60 day') AND
COALESCE(ooi.on_order_qty, 0) = 0
AS is_old_stock,
sa.yesterday_sales,
-- Calculate status using direct CASE statements (inline logic)
CASE
-- Non-replenishable items default to Healthy
WHEN NOT ci.is_replenishable THEN 'Healthy'
-- Calculate lead time and thresholds
ELSE
CASE
-- Check for overstock first
WHEN GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock))) > 0 THEN 'Overstock'
-- Check for Critical stock
WHEN ci.current_stock <= 0 OR
(ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) <= 0 THEN 'Critical'
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
-- Check for reorder soon
WHEN ((ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) + 7) THEN
CASE
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
ELSE 'Reorder Soon'
END
-- Check for 'At Risk' - old stock
WHEN (ci.created_at::date < _current_date - INTERVAL '60 day') AND
(COALESCE(ci.date_last_sold, hd.max_order_date) IS NULL OR COALESCE(ci.date_last_sold, hd.max_order_date) < _current_date - INTERVAL '60 day') AND
(hd.date_last_received_calc IS NULL OR hd.date_last_received_calc < _current_date - INTERVAL '60 day') AND
COALESCE(ooi.on_order_qty, 0) = 0 THEN 'At Risk'
-- Check for 'At Risk' - hasn't sold in a long time
WHEN COALESCE(ci.date_last_sold, hd.max_order_date) IS NOT NULL
AND COALESCE(ci.date_last_sold, hd.max_order_date) < (_current_date - INTERVAL '90 days')
AND (CASE
WHEN ci.created_at IS NULL AND hd.date_first_sold IS NULL THEN 0
WHEN ci.created_at IS NULL THEN (_current_date - hd.date_first_sold)::integer
WHEN hd.date_first_sold IS NULL THEN (_current_date - ci.created_at::date)::integer
ELSE (_current_date - LEAST(ci.created_at::date, hd.date_first_sold))::integer
END) > 180 THEN 'At Risk'
-- Very high stock cover is at risk too
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) > 365 THEN 'At Risk'
-- New products (less than 30 days old)
WHEN (CASE
WHEN ci.created_at IS NULL AND hd.date_first_sold IS NULL THEN 0
WHEN ci.created_at IS NULL THEN (_current_date - hd.date_first_sold)::integer
WHEN hd.date_first_sold IS NULL THEN (_current_date - ci.created_at::date)::integer
ELSE (_current_date - LEAST(ci.created_at::date, hd.date_first_sold))::integer
END) <= 30 THEN 'New'
-- If none of the above, assume Healthy
ELSE 'Healthy'
END
END AS status,
-- Growth Metrics (P3) - using safe_divide and std_numeric for consistency
std_numeric(safe_divide((sa.sales_30d - ppm.sales_prev_30d) * 100.0, ppm.sales_prev_30d), 2) AS sales_growth_30d_vs_prev,
std_numeric(safe_divide((sa.revenue_30d - ppm.revenue_prev_30d) * 100.0, ppm.revenue_prev_30d), 2) AS revenue_growth_30d_vs_prev,
std_numeric(safe_divide((sa.sales_30d - ppm.sales_30d_last_year) * 100.0, ppm.sales_30d_last_year), 2) AS sales_growth_yoy,
std_numeric(safe_divide((sa.revenue_30d - ppm.revenue_30d_last_year) * 100.0, ppm.revenue_30d_last_year), 2) AS revenue_growth_yoy,
-- Demand Variability (P3)
std_numeric(dv.sales_variance, 2) AS sales_variance_30d,
std_numeric(dv.sales_std_dev, 2) AS sales_std_dev_30d,
std_numeric(dv.sales_cv, 2) AS sales_cv_30d,
classify_demand_pattern(dv.avg_daily_sales, dv.sales_cv) AS demand_pattern,
-- Service Levels (P5)
std_numeric(COALESCE(sl.fill_rate_30d, 100), 2) AS fill_rate_30d,
COALESCE(sl.stockout_incidents_30d, 0)::int AS stockout_incidents_30d,
std_numeric(COALESCE(sl.service_level_30d, 100), 2) AS service_level_30d,
COALESCE(sl.lost_sales_incidents_30d, 0)::int AS lost_sales_incidents_30d,
-- Seasonality (P5)
std_numeric(season.seasonality_index, 2) AS seasonality_index,
COALESCE(season.seasonal_pattern, 'none') AS seasonal_pattern,
season.peak_season
FROM CurrentInfo ci
LEFT JOIN OnOrderInfo ooi ON ci.pid = ooi.pid
LEFT JOIN HistoricalDates hd ON ci.pid = hd.pid
LEFT JOIN SnapshotAggregates sa ON ci.pid = sa.pid
LEFT JOIN FirstPeriodMetrics fpm ON ci.pid = fpm.pid
LEFT JOIN Settings s ON ci.pid = s.pid
LEFT JOIN LifetimeRevenue lr ON ci.pid = lr.pid
LEFT JOIN PreviousPeriodMetrics ppm ON ci.pid = ppm.pid
LEFT JOIN DemandVariability dv ON ci.pid = dv.pid
LEFT JOIN ServiceLevels sl ON ci.pid = sl.pid
LEFT JOIN SeasonalityAnalysis season ON ci.pid = season.pid
WHERE s.exclude_forecast IS FALSE OR s.exclude_forecast IS NULL -- Exclude products explicitly marked
ON CONFLICT (pid) DO UPDATE SET
last_calculated = EXCLUDED.last_calculated,
sku = EXCLUDED.sku, title = EXCLUDED.title, brand = EXCLUDED.brand, vendor = EXCLUDED.vendor, image_url = EXCLUDED.image_url, is_visible = EXCLUDED.is_visible, is_replenishable = EXCLUDED.is_replenishable,
barcode = EXCLUDED.barcode, harmonized_tariff_code = EXCLUDED.harmonized_tariff_code, vendor_reference = EXCLUDED.vendor_reference, notions_reference = EXCLUDED.notions_reference, line = EXCLUDED.line, subline = EXCLUDED.subline, artist = EXCLUDED.artist,
moq = EXCLUDED.moq, rating = EXCLUDED.rating, reviews = EXCLUDED.reviews, weight = EXCLUDED.weight, length = EXCLUDED.length, width = EXCLUDED.width, height = EXCLUDED.height, country_of_origin = EXCLUDED.country_of_origin, location = EXCLUDED.location,
baskets = EXCLUDED.baskets, notifies = EXCLUDED.notifies, preorder_count = EXCLUDED.preorder_count, notions_inv_count = EXCLUDED.notions_inv_count,
current_price = EXCLUDED.current_price, current_regular_price = EXCLUDED.current_regular_price, current_cost_price = EXCLUDED.current_cost_price, current_landing_cost_price = EXCLUDED.current_landing_cost_price,
current_stock = EXCLUDED.current_stock, current_stock_cost = EXCLUDED.current_stock_cost, current_stock_retail = EXCLUDED.current_stock_retail, current_stock_gross = EXCLUDED.current_stock_gross,
on_order_qty = EXCLUDED.on_order_qty, on_order_cost = EXCLUDED.on_order_cost, on_order_retail = EXCLUDED.on_order_retail, earliest_expected_date = EXCLUDED.earliest_expected_date,
date_created = EXCLUDED.date_created, date_first_received = EXCLUDED.date_first_received, date_last_received = EXCLUDED.date_last_received, date_first_sold = EXCLUDED.date_first_sold, date_last_sold = EXCLUDED.date_last_sold, age_days = EXCLUDED.age_days,
sales_7d = EXCLUDED.sales_7d, revenue_7d = EXCLUDED.revenue_7d, sales_14d = EXCLUDED.sales_14d, revenue_14d = EXCLUDED.revenue_14d, sales_30d = EXCLUDED.sales_30d, revenue_30d = EXCLUDED.revenue_30d, cogs_30d = EXCLUDED.cogs_30d, profit_30d = EXCLUDED.profit_30d,
returns_units_30d = EXCLUDED.returns_units_30d, returns_revenue_30d = EXCLUDED.returns_revenue_30d, discounts_30d = EXCLUDED.discounts_30d, gross_revenue_30d = EXCLUDED.gross_revenue_30d, gross_regular_revenue_30d = EXCLUDED.gross_regular_revenue_30d,
stockout_days_30d = EXCLUDED.stockout_days_30d, sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
avg_stock_units_30d = EXCLUDED.avg_stock_units_30d, avg_stock_cost_30d = EXCLUDED.avg_stock_cost_30d, avg_stock_retail_30d = EXCLUDED.avg_stock_retail_30d, avg_stock_gross_30d = EXCLUDED.avg_stock_gross_30d,
received_qty_30d = EXCLUDED.received_qty_30d, received_cost_30d = EXCLUDED.received_cost_30d,
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue, lifetime_revenue_quality = EXCLUDED.lifetime_revenue_quality,
first_7_days_sales = EXCLUDED.first_7_days_sales, first_7_days_revenue = EXCLUDED.first_7_days_revenue, first_30_days_sales = EXCLUDED.first_30_days_sales, first_30_days_revenue = EXCLUDED.first_30_days_revenue,
first_60_days_sales = EXCLUDED.first_60_days_sales, first_60_days_revenue = EXCLUDED.first_60_days_revenue, first_90_days_sales = EXCLUDED.first_90_days_sales, first_90_days_revenue = EXCLUDED.first_90_days_revenue,
asp_30d = EXCLUDED.asp_30d, acp_30d = EXCLUDED.acp_30d, avg_ros_30d = EXCLUDED.avg_ros_30d, avg_sales_per_day_30d = EXCLUDED.avg_sales_per_day_30d, avg_sales_per_month_30d = EXCLUDED.avg_sales_per_month_30d,
margin_30d = EXCLUDED.margin_30d, markup_30d = EXCLUDED.markup_30d, gmroi_30d = EXCLUDED.gmroi_30d, stockturn_30d = EXCLUDED.stockturn_30d, return_rate_30d = EXCLUDED.return_rate_30d, discount_rate_30d = EXCLUDED.discount_rate_30d,
stockout_rate_30d = EXCLUDED.stockout_rate_30d, markdown_30d = EXCLUDED.markdown_30d, markdown_rate_30d = EXCLUDED.markdown_rate_30d, sell_through_30d = EXCLUDED.sell_through_30d,
-- avg_lead_time_days = EXCLUDED.avg_lead_time_days, -- Updated Periodically
-- abc_class = EXCLUDED.abc_class, -- Updated Periodically
sales_velocity_daily = EXCLUDED.sales_velocity_daily, config_lead_time = EXCLUDED.config_lead_time, config_days_of_stock = EXCLUDED.config_days_of_stock, config_safety_stock = EXCLUDED.config_safety_stock,
planning_period_days = EXCLUDED.planning_period_days, lead_time_forecast_units = EXCLUDED.lead_time_forecast_units, days_of_stock_forecast_units = EXCLUDED.days_of_stock_forecast_units,
planning_period_forecast_units = EXCLUDED.planning_period_forecast_units, lead_time_closing_stock = EXCLUDED.lead_time_closing_stock, days_of_stock_closing_stock = EXCLUDED.days_of_stock_closing_stock,
replenishment_needed_raw = EXCLUDED.replenishment_needed_raw, replenishment_units = EXCLUDED.replenishment_units, replenishment_cost = EXCLUDED.replenishment_cost, replenishment_retail = EXCLUDED.replenishment_retail, replenishment_profit = EXCLUDED.replenishment_profit,
to_order_units = EXCLUDED.to_order_units, forecast_lost_sales_units = EXCLUDED.forecast_lost_sales_units, forecast_lost_revenue = EXCLUDED.forecast_lost_revenue,
stock_cover_in_days = EXCLUDED.stock_cover_in_days, po_cover_in_days = EXCLUDED.po_cover_in_days, sells_out_in_days = EXCLUDED.sells_out_in_days, replenish_date = EXCLUDED.replenish_date,
overstocked_units = EXCLUDED.overstocked_units, overstocked_cost = EXCLUDED.overstocked_cost, overstocked_retail = EXCLUDED.overstocked_retail, is_old_stock = EXCLUDED.is_old_stock,
yesterday_sales = EXCLUDED.yesterday_sales,
status = EXCLUDED.status,
sales_growth_30d_vs_prev = EXCLUDED.sales_growth_30d_vs_prev,
revenue_growth_30d_vs_prev = EXCLUDED.revenue_growth_30d_vs_prev,
sales_growth_yoy = EXCLUDED.sales_growth_yoy,
revenue_growth_yoy = EXCLUDED.revenue_growth_yoy,
sales_variance_30d = EXCLUDED.sales_variance_30d,
sales_std_dev_30d = EXCLUDED.sales_std_dev_30d,
sales_cv_30d = EXCLUDED.sales_cv_30d,
demand_pattern = EXCLUDED.demand_pattern,
fill_rate_30d = EXCLUDED.fill_rate_30d,
stockout_incidents_30d = EXCLUDED.stockout_incidents_30d,
service_level_30d = EXCLUDED.service_level_30d,
lost_sales_incidents_30d = EXCLUDED.lost_sales_incidents_30d,
seasonality_index = EXCLUDED.seasonality_index,
seasonal_pattern = EXCLUDED.seasonal_pattern,
peak_season = EXCLUDED.peak_season
WHERE -- Only update if at least one key metric has changed
product_metrics.current_stock IS DISTINCT FROM EXCLUDED.current_stock OR
product_metrics.current_price IS DISTINCT FROM EXCLUDED.current_price OR
product_metrics.current_cost_price IS DISTINCT FROM EXCLUDED.current_cost_price OR
product_metrics.on_order_qty IS DISTINCT FROM EXCLUDED.on_order_qty OR
product_metrics.sales_7d IS DISTINCT FROM EXCLUDED.sales_7d OR
product_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
product_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
product_metrics.status IS DISTINCT FROM EXCLUDED.status OR
product_metrics.replenishment_units IS DISTINCT FROM EXCLUDED.replenishment_units OR
product_metrics.stock_cover_in_days IS DISTINCT FROM EXCLUDED.stock_cover_in_days OR
product_metrics.yesterday_sales IS DISTINCT FROM EXCLUDED.yesterday_sales OR
-- Check a few other important fields that might change
product_metrics.date_last_sold IS DISTINCT FROM EXCLUDED.date_last_sold OR
product_metrics.earliest_expected_date IS DISTINCT FROM EXCLUDED.earliest_expected_date OR
product_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
product_metrics.lifetime_revenue_quality IS DISTINCT FROM EXCLUDED.lifetime_revenue_quality
;
-- Update the status table with the timestamp from the START of this run
UPDATE public.calculate_status
SET last_calculation_timestamp = _start_time
WHERE module_name = _module_name;
RAISE NOTICE 'Finished % module. Duration: %', _module_name, clock_timestamp() - _start_time;
END $$;
-- Return metrics about the update operation
WITH update_stats AS (
SELECT
COUNT(*) as total_products,
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
COUNT(*) FILTER (WHERE status = 'Critical') as critical_count,
COUNT(*) FILTER (WHERE status = 'Reorder Soon') as reorder_soon_count,
COUNT(*) FILTER (WHERE status = 'Healthy') as healthy_count,
COUNT(*) FILTER (WHERE status = 'Overstock') as overstock_count,
COUNT(*) FILTER (WHERE status = 'At Risk') as at_risk_count,
COUNT(*) FILTER (WHERE status = 'New') as new_count
FROM public.product_metrics
)
SELECT
rows_processed,
total_products,
critical_count,
reorder_soon_count,
healthy_count,
overstock_count,
at_risk_count,
new_count,
ROUND((rows_processed::numeric / NULLIF(total_products, 0)) * 100, 2) as update_percentage
FROM update_stats;

39
inventory-server/scripts/metrics-new/utils/db.js
View File

@@ -1,39 +0,0 @@
const { Pool } = require('pg');
const path = require('path');
require('dotenv').config({ path: path.resolve(__dirname, '../../..', '.env') });
// Database configuration
const dbConfig = {
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432,
ssl: process.env.DB_SSL === 'true',
// Add performance optimizations
max: 10, // connection pool max size
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 60000
};
// Create a single pool instance to be reused
const pool = new Pool(dbConfig);
// Add event handlers for pool
pool.on('error', (err, client) => {
console.error('Unexpected error on idle client', err);
});
async function getConnection() {
return await pool.connect();
}
async function closePool() {
await pool.end();
}
module.exports = {
dbConfig,
getConnection,
closePool
};

183
inventory-server/scripts/metrics-new/utils/progress.js
View File

@@ -1,183 +0,0 @@
const fs = require('fs');
const path = require('path');
// Helper function to format elapsed time
function formatElapsedTime(startTime) {
let elapsed;
// If startTime is a timestamp (number representing milliseconds since epoch)
if (typeof startTime === 'number') {
// Check if it's a timestamp (will be a large number like 1700000000000)
if (startTime > 1000000000) { // timestamps are in milliseconds since 1970
elapsed = Date.now() - startTime;
} else {
// Assume it's already elapsed milliseconds
elapsed = startTime;
}
} else if (startTime instanceof Date) {
elapsed = Date.now() - startTime.getTime();
} else {
// Default to 0 if invalid input
elapsed = 0;
}
const seconds = Math.floor(elapsed / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
if (hours > 0) {
return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
} else if (minutes > 0) {
return `${minutes}m ${seconds % 60}s`;
} else {
return `${seconds}s`;
}
}
// Helper function to estimate remaining time
function estimateRemaining(startTime, current, total) {
// Handle edge cases
if (!current || current === 0 || !total || total === 0 || current >= total) {
return null;
}
// Calculate elapsed time in milliseconds
const elapsed = Date.now() - startTime;
if (elapsed <= 0) return null;
// Calculate rate (items per millisecond)
const rate = current / elapsed;
if (rate <= 0) return null;
// Calculate remaining time in milliseconds
const remaining = (total - current) / rate;
// Convert to readable format
const seconds = Math.floor(remaining / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
if (hours > 0) {
return `${hours}h ${minutes % 60}m`;
} else if (minutes > 0) {
return `${minutes}m ${seconds % 60}s`;
} else {
return `${seconds}s`;
}
}
// Helper function to calculate rate
function calculateRate(startTime, current) {
const elapsed = (Date.now() - startTime) / 1000; // Convert to seconds
return elapsed > 0 ? Math.round(current / elapsed) : 0;
}
// Set up logging
const LOG_DIR = path.join(__dirname, '../../../logs');
const ERROR_LOG = path.join(LOG_DIR, 'import-errors.log');
const IMPORT_LOG = path.join(LOG_DIR, 'import.log');
const STATUS_FILE = path.join(LOG_DIR, 'metrics-status.json');
// Ensure log directory exists
if (!fs.existsSync(LOG_DIR)) {
fs.mkdirSync(LOG_DIR, { recursive: true });
}
// Helper function to log errors
function logError(error, context = '') {
const timestamp = new Date().toISOString();
const errorMessage = `[${timestamp}] ${context}\nError: ${error.message}\nStack: ${error.stack}\n\n`;
// Log to error file
fs.appendFileSync(ERROR_LOG, errorMessage);
// Also log to console
console.error(`\n${context}\nError: ${error.message}`);
}
// Helper function to log import progress
function logImport(message) {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] ${message}\n`;
fs.appendFileSync(IMPORT_LOG, logMessage);
}
// Helper function to output progress
function outputProgress(data) {
// Save progress to file for resumption
saveProgress(data);
// Format as SSE event
const event = {
progress: data
};
// Always send to stdout for frontend
process.stdout.write(JSON.stringify(event) + '\n');
// Log significant events to disk
const isSignificant =
// Operation starts
(data.operation && !data.current) ||
// Operation completions and errors
data.status === 'complete' ||
data.status === 'error' ||
// Major phase changes
data.operation?.includes('Starting ABC classification') ||
data.operation?.includes('Starting time-based aggregates') ||
data.operation?.includes('Starting vendor metrics');
if (isSignificant) {
logImport(`${data.operation || 'Operation'}${data.message ? ': ' + data.message : ''}${data.error ? ' Error: ' + data.error : ''}${data.status ? ' Status: ' + data.status : ''}`);
}
}
function saveProgress(progress) {
try {
fs.writeFileSync(STATUS_FILE, JSON.stringify({
...progress,
timestamp: Date.now()
}));
} catch (err) {
console.error('Failed to save progress:', err);
}
}
function clearProgress() {
try {
if (fs.existsSync(STATUS_FILE)) {
fs.unlinkSync(STATUS_FILE);
}
} catch (err) {
console.error('Failed to clear progress:', err);
}
}
function getProgress() {
try {
if (fs.existsSync(STATUS_FILE)) {
const progress = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf8'));
// Check if the progress is still valid (less than 1 hour old)
if (progress.timestamp && Date.now() - progress.timestamp < 3600000) {
return progress;
} else {
// Clear old progress
clearProgress();
}
}
} catch (err) {
console.error('Failed to read progress:', err);
clearProgress();
}
return null;
}
module.exports = {
formatElapsedTime,
estimateRemaining,
calculateRate,
logError,
logImport,
outputProgress,
saveProgress,
clearProgress,
getProgress
};