matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import/calculations improvements

Browse Source
This commit is contained in:
Matt
2026-06-11 19:32:20 -04:00
parent 3b2f51e6b8
commit 069a44bd54
19 changed files with 1175 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inventory-server/scripts/calculate-metrics-new.js
+4
View File
@@ -76,6 +76,8 @@ if (process.env.DATABASE_URL && typeof process.env.DATABASE_URL === 'string') {
dbConfig = {
connectionString: process.env.DATABASE_URL,
ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
// Required by cancelCalculation(): pg_cancel_backend targets this name
application_name: 'node-metrics-calculator',
// Add performance optimizations
max: 10, // connection pool max size
idleTimeoutMillis: 30000,
@@ -93,6 +95,8 @@ if (process.env.DATABASE_URL && typeof process.env.DATABASE_URL === 'string') {
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432,
ssl: process.env.DB_SSL === 'true',
// Required by cancelCalculation(): pg_cancel_backend targets this name
application_name: 'node-metrics-calculator',
// Add performance optimizations
max: 10, // connection pool max size
idleTimeoutMillis: 30000,
inventory-server/scripts/full-update.js
+16 -6
View File
@@ -1,6 +1,12 @@
const path = require('path');
const fs = require('fs');
const { spawn } = require('child_process');
// Maintenance switch: `touch .pause-auto-update` in inventory-server/ to make the
// recurring full-update a no-op (e.g. during a long manual full re-import or a
// snapshot rebuild). Remove the file to resume.
const PAUSE_FILE = path.join(__dirname, '..', '.pause-auto-update');
function outputProgress(data) {
if (!data.status) {
data = {
@@ -22,12 +28,8 @@ function runScript(scriptPath) {
child.stdout.on('data', (data) => {
const lines = data.toString().split('\n');
lines.filter(line => line.trim()).forEach(line => {
try {
console.log(line); // Pass through the JSON output
output += line + '\n';
} catch (e) {
console.log(line); // If not JSON, just log it directly
}
console.log(line); // Pass through the (usually JSON) output
output += line + '\n';
});
});
@@ -50,6 +52,14 @@ function runScript(scriptPath) {
}
async function fullUpdate() {
if (fs.existsSync(PAUSE_FILE)) {
outputProgress({
status: 'complete',
operation: 'Full update skipped',
message: `Auto-update is paused (${PAUSE_FILE} exists) — remove the file to resume`
});
return;
}
try {
// Step 1: Import from Production
outputProgress({
inventory-server/scripts/import/categories.js
+15 -13
View File
@@ -13,10 +13,14 @@ async function importCategories(prodConnection, localConnection) {
let skippedCategories = [];
try {
// Start a single transaction for the entire import
await localConnection.query('BEGIN');
// Temporarily disable the trigger that's causing problems
// Start a single transaction for the entire import.
// Must use the wrapper's beginTransaction() (dedicated client) — query('BEGIN')
// checks out a client per call, so BEGIN/work/COMMIT would not be guaranteed
// to share a connection.
await localConnection.beginTransaction();
// Temporarily disable the trigger that's causing problems.
// ALTER TABLE ... DISABLE TRIGGER is transactional: a rollback restores it.
await localConnection.query('ALTER TABLE categories DISABLE TRIGGER update_categories_updated_at');
// Process each type in order with its own savepoint
@@ -148,8 +152,11 @@ async function importCategories(prodConnection, localConnection) {
}
}
// Re-enable the trigger INSIDE the transaction so disable/enable are atomic
await localConnection.query('ALTER TABLE categories ENABLE TRIGGER update_categories_updated_at');
// Commit the entire transaction - we'll do this even if we have skipped categories
await localConnection.query('COMMIT');
await localConnection.commit();
// Update sync status
await localConnection.query(`
@@ -158,9 +165,6 @@ async function importCategories(prodConnection, localConnection) {
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
// Re-enable the trigger
await localConnection.query('ALTER TABLE categories ENABLE TRIGGER update_categories_updated_at');
outputProgress({
status: "complete",
@@ -187,12 +191,10 @@ async function importCategories(prodConnection, localConnection) {
} catch (error) {
console.error("Error importing categories:", error);
// Only rollback if we haven't committed yet
// Only rollback if we haven't committed yet. The rollback also restores the
// trigger state (DISABLE TRIGGER was inside the transaction).
try {
await localConnection.query('ROLLBACK');
// Make sure we re-enable the trigger even if there was an error
await localConnection.query('ALTER TABLE categories ENABLE TRIGGER update_categories_updated_at');
await localConnection.rollback();
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
}
inventory-server/scripts/import/daily-deals.js
+4 -3
View File
@@ -24,7 +24,8 @@ async function importDailyDeals(prodConnection, localConnection) {
const startTime = Date.now();
try {
await localConnection.query('BEGIN');
// Wrapper's beginTransaction() pins a dedicated client; query('BEGIN') would not.
await localConnection.beginTransaction();
// Fetch recent daily deals from production (MySQL 5.7, no CTEs)
// Join product_current_prices to get the actual deal price
@@ -127,7 +128,7 @@ async function importDailyDeals(prodConnection, localConnection) {
last_sync_timestamp = NOW()
`);
await localConnection.query('COMMIT');
await localConnection.commit();
outputProgress({
status: "complete",
@@ -149,7 +150,7 @@ async function importDailyDeals(prodConnection, localConnection) {
console.error("Error importing daily deals:", error);
try {
await localConnection.query('ROLLBACK');
await localConnection.rollback();
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
}
inventory-server/scripts/import/orders.js
+108 -103
View File
@@ -1,5 +1,4 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
/**
* Imports orders from a production MySQL database to a local PostgreSQL database.
@@ -28,6 +27,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
22: 'placed_incomplete',
30: 'canceled',
40: 'awaiting_payment',
45: 'payment_pending',
50: 'awaiting_products',
55: 'shipping_later',
56: 'shipping_together',
@@ -35,6 +35,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
61: 'flagged',
62: 'fix_before_pick',
65: 'manual_picking',
67: 'remote_send',
70: 'in_pt',
80: 'picked',
90: 'awaiting_shipment',
@@ -65,6 +66,12 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
console.log('Orders: Using last sync time:', lastSyncTime, '(adjusted:', mysqlSyncTime, ')');
// Capture the next watermark from MySQL's own clock BEFORE querying any data.
// Rows modified while the import runs stay above this watermark for the next
// incremental run (overlap re-imports are harmless upserts); writing NOW()
// after the import finishes would permanently skip them.
const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');
// First get count of order items - Keep MySQL compatible for production
const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total
@@ -100,7 +107,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
COALESCE(NULLIF(TRIM(oi.prod_itemnumber), ''), 'NO-SKU') as SKU,
oi.prod_price as price,
oi.qty_ordered as quantity,
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
oi.stamp as last_modified
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
@@ -131,10 +137,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
await localConnection.query(`
DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS temp_order_costs;
DROP TABLE IF EXISTS temp_main_discounts;
DROP TABLE IF EXISTS temp_item_discounts;
CREATE TEMP TABLE temp_order_items (
@@ -143,7 +147,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
sku TEXT NOT NULL,
price NUMERIC(14, 4) NOT NULL,
quantity INTEGER NOT NULL,
base_discount NUMERIC(14, 4) DEFAULT 0,
PRIMARY KEY (order_id, pid)
);
@@ -160,20 +163,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
PRIMARY KEY (order_id)
);
CREATE TEMP TABLE temp_order_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
discount NUMERIC(14, 4) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TEMP TABLE temp_main_discounts (
order_id INTEGER NOT NULL,
discount_id INTEGER NOT NULL,
discount_amount_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
PRIMARY KEY (order_id, discount_id)
);
CREATE TEMP TABLE temp_item_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
@@ -198,10 +187,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
CREATE INDEX idx_temp_order_items_pid ON temp_order_items(pid);
CREATE INDEX idx_temp_order_meta_order_id ON temp_order_meta(order_id);
CREATE INDEX idx_temp_order_discounts_order_pid ON temp_order_discounts(order_id, pid);
CREATE INDEX idx_temp_order_taxes_order_pid ON temp_order_taxes(order_id, pid);
CREATE INDEX idx_temp_order_costs_order_pid ON temp_order_costs(order_id, pid);
CREATE INDEX idx_temp_main_discounts_discount_id ON temp_main_discounts(discount_id);
CREATE INDEX idx_temp_item_discounts_order_pid ON temp_item_discounts(order_id, pid);
CREATE INDEX idx_temp_item_discounts_discount_id ON temp_item_discounts(discount_id);
`);
@@ -216,21 +203,20 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
await localConnection.beginTransaction();
try {
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
const placeholders = batch.map((_, idx) =>
`($${idx * 6 + 1}, $${idx * 6 + 2}, $${idx * 6 + 3}, $${idx * 6 + 4}, $${idx * 6 + 5}, $${idx * 6 + 6})`
const placeholders = batch.map((_, idx) =>
`($${idx * 5 + 1}, $${idx * 5 + 2}, $${idx * 5 + 3}, $${idx * 5 + 4}, $${idx * 5 + 5})`
).join(",");
const values = batch.flatMap(item => [
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity, item.base_discount
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity
]);
await localConnection.query(`
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity, base_discount)
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
sku = EXCLUDED.sku,
price = EXCLUDED.price,
quantity = EXCLUDED.quantity,
base_discount = EXCLUDED.base_discount
quantity = EXCLUDED.quantity
`, values);
await localConnection.commit();
@@ -337,49 +323,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
};
const processDiscountsBatch = async (batchIds) => {
// First, load main discount records
const [mainDiscounts] = await prodConnection.query(`
SELECT order_id, discount_id, discount_amount_subtotal
FROM order_discounts
WHERE order_id IN (?)
`, [batchIds]);
if (mainDiscounts.length > 0) {
await localConnection.beginTransaction();
try {
for (let j = 0; j < mainDiscounts.length; j += PG_BATCH_SIZE) {
const subBatch = mainDiscounts.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
).join(",");
const values = subBatch.flatMap(d => [
d.order_id,
d.discount_id,
d.discount_amount_subtotal || 0
]);
await localConnection.query(`
INSERT INTO temp_main_discounts (order_id, discount_id, discount_amount_subtotal)
VALUES ${placeholders}
ON CONFLICT (order_id, discount_id) DO UPDATE SET
discount_amount_subtotal = EXCLUDED.discount_amount_subtotal
`, values);
}
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
throw error;
}
}
// Then, load item discount records
// Load item-level discount records. Only which = 2 rows are real per-item
// discount amounts; which = 1 rows store the price of free promo-added
// items and which = 3 rows are usage records (neither is a discount).
// These amounts are NOT included in summary_discount_subtotal, so they
// must be added on top of the prorated subtotal discount unconditionally.
const [discounts] = await prodConnection.query(`
SELECT order_id, pid, discount_id, amount
FROM order_discount_items
WHERE order_id IN (?)
WHERE order_id IN (?) AND which = 2
`, [batchIds]);
if (discounts.length === 0) return;
@@ -418,16 +370,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
`, values);
}
// Create aggregated view with a simpler, safer query that avoids duplicates
await localConnection.query(`
TRUNCATE temp_order_discounts;
INSERT INTO temp_order_discounts (order_id, pid, discount)
SELECT order_id, pid, SUM(amount) as discount
FROM temp_item_discounts
GROUP BY order_id, pid
`);
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
@@ -603,42 +545,54 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
try {
const [orders] = await localConnection.query(`
WITH order_totals AS (
SELECT
SELECT
oi.order_id,
oi.pid,
-- Instead of using ARRAY_AGG which can cause duplicate issues, use SUM with a CASE
SUM(CASE
WHEN COALESCE(md.discount_amount_subtotal, 0) > 0 THEN id.amount
ELSE 0
END) as promo_discount_sum,
-- Item-level promo discounts (which = 2 rows). These live outside
-- summary_discount_subtotal, so they are summed unconditionally.
SUM(COALESCE(id.amount, 0)) as promo_discount_sum,
COALESCE(ot.tax, 0) as total_tax,
COALESCE(oc.costeach, pc.cost_price, oi.price * 0.5) as costeach
FROM temp_order_items oi
LEFT JOIN temp_item_discounts id ON oi.order_id = id.order_id AND oi.pid = id.pid
LEFT JOIN temp_main_discounts md ON id.order_id = md.order_id AND id.discount_id = md.discount_id
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
LEFT JOIN temp_order_costs oc ON oi.order_id = oc.order_id AND oi.pid = oc.pid
LEFT JOIN temp_product_costs pc ON oi.pid = pc.pid
WHERE oi.order_id = ANY($1)
GROUP BY oi.order_id, oi.pid, ot.tax, oc.costeach, pc.cost_price
)
SELECT
SELECT
oi.order_id as order_number,
oi.pid::bigint as pid,
oi.sku,
om.date,
oi.price,
oi.quantity,
-- Discount = prorated order-level subtotal discount + item-level promo
-- discounts, clamped so a sale line can never be discounted below free.
(
-- Prorated Points Discount (e.g. loyalty points applied at order level)
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
CASE WHEN oi.quantity > 0 THEN
LEAST(
(
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
END
+ COALESCE(ot.promo_discount_sum, 0)
),
oi.price * oi.quantity
)
ELSE
(
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
END
+ COALESCE(ot.promo_discount_sum, 0)
)
END
+
-- Specific Item-Level Promo Discount (coupon codes, etc.)
COALESCE(ot.promo_discount_sum, 0)
)::NUMERIC(14, 4) as discount,
COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
false as tax_included,
@@ -765,34 +719,83 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
}
}
// Start a transaction for updating sync status and dropping temp tables
// Reconciliation 2 prep: fetch canceled (15) / combined (16) orders from MySQL
// WITHOUT a date_placed filter — combine_orders zeroes date_placed on the source
// orders, so the main item query can never re-fetch them. Done before opening
// the PG transaction so we don't hold it across a MySQL round-trip.
const [statusSweepRows] = await prodConnection.query(`
SELECT order_id, order_status
FROM _order
WHERE order_status IN (15, 16)
${incrementalUpdate ? 'AND stamp > ?' : ''}
`, incrementalUpdate ? [mysqlSyncTime] : []);
let staleItemsDeleted = 0;
let sweepUpdated = 0;
// Final transaction: reconcile deletions, sweep statuses, update sync status, drop temps
await localConnection.beginTransaction();
try {
// Update sync status
// Reconciliation 1: delete PG item rows that no longer exist in MySQL for the
// orders fetched this run. temp_order_items holds the complete current item
// set of every fetched order (staff edits and unpicked promo items DELETE
// order_items rows in MySQL, which an upsert-only import never removes).
const [reconcileResult] = await localConnection.query(`
DELETE FROM orders o
USING (SELECT DISTINCT order_id FROM temp_order_items) fetched
WHERE o.order_number = fetched.order_id::text -- orders.order_number is TEXT
AND NOT EXISTS (
SELECT 1 FROM temp_order_items t
WHERE t.order_id = fetched.order_id AND t.pid = o.pid
)
`);
staleItemsDeleted = reconcileResult.rowCount || 0;
// Reconciliation 2: mark canceled/combined orders. 'combined' source orders were
// merged into a new order that carries the same items — counting both would
// double-count, so they also get canceled = true (routes filter on canceled).
for (const [code, statusText] of [[15, 'canceled'], [16, 'combined']]) {
const ids = statusSweepRows.filter(r => r.order_status === code).map(r => r.order_id);
for (let i = 0; i < ids.length; i += 5000) {
const chunk = ids.slice(i, i + 5000);
const [sweepResult] = await localConnection.query(`
UPDATE orders
SET status = $1, canceled = true
WHERE order_number = ANY($2::text[])
AND (status IS DISTINCT FROM $1 OR canceled IS DISTINCT FROM true)
`, [statusText, chunk.map(String)]);
sweepUpdated += sweepResult.rowCount || 0;
}
}
// Update sync status with the watermark captured from MySQL BEFORE the
// source queries ran (see sourceNow above).
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('orders', NOW())
VALUES ('orders', $1)
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
last_sync_timestamp = $1
`, [sourceNow]);
// Cleanup temporary tables
await localConnection.query(`
DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS temp_order_costs;
DROP TABLE IF EXISTS temp_main_discounts;
DROP TABLE IF EXISTS temp_item_discounts;
DROP TABLE IF EXISTS temp_product_costs;
`);
// Commit final transaction
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
throw error;
throw error;
}
if (staleItemsDeleted > 0 || sweepUpdated > 0) {
console.log(`Orders: reconciliation removed ${staleItemsDeleted} stale item rows, swept ${sweepUpdated} canceled/combined rows`);
}
return {
@@ -800,6 +803,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
totalImported: Math.floor(importedCount) || 0,
recordsAdded: parseInt(recordsAdded) || 0,
recordsUpdated: parseInt(recordsUpdated) || 0,
recordsDeleted: staleItemsDeleted,
statusSweepUpdated: sweepUpdated,
totalSkipped: skippedOrders.size || 0,
missingProducts: missingProducts.size || 0,
totalProcessed: orderItems.length, // Total order items in source
inventory-server/scripts/import/products.js
+132 -5
View File
@@ -622,6 +622,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
AND t.total_sold IS NOT DISTINCT FROM p.total_sold
AND t.date_online IS NOT DISTINCT FROM p.date_online
AND t.shop_score IS NOT DISTINCT FROM p.shop_score
AND t.categories IS NOT DISTINCT FROM p.categories
`);
// Get count of products that need updating
@@ -662,6 +663,11 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
}
}
// Capture the next watermark from MySQL's own clock BEFORE querying any data.
// Rows modified while the import runs stay above this watermark for the next
// incremental run (overlap re-imports are harmless upserts).
const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');
// Start a transaction to ensure temporary tables persist
await localConnection.beginTransaction();
@@ -927,16 +933,22 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
// legacy PHP backend will stamp onto the PO line item.
await syncSupplierCosts(prodConnection, localConnection);
// Sync category assignments for ALL products. product_category_index has no
// stamp column, so category-only changes never bump any of the incremental
// WHERE timestamps — without this pass PG categories go permanently stale.
await syncProductCategories(prodConnection, localConnection);
// Commit the transaction
await localConnection.commit();
// Update sync status
// Update sync status with the watermark captured from MySQL BEFORE the
// source queries ran (see sourceNow above).
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('products', NOW())
VALUES ('products', $1)
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
last_sync_timestamp = $1
`, [sourceNow]);
return {
status: 'complete',
@@ -1028,11 +1040,126 @@ async function syncSupplierCosts(prodConnection, localConnection) {
return { updated };
}
// Full category-assignment sweep. The incremental product import keys on
// p.stamp / ci.stamp / price / b2b dates — none of which change when a product
// is recategorized in product_category_index (the table has no stamp column).
// This pass compares the canonical GROUP_CONCAT representation against
// products.categories and rewrites product_categories only for changed pids.
// Must run inside the caller's transaction (uses ON COMMIT DROP temp table).
async function syncProductCategories(prodConnection, localConnection) {
outputProgress({
status: "running",
operation: "Products import",
message: "Syncing category assignments"
});
// Same expression as the main import query so representations compare equal
// (GROUP_CONCAT(DISTINCT int) returns values numerically sorted).
const [rows] = await prodConnection.query(`
SELECT
p.pid,
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
AND pc.type IN (10, 20, 11, 21, 12, 13)
AND pci.cat_id NOT IN (16, 17)
THEN pci.cat_id
END) as category_ids
FROM products p
LEFT JOIN product_category_index pci ON p.pid = pci.pid
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
GROUP BY p.pid
`);
if (!rows || rows.length === 0) {
return { updated: 0 };
}
await localConnection.query(`
CREATE TEMP TABLE temp_category_sync (
pid BIGINT PRIMARY KEY,
categories TEXT
) ON COMMIT DROP
`);
const CHUNK = 5000;
for (let i = 0; i < rows.length; i += CHUNK) {
const batch = rows.slice(i, i + CHUNK);
const pids = batch.map(r => r.pid);
const cats = batch.map(r => r.category_ids);
await localConnection.query(
`INSERT INTO temp_category_sync (pid, categories)
SELECT * FROM UNNEST($1::bigint[], $2::text[])
ON CONFLICT (pid) DO NOTHING`,
[pids, cats]
);
}
// Which existing products actually changed?
const [changed] = await localConnection.query(`
SELECT t.pid, t.categories
FROM temp_category_sync t
JOIN products p ON p.pid = t.pid
WHERE t.categories IS DISTINCT FROM p.categories
`);
if (changed.rows.length === 0) {
return { updated: 0 };
}
await localConnection.query(`
UPDATE products p
SET categories = t.categories
FROM temp_category_sync t
WHERE p.pid = t.pid
AND t.categories IS DISTINCT FROM p.categories
`);
// Rewrite the relationship rows for changed products only
const REL_CHUNK = 1000;
for (let i = 0; i < changed.rows.length; i += REL_CHUNK) {
const batch = changed.rows.slice(i, i + REL_CHUNK);
const pids = batch.map(r => r.pid);
await localConnection.query(
'DELETE FROM product_categories WHERE pid = ANY($1)',
[pids]
);
const relPids = [];
const relCats = [];
for (const row of batch) {
if (!row.categories) continue;
for (const catId of row.categories.split(',')) {
if (catId && catId.trim()) {
relPids.push(row.pid);
relCats.push(parseInt(catId.trim(), 10));
}
}
}
if (relPids.length > 0) {
await localConnection.query(`
INSERT INTO product_categories (pid, cat_id)
SELECT * FROM UNNEST($1::bigint[], $2::int[])
ON CONFLICT (pid, cat_id) DO NOTHING
`, [relPids, relCats]);
}
}
outputProgress({
status: "running",
operation: "Products import",
message: `Category assignments updated for ${changed.rows.length} products`
});
return { updated: changed.rows.length };
}
module.exports = {
importProducts,
importMissingProducts,
setupTemporaryTables,
cleanupTemporaryTables,
materializeCalculations,
syncSupplierCosts
syncSupplierCosts,
syncProductCategories
};
inventory-server/scripts/import/purchase-orders.js
+54 -35
View File
@@ -72,6 +72,11 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
console.log('Purchase Orders: Using last sync time:', lastSyncTime, '(adjusted:', mysqlSyncTime, ')');
// Capture the next watermark from MySQL's own clock BEFORE querying any data.
// Rows modified while the import runs stay above this watermark for the next
// incremental run (overlap re-imports are harmless upserts).
const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');
// Create temp tables for processing
await localConnection.query(`
DROP TABLE IF EXISTS temp_purchase_orders;
@@ -267,13 +272,16 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
if (totalPOs === 0) {
console.log('No purchase orders to process, skipping PO import step');
} else {
// Fetch and process POs in batches
let offset = 0;
// Fetch and process POs in batches using keyset pagination on po_id.
// LIMIT/OFFSET over a date_updated predicate silently skips rows when
// concurrent updates shift rows between pages.
let processedPOCount = 0;
let lastPoId = 0;
let allPOsProcessed = false;
while (!allPOsProcessed) {
const [poList] = await prodConnection.query(`
SELECT
SELECT
p.po_id,
p.supplier_id,
s.companyname AS vendor,
@@ -286,21 +294,23 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
FROM po p
LEFT JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
AND p.po_id > ?
${incrementalUpdate ? `
AND (
p.date_updated > ?
OR p.date_ordered > ?
p.date_updated > ?
OR p.date_ordered > ?
OR p.date_estin > ?
)
` : ''}
ORDER BY p.po_id
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
`, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : []);
LIMIT ${PO_BATCH_SIZE}
`, incrementalUpdate ? [lastPoId, mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : [lastPoId]);
if (poList.length === 0) {
allPOsProcessed = true;
break;
}
lastPoId = poList[poList.length - 1].po_id;
// Get products for these POs
const poIds = poList.map(po => po.po_id);
@@ -332,7 +342,11 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
vendor: po.vendor || 'Unknown Vendor',
date: validateDate(po.date_ordered) || validateDate(po.date_created),
expected_date: validateDate(po.date_estin),
status: poStatusMap[po.status] || 'created',
// Unknown codes get a sentinel rather than 'created': defaulting an
// unknown cancel-like code to an OPEN status would inflate on-order
// FIFO (the metrics CTEs whitelist known-open statuses, so a sentinel
// is simply ignored there).
status: poStatusMap[po.status] || `unknown_${po.status}`,
notes: po.notes || '',
long_note: po.long_note || '',
ordered: product.qty_each,
@@ -393,20 +407,20 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
`, values);
}
offset += poList.length;
processedPOCount += poList.length;
totalProcessed += completePOs.length;
outputProgress({
status: "running",
operation: "Purchase orders import",
message: `Processed ${offset} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
current: offset,
message: `Processed ${processedPOCount} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
current: processedPOCount,
total: totalPOs,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, offset, totalPOs),
rate: calculateRate(startTime, offset)
remaining: estimateRemaining(startTime, processedPOCount, totalPOs),
rate: calculateRate(startTime, processedPOCount)
});
if (poList.length < PO_BATCH_SIZE) {
allPOsProcessed = true;
}
@@ -439,13 +453,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
if (totalReceivings === 0) {
console.log('No receivings to process, skipping receivings import step');
} else {
// Fetch and process receivings in batches
offset = 0; // Reset offset for receivings
// Fetch and process receivings in batches (keyset pagination, see POs above)
let processedReceivingCount = 0;
let lastReceivingId = 0;
let allReceivingsProcessed = false;
while (!allReceivingsProcessed) {
const [receivingList] = await prodConnection.query(`
SELECT
SELECT
r.receiving_id,
r.supplier_id,
r.status,
@@ -459,6 +474,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
r.date_checked
FROM receivings r
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
AND r.receiving_id > ?
${incrementalUpdate ? `
AND (
r.date_updated > ?
@@ -466,13 +482,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
)
` : ''}
ORDER BY r.receiving_id
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
`, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime] : []);
LIMIT ${PO_BATCH_SIZE}
`, incrementalUpdate ? [lastReceivingId, mysqlSyncTime, mysqlSyncTime] : [lastReceivingId]);
if (receivingList.length === 0) {
allReceivingsProcessed = true;
break;
}
lastReceivingId = receivingList[receivingList.length - 1].receiving_id;
// Get products for these receivings
const receivingIds = receivingList.map(r => r.receiving_id);
@@ -545,7 +562,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
receiving_created_date: validateDate(product.receiving_created_date),
supplier_id: receiving.supplier_id,
status: receivingStatusMap[receiving.status] || 'created'
// Sentinel for unknown codes — see PO status mapping note above
status: receivingStatusMap[receiving.status] || `unknown_${receiving.status}`
});
}
@@ -600,18 +618,18 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
`, values);
}
offset += receivingList.length;
processedReceivingCount += receivingList.length;
totalProcessed += completeReceivings.length;
outputProgress({
status: "running",
operation: "Purchase orders import",
message: `Processed ${offset} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
current: offset,
message: `Processed ${processedReceivingCount} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
current: processedReceivingCount,
total: totalReceivings,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, offset, totalReceivings),
rate: calculateRate(startTime, offset)
remaining: estimateRemaining(startTime, processedReceivingCount, totalReceivings),
rate: calculateRate(startTime, processedReceivingCount)
});
if (receivingList.length < PO_BATCH_SIZE) {
@@ -829,13 +847,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
receivingRecordsAdded = receivingsResult.rows.filter(r => r.inserted).length;
receivingRecordsUpdated = receivingsResult.rows.filter(r => !r.inserted).length;
// Update sync status
// Update sync status with the watermark captured from MySQL BEFORE the
// source queries ran (see sourceNow above).
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW())
VALUES ('purchase_orders', $1)
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
last_sync_timestamp = $1
`, [sourceNow]);
// Clean up temporary tables
await localConnection.query(`
inventory-server/scripts/import/stock-snapshots.js
+4 -1
View File
@@ -151,7 +151,10 @@ async function importStockSnapshots(prodConnection, localConnection, incremental
recordsAdded += batch.length;
} catch (err) {
// Fail the step: the next incremental starts at MAX(snapshot_date), so a
// swallowed batch error would leave a permanent hole that is never revisited.
console.error(`Error inserting batch at offset ${i} (date range ending ${currentDate}):`, err.message);
throw err;
}
}
@@ -165,7 +168,7 @@ async function importStockSnapshots(prodConnection, localConnection, incremental
current: processedRows,
total: totalRows,
elapsed: formatElapsedTime(startTime),
rate: calculateRate(processedRows, startTime)
rate: calculateRate(startTime, processedRows)
});
}
inventory-server/scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
+21 -13
View File
@@ -10,7 +10,7 @@ DECLARE
_date DATE;
_count INT;
_total_records INT := 0;
_begin_date DATE := (SELECT MIN(date)::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data
_begin_date DATE := (SELECT MIN((date AT TIME ZONE 'America/Chicago'))::date FROM orders WHERE date >= '2020-01-01'); -- Starting point: captures all historical order data (business days, Central time)
_end_date DATE := CURRENT_DATE;
BEGIN
RAISE NOTICE 'Beginning daily snapshots rebuild from % to %. Starting at %', _begin_date, _end_date, _start_time;
@@ -32,26 +32,34 @@ BEGIN
p.sku,
-- Count orders to ensure we only include products with real activity
COUNT(o.id) as order_count,
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned/Combined)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN
COALESCE(
o.costeach,
get_weighted_avg_cost(p.pid, o.date::date),
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date),
p.cost_price
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue,
-- Aggregate Returns (Quantity < 0 or Status = Returned)
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN ABS(o.quantity) ELSE 0 END), 0) AS units_returned,
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue,
-- Returns COGS: cost of returned goods offsets sales COGS
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN
COALESCE(
o.costeach,
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date),
p.cost_price
) * ABS(o.quantity)
ELSE 0 END), 0.00) AS returns_cogs
FROM public.products p
LEFT JOIN public.orders o
ON p.pid = o.pid
AND o.date::date = _date
AND (o.date AT TIME ZONE 'America/Chicago')::date = _date -- business day (Central)
GROUP BY p.pid, p.sku
HAVING COUNT(o.id) > 0 -- Only include products with actual orders for this date
),
@@ -65,7 +73,7 @@ BEGIN
-- Calculate received cost for this day
SUM(r.qty_each * r.cost_each) AS cost_received
FROM public.receivings r
WHERE r.received_date::date = _date
WHERE (r.received_date AT TIME ZONE 'America/Chicago')::date = _date
GROUP BY r.pid
HAVING COUNT(DISTINCT r.receiving_id) > 0 OR SUM(r.qty_each) > 0
),
@@ -120,9 +128,9 @@ BEGIN
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00) AS cogs, -- net of returned goods' cost
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - (COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00)) AS profit,
-- Receiving metrics
COALESCE(rd.units_received, 0),
COALESCE(rd.cost_received, 0.00),
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
+4 -1
View File
@@ -123,7 +123,10 @@ BEGIN
brand_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
brand_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
brand_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
brand_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
brand_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
-- Cost revisions can change profit/cogs with unchanged sales/revenue
brand_metrics.profit_30d IS DISTINCT FROM EXCLUDED.profit_30d OR
brand_metrics.cogs_30d IS DISTINCT FROM EXCLUDED.cogs_30d;
-- Update calculate_status
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
inventory-server/scripts/metrics-new/calculate_category_metrics.sql
+15 -10
View File
@@ -23,17 +23,19 @@ BEGIN
SUM(pm.current_stock) AS current_stock_units,
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
-- Sales metrics with proper filtering
-- Sales metrics — revenue uses plain COALESCE (matching brand/vendor);
-- a positive-only revenue filter while cogs/profit sum everything put
-- the margin numerator and denominator on different row populations.
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(COALESCE(pm.revenue_7d, 0)) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(COALESCE(pm.revenue_30d, 0)) AS revenue_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(COALESCE(pm.revenue_365d, 0)) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
SUM(COALESCE(pm.lifetime_revenue, 0)) AS lifetime_revenue
FROM public.product_categories pc
JOIN public.product_metrics pm ON pc.pid = pm.pid
GROUP BY pc.cat_id
@@ -62,15 +64,15 @@ BEGIN
SUM(pm.current_stock_cost) AS current_stock_cost,
SUM(pm.current_stock_retail) AS current_stock_retail,
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
SUM(COALESCE(pm.revenue_7d, 0)) AS revenue_7d,
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
SUM(COALESCE(pm.revenue_30d, 0)) AS revenue_30d,
SUM(COALESCE(pm.cogs_30d, 0)) AS cogs_30d,
SUM(COALESCE(pm.profit_30d, 0)) AS profit_30d,
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
SUM(COALESCE(pm.revenue_365d, 0)) AS revenue_365d,
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
SUM(COALESCE(pm.lifetime_revenue, 0)) AS lifetime_revenue
FROM CategoryProducts cp
JOIN public.product_metrics pm ON cp.pid = pm.pid
GROUP BY cp.ancestor_cat_id
@@ -200,7 +202,10 @@ BEGIN
category_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
category_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
category_metrics.direct_product_count IS DISTINCT FROM EXCLUDED.direct_product_count OR
category_metrics.direct_sales_30d IS DISTINCT FROM EXCLUDED.direct_sales_30d;
category_metrics.direct_sales_30d IS DISTINCT FROM EXCLUDED.direct_sales_30d OR
-- Cost revisions can change profit/cogs with unchanged sales/revenue
category_metrics.profit_30d IS DISTINCT FROM EXCLUDED.profit_30d OR
category_metrics.cogs_30d IS DISTINCT FROM EXCLUDED.cogs_30d;
-- Update calculate_status
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
inventory-server/scripts/metrics-new/calculate_vendor_metrics.sql
+29 -20
View File
@@ -60,26 +60,31 @@ BEGIN
GROUP BY p.vendor
),
VendorPOAggregates AS (
-- Aggregate PO related stats including lead time calculated from POs to receivings
-- Lead time per PO line = days to its FIRST receiving from the same supplier
-- (within 180 days), then averaged per vendor. Joining each PO line to EVERY
-- later receiving overstated lead time and weighted it toward busy products.
-- Same shape as the per-product calc in update_periodic_metrics.sql.
SELECT
po.vendor,
COUNT(DISTINCT po.po_id) AS po_count_365d,
-- Calculate lead time by averaging the days between PO date and receiving date
AVG(GREATEST(1, CASE
WHEN r.received_date IS NOT NULL AND po.date IS NOT NULL
THEN (r.received_date::date - po.date::date)
ELSE NULL
END))::int AS avg_lead_time_days_hist -- Avg lead time from HISTORICAL received POs
FROM public.purchase_orders po
-- Join to receivings table to find when items were received
LEFT JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id
WHERE po.vendor IS NOT NULL AND po.vendor <> ''
AND po.date >= CURRENT_DATE - INTERVAL '1 year' -- Look at POs created in the last year
AND po.status = 'done' -- Only calculate lead time on completed POs
AND r.received_date IS NOT NULL
AND po.date IS NOT NULL
AND r.received_date >= po.date
GROUP BY po.vendor
vendor,
COUNT(DISTINCT po_id) AS po_count_365d,
ROUND(AVG(GREATEST(1, first_receive_date - po_date)))::int AS avg_lead_time_days_hist
FROM (
SELECT
po.vendor,
po.po_id,
po.pid,
po.date::date AS po_date,
MIN(r.received_date::date) AS first_receive_date
FROM public.purchase_orders po
JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id
AND r.received_date >= po.date
AND r.received_date <= po.date + INTERVAL '180 days'
WHERE po.status = 'done'
AND po.date >= CURRENT_DATE - INTERVAL '1 year'
AND po.vendor IS NOT NULL AND po.vendor <> ''
GROUP BY po.vendor, po.po_id, po.pid, po.date
) po_first_receiving
GROUP BY vendor
),
AllVendors AS (
-- Ensure all vendors from products table are included
@@ -154,7 +159,11 @@ BEGIN
vendor_metrics.on_order_units IS DISTINCT FROM EXCLUDED.on_order_units OR
vendor_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
vendor_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
vendor_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
vendor_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
-- Cost revisions can change profit/cogs with unchanged sales/revenue
vendor_metrics.profit_30d IS DISTINCT FROM EXCLUDED.profit_30d OR
vendor_metrics.cogs_30d IS DISTINCT FROM EXCLUDED.cogs_30d OR
vendor_metrics.avg_lead_time_days IS DISTINCT FROM EXCLUDED.avg_lead_time_days;
-- Update calculate_status
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
inventory-server/scripts/metrics-new/migrations/003_fix_item_promo_discounts_and_business_days.sql
+69
View File
@@ -0,0 +1,69 @@
-- Migration 003: Item-level promo discounts + business-day (America/Chicago) bucketing
-- (applied 2026-06-11, together with the IMPORT_METRICS_FIX_PLAN.md batch)
--
-- PROBLEM 1 — dropped item-level promo discounts (~$26K / 30 days):
-- orders.js applied item-level discounts from order_discount_items only when the
-- parent order_discounts row had discount_amount_subtotal > 0:
-- SUM(CASE WHEN COALESCE(md.discount_amount_subtotal, 0) > 0 THEN id.amount ELSE 0 END)
-- In the PHP source, item-level promo discounts (which = 2) are applied to the order
-- total SEPARATELY from summary_discount_subtotal, so the gate zeroed essentially all
-- of them (90d live check: of 10,010 type-10 promos, 8,070 had item rows but only 8 had
-- discount_amount_subtotal > 0). Net effect: orders.discount understated, net_revenue /
-- profit_30d / margin_30d overstated by ~10% of revenue, discounts_30d ~3x understated.
--
-- FIX (orders.js): fetch only order_discount_items rows with which = 2 (which = 1 rows
-- are prices of free promo-added items, which = 3 are usage records), sum them
-- unconditionally, and clamp each sale line's total discount to price * quantity.
-- temp_main_discounts / temp_order_discounts staging removed (unused after the fix).
--
-- PROBLEM 2 — Europe/Berlin day bucketing:
-- orders.date is timestamptz and the PG server timezone is Europe/Berlin, so ::date
-- casts shifted every order placed after ~5 PM Central onto the NEXT calendar day in
-- daily_product_snapshots (and skewed yesterday_sales, DOW patterns, forecast accuracy).
--
-- FIX (update_daily_snapshots.sql, backfill/rebuild_daily_snapshots.sql,
-- update_product_metrics.sql): every day-bucketing cast is now
-- (ts AT TIME ZONE 'America/Chicago')::date
-- Supporting expression indexes:
-- CREATE INDEX idx_orders_date_chicago ON orders (((date AT TIME ZONE 'America/Chicago')::date));
-- CREATE INDEX idx_receivings_received_chicago ON receivings (((received_date AT TIME ZONE 'America/Chicago')::date));
--
-- ALSO IN THIS BATCH (same re-import/rebuild):
-- * 'combined' order status (code 16) excluded from all sales aggregates, and a sweep
-- in orders.js marks canceled/combined source orders (canceled = true) even though
-- combine_orders zeroes date_placed (Fixes 4/5).
-- * Returns now subtract COGS (returns_cogs) in daily snapshots (Fix 8).
-- * return_rate_30d = returns / sales (Fix 9); gmroi_30d annualized ×12.17 (Fix 10).
-- * stockout/avg-stock/service-level derived from stock_snapshots presence (Fix 7).
--
-- REQUIRED ACTION (cannot be fixed by SQL alone — discount values are baked into rows):
-- 1. Deploy updated orders.js + snapshot SQL files.
-- 2. Pause the recurring import: touch inventory-server/.pause-auto-update
-- 3. FULL orders re-import: INCREMENTAL_UPDATE=false node scripts/import-from-prod.js
-- 4. Rebuild snapshots: psql -f scripts/metrics-new/backfill/rebuild_daily_snapshots.sql
-- 5. Recalculate metrics: node scripts/calculate-metrics-new.js
-- 6. Resume: rm inventory-server/.pause-auto-update
--
-- EXPECTED AFTER RE-IMPORT: margin_30d down ~8-10 points (real, not a data incident),
-- discounts_30d ~3x up, daily sales curves shifted onto correct business days.
--
-- VERIFICATION:
-- (a) PG SUM(discount) over a 30-day window should approximate MySQL
-- Σ summary_discount_subtotal (prorated) + Σ order_discount_items.amount (which=2)
-- over the same orders.
-- (b) Per-day units in daily_product_snapshots should match MySQL
-- SELECT date_placed_onlydate, SUM(qty_ordered) FROM order_items JOIN _order ...
-- WHERE order_status >= 20 GROUP BY 1 (MySQL stores Central days).
-- (c) Migration 002 regression check (discount double-counting) still holds:
SELECT
o.pid,
o.order_number,
o.price,
o.quantity,
o.discount,
(o.price * o.quantity - o.discount) as net_revenue
FROM orders o
WHERE o.pid IN (624756, 614513)
ORDER BY o.date DESC
LIMIT 10;
-- Expected: discount 0 (or genuine promo amount) for regular sales; net close to gross.
inventory-server/scripts/metrics-new/migrations/004_map_remaining_order_statuses.sql
+9
View File
@@ -0,0 +1,9 @@
-- Migration 004: Map order status codes 45 and 67 to text
--
-- Follow-up to 001_map_order_statuses.sql: the orders.js orderStatusMap lacked
-- codes 45 (payment_pending) and 67 (remote_send), so any such orders imported
-- as numeric strings '45' / '67'. orders.js now maps them; this updates any
-- existing rows (a full re-import also fixes them — safe to run either way).
UPDATE orders SET status = 'payment_pending' WHERE status = '45';
UPDATE orders SET status = 'remote_send' WHERE status = '67';
inventory-server/scripts/metrics-new/update_daily_snapshots.sql
+56 -28
View File
@@ -39,50 +39,68 @@ BEGIN
-- 2. Stale detection: existing snapshots where aggregates don't match source data
-- (catches backfilled imports that arrived after snapshot was calculated)
-- 3. Recent recheck: last N days always reprocessed (picks up new orders, corrections)
-- NOTE: all order/receiving timestamps are bucketed into business days using
-- America/Chicago. The PG server timezone is Europe/Berlin, so a bare ::date
-- cast would shift every evening order onto the next day.
FOR _target_date IN
SELECT d FROM (
-- Gap fill: find dates with activity but missing snapshots
SELECT activity_dates.d
FROM (
SELECT DISTINCT date::date AS d FROM public.orders
WHERE date::date >= _backfill_start AND date::date < CURRENT_DATE - _recent_recheck_days
SELECT DISTINCT (date AT TIME ZONE 'America/Chicago')::date AS d FROM public.orders
WHERE (date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
UNION
SELECT DISTINCT received_date::date AS d FROM public.receivings
WHERE received_date::date >= _backfill_start AND received_date::date < CURRENT_DATE - _recent_recheck_days
SELECT DISTINCT (received_date AT TIME ZONE 'America/Chicago')::date AS d FROM public.receivings
WHERE (received_date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (received_date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
) activity_dates
WHERE NOT EXISTS (
SELECT 1 FROM public.daily_product_snapshots dps WHERE dps.snapshot_date = activity_dates.d
)
UNION
-- Stale detection: compare snapshot aggregates against source tables
-- (must bucket identically to SalesData/ReceivingData or every day
-- looks permanently stale)
SELECT snap_agg.snapshot_date AS d
FROM (
SELECT snapshot_date,
COALESCE(SUM(units_received), 0)::bigint AS snap_received,
COALESCE(SUM(units_sold), 0)::bigint AS snap_sold
COALESCE(SUM(units_sold), 0)::bigint AS snap_sold,
ROUND(COALESCE(SUM(net_revenue), 0), 2) AS snap_net_revenue
FROM public.daily_product_snapshots
WHERE snapshot_date >= _backfill_start
AND snapshot_date < CURRENT_DATE - _recent_recheck_days
GROUP BY snapshot_date
) snap_agg
LEFT JOIN (
SELECT received_date::date AS d, SUM(qty_each)::bigint AS actual_received
SELECT (received_date AT TIME ZONE 'America/Chicago')::date AS d, SUM(qty_each)::bigint AS actual_received
FROM public.receivings
WHERE received_date::date >= _backfill_start
AND received_date::date < CURRENT_DATE - _recent_recheck_days
GROUP BY received_date::date
WHERE (received_date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (received_date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
GROUP BY 1
) recv_agg ON snap_agg.snapshot_date = recv_agg.d
LEFT JOIN (
SELECT date::date AS d,
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned')
THEN quantity ELSE 0 END)::bigint AS actual_sold
SELECT (date AT TIME ZONE 'America/Chicago')::date AS d,
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned', 'combined')
THEN quantity ELSE 0 END)::bigint AS actual_sold,
-- Mirrors SalesData's net_revenue (gross - discounts - returns)
-- so price/discount corrections older than the recheck window
-- get repaired, not just unit-count changes.
ROUND(
SUM(CASE WHEN quantity > 0 AND COALESCE(status, 'pending') NOT IN ('canceled', 'returned', 'combined')
THEN price * quantity - discount ELSE 0 END)
- SUM(CASE WHEN quantity < 0 OR COALESCE(status, 'pending') = 'returned'
THEN price * ABS(quantity) ELSE 0 END)
, 2) AS actual_net_revenue
FROM public.orders
WHERE date::date >= _backfill_start
AND date::date < CURRENT_DATE - _recent_recheck_days
GROUP BY date::date
WHERE (date AT TIME ZONE 'America/Chicago')::date >= _backfill_start
AND (date AT TIME ZONE 'America/Chicago')::date < CURRENT_DATE - _recent_recheck_days
GROUP BY 1
) orders_agg ON snap_agg.snapshot_date = orders_agg.d
WHERE snap_agg.snap_received != COALESCE(recv_agg.actual_received, 0)
OR snap_agg.snap_sold != COALESCE(orders_agg.actual_sold, 0)
OR snap_agg.snap_net_revenue != ROUND(COALESCE(orders_agg.actual_net_revenue, 0), 2)
UNION
-- Recent days: always reprocess
SELECT d::date
@@ -116,26 +134,36 @@ BEGIN
p.sku,
-- Track number of orders to ensure we have real data
COUNT(o.id) as order_count,
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, -- Before discount
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN
-- Aggregate Sales (Quantity > 0, Status not Canceled/Returned/Combined)
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.quantity ELSE 0 END), 0) AS units_sold,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.price * o.quantity ELSE 0 END), 0.00) AS gross_revenue_unadjusted, -- Before discount
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN o.discount ELSE 0 END), 0.00) AS discounts,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN
COALESCE(
o.costeach, -- First use order-specific cost if available
get_weighted_avg_cost(p.pid, o.date::date), -- Then use weighted average cost
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date), -- Then use weighted average cost
p.cost_price -- Final fallback to current cost
) * o.quantity
) * o.quantity
ELSE 0 END), 0.00) AS cogs,
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Use current regular price for simplicity here
COALESCE(SUM(CASE WHEN o.quantity > 0 AND COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned', 'combined') THEN p.regular_price * o.quantity ELSE 0 END), 0.00) AS gross_regular_revenue, -- Use current regular price for simplicity here
-- Aggregate Returns (Quantity < 0 or Status = Returned)
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN ABS(o.quantity) ELSE 0 END), 0) AS units_returned,
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN o.price * ABS(o.quantity) ELSE 0 END), 0.00) AS returns_revenue,
-- Returns COGS: returned goods come back into stock, so their cost
-- offsets the sales COGS for the day (margin would otherwise be
-- understated in return-heavy periods).
COALESCE(SUM(CASE WHEN o.quantity < 0 OR COALESCE(o.status, 'pending') = 'returned' THEN
COALESCE(
o.costeach,
get_weighted_avg_cost(p.pid, (o.date AT TIME ZONE 'America/Chicago')::date),
p.cost_price
) * ABS(o.quantity)
ELSE 0 END), 0.00) AS returns_cogs
FROM public.products p -- Start from products to include those with no orders today
JOIN public.orders o -- Changed to INNER JOIN to only process products with orders
ON p.pid = o.pid
AND o.date::date = _target_date -- Cast to date to ensure compatibility regardless of original type
AND (o.date AT TIME ZONE 'America/Chicago')::date = _target_date -- Bucket by business day (Central)
GROUP BY p.pid, p.sku
-- No HAVING clause here - we always want to include all orders
),
@@ -149,7 +177,7 @@ BEGIN
-- Calculate the cost received (qty * cost)
SUM(r.qty_each * r.cost_each) AS cost_received
FROM public.receivings r
WHERE r.received_date::date = _target_date
WHERE (r.received_date AT TIME ZONE 'America/Chicago')::date = _target_date
-- Optional: Filter out canceled receivings if needed
-- AND r.status <> 'canceled'
GROUP BY r.pid
@@ -217,9 +245,9 @@ BEGIN
COALESCE(sd.discounts, 0.00),
COALESCE(sd.returns_revenue, 0.00),
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue,
COALESCE(sd.cogs, 0.00),
COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00) AS cogs, -- net of returned goods' cost
COALESCE(sd.gross_regular_revenue, 0.00),
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - COALESCE(sd.cogs, 0.00) AS profit,
(COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00)) - (COALESCE(sd.cogs, 0.00) - COALESCE(sd.returns_cogs, 0.00)) AS profit,
-- Receiving Metrics (From ReceivingData)
COALESCE(rd.units_received, 0),
COALESCE(rd.cost_received, 0.00),
inventory-server/scripts/metrics-new/update_product_metrics.sql
+169 -68
View File
@@ -131,18 +131,19 @@ BEGIN
HistoricalDates AS (
-- Note: Calculating these MIN/MAX values hourly can be slow on large tables.
-- Consider calculating periodically or storing on products if import can populate them.
-- Dates are bucketed in business time (America/Chicago) to match daily snapshots.
SELECT
p.pid,
MIN(o.date)::date AS date_first_sold,
MAX(o.date)::date AS max_order_date, -- Use MAX for potential recalc of date_last_sold
MIN((o.date AT TIME ZONE 'America/Chicago'))::date AS date_first_sold,
MAX((o.date AT TIME ZONE 'America/Chicago'))::date AS max_order_date, -- Use MAX for potential recalc of date_last_sold
-- For first received, use the new receivings table
MIN(r.received_date)::date AS date_first_received_calc,
MIN((r.received_date AT TIME ZONE 'America/Chicago'))::date AS date_first_received_calc,
-- For last received, use the new receivings table
MAX(r.received_date)::date AS date_last_received_calc
MAX((r.received_date AT TIME ZONE 'America/Chicago'))::date AS date_last_received_calc
FROM public.products p
LEFT JOIN public.orders o ON p.pid = o.pid AND o.quantity > 0 AND o.status NOT IN ('canceled', 'returned')
LEFT JOIN public.orders o ON p.pid = o.pid AND o.quantity > 0 AND o.status NOT IN ('canceled', 'returned', 'combined')
LEFT JOIN public.receivings r ON p.pid = r.pid
GROUP BY p.pid
),
@@ -174,17 +175,19 @@ BEGIN
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN discounts ELSE 0 END) AS discounts_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN gross_revenue ELSE 0 END) AS gross_revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN gross_regular_revenue ELSE 0 END) AS gross_regular_revenue_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date AND stockout_flag THEN 1 ELSE 0 END) AS stockout_days_30d,
-- NOTE: stockout days and avg stock units/cost now come from StockCoverage
-- (stock_snapshots has full daily coverage; these activity-only snapshots
-- only exist on days with sales/receivings, which made stockout_days ~0
-- exactly when stockouts mattered and biased stock averages upward).
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '364 days' AND snapshot_date <= _current_date THEN units_sold ELSE 0 END) AS sales_365d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '364 days' AND snapshot_date <= _current_date THEN net_revenue ELSE 0 END) AS revenue_365d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN units_received ELSE 0 END) AS received_qty_30d,
SUM(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN cost_received ELSE 0 END) AS received_cost_30d,
-- Averages for stock levels - only include dates within the specified period
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_quantity END) AS avg_stock_units_30d,
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_cost END) AS avg_stock_cost_30d,
-- Retail/gross stock averages stay on activity snapshots: stock_snapshots
-- has no eod_stock_retail equivalent (cost-only source table).
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_retail END) AS avg_stock_retail_30d,
AVG(CASE WHEN snapshot_date >= _current_date - INTERVAL '29 days' AND snapshot_date <= _current_date THEN eod_stock_gross END) AS avg_stock_gross_30d,
@@ -240,16 +243,89 @@ BEGIN
LEFT JOIN public.settings_vendor sv ON p.vendor = sv.vendor
),
LifetimeRevenue AS (
-- Calculate actual revenue from orders table
-- Calculate actual revenue from orders table. Negative-quantity rows
-- (returns) are included so lifetime revenue nets out returns;
-- price * quantity is already signed.
SELECT
o.pid,
SUM(o.price * o.quantity - COALESCE(o.discount, 0)) AS lifetime_revenue_from_orders,
SUM(o.quantity) AS lifetime_units_from_orders
FROM public.orders o
WHERE o.status NOT IN ('canceled', 'returned')
AND o.quantity > 0
WHERE o.status NOT IN ('canceled', 'returned', 'combined')
GROUP BY o.pid
),
-- Full-coverage stock presence from stock_snapshots (MySQL snap_product_value).
-- That source only writes rows for products WITH stock on hand, so a product
-- missing from a day the cron ran was out of stock that day. Days before the
-- product was created are not counted against it.
StockCoverage AS (
SELECT
pid,
eligible_days_30d,
days_in_stock_30d,
CASE WHEN eligible_days_30d > 0
THEN GREATEST(0, eligible_days_30d - days_in_stock_30d)
END AS stockout_days_30d,
-- Absent days count as zero stock (the old activity-only average was
-- biased toward in-stock days)
CASE WHEN eligible_days_30d > 0
THEN sum_qty::numeric / eligible_days_30d
END AS avg_stock_units_30d,
CASE WHEN eligible_days_30d > 0
THEN sum_value::numeric / eligible_days_30d
END AS avg_stock_cost_30d
FROM (
SELECT
p.pid,
LEAST(
cal.covered_days,
CASE WHEN p.created_at IS NULL THEN cal.covered_days
ELSE GREATEST(0, (_current_date - GREATEST(p.created_at::date, _current_date - 29) + 1))
END
) AS eligible_days_30d,
COALESCE(pres.days_in_stock, 0) AS days_in_stock_30d,
COALESCE(pres.sum_qty, 0) AS sum_qty,
COALESCE(pres.sum_value, 0) AS sum_value
FROM public.products p
CROSS JOIN (
SELECT COUNT(DISTINCT snapshot_date) AS covered_days
FROM public.stock_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
) cal
LEFT JOIN (
SELECT pid,
COUNT(*) AS days_in_stock,
SUM(stock_quantity) AS sum_qty,
SUM(stock_value) AS sum_value
FROM public.stock_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
GROUP BY pid
) pres ON pres.pid = p.pid
) base
),
-- Sales that happened on out-of-stock days (per the stock snapshot), for
-- lost-sales incidents and the fill-rate heuristic. Restricted to days the
-- stock cron actually ran so e.g. today's sales aren't misread as stockouts.
SalesDayStock AS (
SELECT
dps.pid,
SUM(dps.units_sold) AS units_sold_covered,
COUNT(*) FILTER (WHERE dps.units_sold > 0 AND ss.pid IS NULL) AS lost_sales_incidents_30d,
SUM(CASE WHEN ss.pid IS NULL THEN dps.units_sold ELSE 0 END) AS units_sold_on_stockout_days
FROM public.daily_product_snapshots dps
JOIN (
SELECT DISTINCT snapshot_date FROM public.stock_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
) cal ON cal.snapshot_date = dps.snapshot_date
LEFT JOIN public.stock_snapshots ss
ON ss.pid = dps.pid AND ss.snapshot_date = dps.snapshot_date
WHERE dps.snapshot_date >= _current_date - INTERVAL '29 days'
AND dps.snapshot_date <= _current_date
GROUP BY dps.pid
),
PreviousPeriodMetrics AS (
-- Calculate metrics for previous 30-day period for growth comparison
SELECT
@@ -302,24 +378,43 @@ BEGIN
GROUP BY pid
),
ServiceLevels AS (
-- Calculate service level and fill rate metrics
-- Service level and fill rate built on full-coverage stock data
-- (StockCoverage / SalesDayStock) instead of activity-only snapshots.
SELECT
pid,
COUNT(*) FILTER (WHERE stockout_flag = true) AS stockout_incidents_30d,
COUNT(*) FILTER (WHERE stockout_flag = true AND units_sold > 0) AS lost_sales_incidents_30d,
-- Service level: percentage of days without stockouts
(1.0 - (COUNT(*) FILTER (WHERE stockout_flag = true)::NUMERIC / NULLIF(COUNT(*), 0))) * 100 AS service_level_30d,
-- Fill rate: units sold / (units sold + potential lost sales)
CASE
WHEN SUM(units_sold) > 0 THEN
(SUM(units_sold)::NUMERIC /
(SUM(units_sold) + SUM(CASE WHEN stockout_flag THEN units_sold * 0.2 ELSE 0 END))) * 100
sc.pid,
sc.stockout_days_30d AS stockout_incidents_30d,
sds.lost_sales_incidents_30d,
-- Service level: percentage of covered days the product was in stock
CASE WHEN sc.eligible_days_30d > 0 THEN
(1.0 - (sc.stockout_days_30d::NUMERIC / sc.eligible_days_30d)) * 100
END AS service_level_30d,
-- Fill rate: units sold / (units sold + potential lost sales).
-- The 0.2 lost-sales factor is an arbitrary heuristic: each unit sold on
-- an out-of-stock day is assumed to represent 20% additional missed demand.
CASE
WHEN COALESCE(sds.units_sold_covered, 0) > 0 THEN
(sds.units_sold_covered::NUMERIC /
(sds.units_sold_covered + COALESCE(sds.units_sold_on_stockout_days, 0) * 0.2)) * 100
ELSE NULL
END AS fill_rate_30d
FROM public.daily_product_snapshots
WHERE snapshot_date >= _current_date - INTERVAL '29 days'
AND snapshot_date <= _current_date
GROUP BY pid
FROM StockCoverage sc
LEFT JOIN SalesDayStock sds ON sds.pid = sc.pid
),
ProductVelocity AS (
-- Single source for sales velocity so every replenishment/cover column stays
-- consistent. NULL when the product is excluded from forecasting: excluded
-- products now still get a product_metrics row (they used to be filtered out
-- entirely and vanished from brand/vendor/category rollups), but their
-- forecast-derived columns go NULL / zero.
SELECT
ci.pid,
CASE WHEN COALESCE(s.exclude_forecast, FALSE) THEN NULL
ELSE calculate_sales_velocity(sa.sales_30d::int, COALESCE(sc.stockout_days_30d, 0)::int)
END AS daily
FROM CurrentInfo ci
LEFT JOIN SnapshotAggregates sa ON ci.pid = sa.pid
LEFT JOIN StockCoverage sc ON ci.pid = sc.pid
LEFT JOIN Settings s ON ci.pid = s.pid
),
SeasonalityAnalysis AS (
-- Set-based seasonality detection (replaces per-product function calls)
@@ -424,8 +519,8 @@ BEGIN
END AS age_days,
sa.sales_7d, sa.revenue_7d, sa.sales_14d, sa.revenue_14d, sa.sales_30d, sa.revenue_30d, sa.cogs_30d, sa.profit_30d,
sa.returns_units_30d, sa.returns_revenue_30d, sa.discounts_30d, sa.gross_revenue_30d, sa.gross_regular_revenue_30d,
sa.stockout_days_30d, sa.sales_365d, sa.revenue_365d,
sa.avg_stock_units_30d, sa.avg_stock_cost_30d, sa.avg_stock_retail_30d, sa.avg_stock_gross_30d,
sc.stockout_days_30d, sa.sales_365d, sa.revenue_365d,
sc.avg_stock_units_30d, sc.avg_stock_cost_30d, sa.avg_stock_retail_30d, sa.avg_stock_gross_30d,
sa.received_qty_30d, sa.received_cost_30d,
-- Use total_sold from products table as the source of truth for lifetime sales
-- This includes all historical data from the production database
@@ -463,66 +558,68 @@ BEGIN
sa.sales_30d AS avg_sales_per_month_30d, -- Using 30d sales as proxy for month
(sa.profit_30d / NULLIF(sa.revenue_30d, 0)) * 100 AS margin_30d,
(sa.profit_30d / NULLIF(sa.cogs_30d, 0)) * 100 AS markup_30d,
sa.profit_30d / NULLIF(sa.avg_stock_cost_30d, 0) AS gmroi_30d,
sa.sales_30d / NULLIF(sa.avg_stock_units_30d, 0) AS stockturn_30d,
(sa.returns_units_30d / NULLIF(sa.sales_30d + sa.returns_units_30d, 0)) * 100 AS return_rate_30d,
-- Annualized GMROI (30-day profit extrapolated to a year: × 365/30).
-- Conventional benchmark for healthy retail is ≥ 2-3 on this scale.
(sa.profit_30d / NULLIF(sc.avg_stock_cost_30d, 0)) * 12.17 AS gmroi_30d,
sa.sales_30d / NULLIF(sc.avg_stock_units_30d, 0) AS stockturn_30d,
-- Industry-standard definition: returns / sales (not returns / (sales+returns))
(sa.returns_units_30d / NULLIF(sa.sales_30d, 0)) * 100 AS return_rate_30d,
(sa.discounts_30d / NULLIF(sa.gross_revenue_30d, 0)) * 100 AS discount_rate_30d,
(sa.stockout_days_30d / 30.0) * 100 AS stockout_rate_30d,
(sc.stockout_days_30d::numeric / NULLIF(sc.eligible_days_30d, 0)) * 100 AS stockout_rate_30d,
sa.gross_regular_revenue_30d - sa.gross_revenue_30d AS markdown_30d,
((sa.gross_regular_revenue_30d - sa.gross_revenue_30d) / NULLIF(sa.gross_regular_revenue_30d, 0)) * 100 AS markdown_rate_30d,
-- Sell-through rate: Industry standard is Units Sold / (Beginning Inventory + Units Received)
-- Uses actual snapshot from 30 days ago as beginning stock, falls back to avg_stock_units_30d
(sa.sales_30d / NULLIF(
COALESCE(bs.beginning_stock_30d, sa.avg_stock_units_30d::int, 0) + sa.received_qty_30d,
COALESCE(bs.beginning_stock_30d, sc.avg_stock_units_30d::int, 0) + sa.received_qty_30d,
0
)) * 100 AS sell_through_30d,
-- Forecasting intermediate values
-- Use the calculate_sales_velocity function instead of repetitive calculation
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) AS sales_velocity_daily,
-- Forecasting intermediate values (ProductVelocity; NULL when excluded from forecast)
vel.daily AS sales_velocity_daily,
s.effective_lead_time AS config_lead_time,
s.effective_days_of_stock AS config_days_of_stock,
s.effective_safety_stock AS config_safety_stock,
(s.effective_lead_time + s.effective_days_of_stock) AS planning_period_days,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time AS lead_time_forecast_units,
vel.daily * s.effective_lead_time AS lead_time_forecast_units,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock AS days_of_stock_forecast_units,
vel.daily * s.effective_days_of_stock AS days_of_stock_forecast_units,
calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * (s.effective_lead_time + s.effective_days_of_stock) AS planning_period_forecast_units,
vel.daily * (s.effective_lead_time + s.effective_days_of_stock) AS planning_period_forecast_units,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time)) AS lead_time_closing_stock,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (vel.daily * s.effective_lead_time)) AS lead_time_closing_stock,
((ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock) AS days_of_stock_closing_stock,
((ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (vel.daily * s.effective_lead_time))) - (vel.daily * s.effective_days_of_stock) AS days_of_stock_closing_stock,
((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0) AS replenishment_needed_raw,
((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0) AS replenishment_needed_raw,
-- Final Forecasting / Replenishment Metrics
CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS replenishment_units,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_effective_cost AS replenishment_cost,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_price AS replenishment_retail,
(CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * (ci.current_price - ci.current_effective_cost) AS replenishment_profit,
CEILING(GREATEST(0, (((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS replenishment_units,
(CEILING(GREATEST(0, (((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_effective_cost AS replenishment_cost,
(CEILING(GREATEST(0, (((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * ci.current_price AS replenishment_retail,
(CEILING(GREATEST(0, (((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int) * (ci.current_price - ci.current_effective_cost) AS replenishment_profit,
-- To Order (Apply MOQ/UOM logic here if needed, otherwise equals replenishment)
CEILING(GREATEST(0, (((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS to_order_units,
CEILING(GREATEST(0, (((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)) + s.effective_safety_stock - ci.current_stock - COALESCE(ooi.on_order_qty, 0))))::int AS to_order_units,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) AS forecast_lost_sales_units,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time))) * ci.current_price AS forecast_lost_revenue,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (vel.daily * s.effective_lead_time))) AS forecast_lost_sales_units,
GREATEST(0, - (ci.current_stock + COALESCE(ooi.on_order_qty, 0) - (vel.daily * s.effective_lead_time))) * ci.current_price AS forecast_lost_revenue,
ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS stock_cover_in_days,
COALESCE(ooi.on_order_qty, 0) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS po_cover_in_days,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0) AS sells_out_in_days,
ci.current_stock / NULLIF(vel.daily, 0) AS stock_cover_in_days,
COALESCE(ooi.on_order_qty, 0) / NULLIF(vel.daily, 0) AS po_cover_in_days,
(ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(vel.daily, 0) AS sells_out_in_days,
-- Replenish Date: Date when stock is projected to hit safety stock, minus lead time
CASE
WHEN calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) > 0
THEN _current_date + FLOOR(GREATEST(0, ci.current_stock - s.effective_safety_stock) / calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int))::int - s.effective_lead_time
WHEN vel.daily > 0
THEN _current_date + FLOOR(GREATEST(0, ci.current_stock - s.effective_safety_stock) / vel.daily)::int - s.effective_lead_time
ELSE NULL
END AS replenish_date,
GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))::int AS overstocked_units,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))) * ci.current_effective_cost AS overstocked_cost,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock)))) * ci.current_price AS overstocked_retail,
GREATEST(0, ci.current_stock - s.effective_safety_stock - ((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)))::int AS overstocked_units,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)))) * ci.current_effective_cost AS overstocked_cost,
(GREATEST(0, ci.current_stock - s.effective_safety_stock - ((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock)))) * ci.current_price AS overstocked_retail,
-- Old Stock Flag
(ci.created_at::date < _current_date - INTERVAL '60 day') AND
@@ -542,18 +639,18 @@ BEGIN
ELSE
CASE
-- Check for overstock first
WHEN GREATEST(0, ci.current_stock - s.effective_safety_stock - ((calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_lead_time) + (calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int) * s.effective_days_of_stock))) > 0 THEN 'Overstock'
WHEN GREATEST(0, ci.current_stock - s.effective_safety_stock - ((vel.daily * s.effective_lead_time) + (vel.daily * s.effective_days_of_stock))) > 0 THEN 'Overstock'
-- Check for Critical stock
WHEN ci.current_stock <= 0 OR
(ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) <= 0 THEN 'Critical'
(ci.current_stock / NULLIF(vel.daily, 0)) <= 0 THEN 'Critical'
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
WHEN (ci.current_stock / NULLIF(vel.daily, 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
-- Check for reorder soon
WHEN ((ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) + 7) THEN
WHEN ((ci.current_stock + COALESCE(ooi.on_order_qty, 0)) / NULLIF(vel.daily, 0)) < (COALESCE(s.effective_lead_time, 30) + 7) THEN
CASE
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
WHEN (ci.current_stock / NULLIF(vel.daily, 0)) < (COALESCE(s.effective_lead_time, 30) * 0.5) THEN 'Critical'
ELSE 'Reorder Soon'
END
@@ -574,7 +671,7 @@ BEGIN
END) > 180 THEN 'At Risk'
-- Very high stock cover is at risk too
WHEN (ci.current_stock / NULLIF(calculate_sales_velocity(sa.sales_30d::int, sa.stockout_days_30d::int), 0)) > 365 THEN 'At Risk'
WHEN (ci.current_stock / NULLIF(vel.daily, 0)) > 365 THEN 'At Risk'
-- New products (less than 30 days old)
WHEN (CASE
@@ -624,7 +721,11 @@ BEGIN
LEFT JOIN ServiceLevels sl ON ci.pid = sl.pid
LEFT JOIN BeginningStock bs ON ci.pid = bs.pid
LEFT JOIN SeasonalityAnalysis season ON ci.pid = season.pid
WHERE s.exclude_forecast IS FALSE OR s.exclude_forecast IS NULL -- Exclude products explicitly marked
LEFT JOIN StockCoverage sc ON ci.pid = sc.pid
LEFT JOIN ProductVelocity vel ON ci.pid = vel.pid
-- NOTE: products with exclude_from_forecast still get a metrics row (so they
-- appear in brand/vendor/category rollups); only their forecast-derived
-- columns are NULLed via ProductVelocity.
ON CONFLICT (pid) DO UPDATE SET
last_calculated = EXCLUDED.last_calculated,