matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import/calculations improvements

Browse Source
This commit is contained in:
Matt
2026-06-11 19:32:20 -04:00
parent 3b2f51e6b8
commit 069a44bd54
19 changed files with 1175 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inventory-server/scripts/import/orders.js
+108 -103
View File
@@ -1,5 +1,4 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
/**
* Imports orders from a production MySQL database to a local PostgreSQL database.
@@ -28,6 +27,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
22: 'placed_incomplete',
30: 'canceled',
40: 'awaiting_payment',
45: 'payment_pending',
50: 'awaiting_products',
55: 'shipping_later',
56: 'shipping_together',
@@ -35,6 +35,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
61: 'flagged',
62: 'fix_before_pick',
65: 'manual_picking',
67: 'remote_send',
70: 'in_pt',
80: 'picked',
90: 'awaiting_shipment',
@@ -65,6 +66,12 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
console.log('Orders: Using last sync time:', lastSyncTime, '(adjusted:', mysqlSyncTime, ')');
// Capture the next watermark from MySQL's own clock BEFORE querying any data.
// Rows modified while the import runs stay above this watermark for the next
// incremental run (overlap re-imports are harmless upserts); writing NOW()
// after the import finishes would permanently skip them.
const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');
// First get count of order items - Keep MySQL compatible for production
const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total
@@ -100,7 +107,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
COALESCE(NULLIF(TRIM(oi.prod_itemnumber), ''), 'NO-SKU') as SKU,
oi.prod_price as price,
oi.qty_ordered as quantity,
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
oi.stamp as last_modified
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
@@ -131,10 +137,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
await localConnection.query(`
DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS temp_order_costs;
DROP TABLE IF EXISTS temp_main_discounts;
DROP TABLE IF EXISTS temp_item_discounts;
CREATE TEMP TABLE temp_order_items (
@@ -143,7 +147,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
sku TEXT NOT NULL,
price NUMERIC(14, 4) NOT NULL,
quantity INTEGER NOT NULL,
base_discount NUMERIC(14, 4) DEFAULT 0,
PRIMARY KEY (order_id, pid)
);
@@ -160,20 +163,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
PRIMARY KEY (order_id)
);
CREATE TEMP TABLE temp_order_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
discount NUMERIC(14, 4) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TEMP TABLE temp_main_discounts (
order_id INTEGER NOT NULL,
discount_id INTEGER NOT NULL,
discount_amount_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
PRIMARY KEY (order_id, discount_id)
);
CREATE TEMP TABLE temp_item_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
@@ -198,10 +187,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
CREATE INDEX idx_temp_order_items_pid ON temp_order_items(pid);
CREATE INDEX idx_temp_order_meta_order_id ON temp_order_meta(order_id);
CREATE INDEX idx_temp_order_discounts_order_pid ON temp_order_discounts(order_id, pid);
CREATE INDEX idx_temp_order_taxes_order_pid ON temp_order_taxes(order_id, pid);
CREATE INDEX idx_temp_order_costs_order_pid ON temp_order_costs(order_id, pid);
CREATE INDEX idx_temp_main_discounts_discount_id ON temp_main_discounts(discount_id);
CREATE INDEX idx_temp_item_discounts_order_pid ON temp_item_discounts(order_id, pid);
CREATE INDEX idx_temp_item_discounts_discount_id ON temp_item_discounts(discount_id);
`);
@@ -216,21 +203,20 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
await localConnection.beginTransaction();
try {
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
const placeholders = batch.map((_, idx) =>
`($${idx * 6 + 1}, $${idx * 6 + 2}, $${idx * 6 + 3}, $${idx * 6 + 4}, $${idx * 6 + 5}, $${idx * 6 + 6})`
const placeholders = batch.map((_, idx) =>
`($${idx * 5 + 1}, $${idx * 5 + 2}, $${idx * 5 + 3}, $${idx * 5 + 4}, $${idx * 5 + 5})`
).join(",");
const values = batch.flatMap(item => [
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity, item.base_discount
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity
]);
await localConnection.query(`
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity, base_discount)
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
sku = EXCLUDED.sku,
price = EXCLUDED.price,
quantity = EXCLUDED.quantity,
base_discount = EXCLUDED.base_discount
quantity = EXCLUDED.quantity
`, values);
await localConnection.commit();
@@ -337,49 +323,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
};
const processDiscountsBatch = async (batchIds) => {
// First, load main discount records
const [mainDiscounts] = await prodConnection.query(`
SELECT order_id, discount_id, discount_amount_subtotal
FROM order_discounts
WHERE order_id IN (?)
`, [batchIds]);
if (mainDiscounts.length > 0) {
await localConnection.beginTransaction();
try {
for (let j = 0; j < mainDiscounts.length; j += PG_BATCH_SIZE) {
const subBatch = mainDiscounts.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
).join(",");
const values = subBatch.flatMap(d => [
d.order_id,
d.discount_id,
d.discount_amount_subtotal || 0
]);
await localConnection.query(`
INSERT INTO temp_main_discounts (order_id, discount_id, discount_amount_subtotal)
VALUES ${placeholders}
ON CONFLICT (order_id, discount_id) DO UPDATE SET
discount_amount_subtotal = EXCLUDED.discount_amount_subtotal
`, values);
}
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
throw error;
}
}
// Then, load item discount records
// Load item-level discount records. Only which = 2 rows are real per-item
// discount amounts; which = 1 rows store the price of free promo-added
// items and which = 3 rows are usage records (neither is a discount).
// These amounts are NOT included in summary_discount_subtotal, so they
// must be added on top of the prorated subtotal discount unconditionally.
const [discounts] = await prodConnection.query(`
SELECT order_id, pid, discount_id, amount
FROM order_discount_items
WHERE order_id IN (?)
WHERE order_id IN (?) AND which = 2
`, [batchIds]);
if (discounts.length === 0) return;
@@ -418,16 +370,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
`, values);
}
// Create aggregated view with a simpler, safer query that avoids duplicates
await localConnection.query(`
TRUNCATE temp_order_discounts;
INSERT INTO temp_order_discounts (order_id, pid, discount)
SELECT order_id, pid, SUM(amount) as discount
FROM temp_item_discounts
GROUP BY order_id, pid
`);
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
@@ -603,42 +545,54 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
try {
const [orders] = await localConnection.query(`
WITH order_totals AS (
SELECT
SELECT
oi.order_id,
oi.pid,
-- Instead of using ARRAY_AGG which can cause duplicate issues, use SUM with a CASE
SUM(CASE
WHEN COALESCE(md.discount_amount_subtotal, 0) > 0 THEN id.amount
ELSE 0
END) as promo_discount_sum,
-- Item-level promo discounts (which = 2 rows). These live outside
-- summary_discount_subtotal, so they are summed unconditionally.
SUM(COALESCE(id.amount, 0)) as promo_discount_sum,
COALESCE(ot.tax, 0) as total_tax,
COALESCE(oc.costeach, pc.cost_price, oi.price * 0.5) as costeach
FROM temp_order_items oi
LEFT JOIN temp_item_discounts id ON oi.order_id = id.order_id AND oi.pid = id.pid
LEFT JOIN temp_main_discounts md ON id.order_id = md.order_id AND id.discount_id = md.discount_id
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
LEFT JOIN temp_order_costs oc ON oi.order_id = oc.order_id AND oi.pid = oc.pid
LEFT JOIN temp_product_costs pc ON oi.pid = pc.pid
WHERE oi.order_id = ANY($1)
GROUP BY oi.order_id, oi.pid, ot.tax, oc.costeach, pc.cost_price
)
SELECT
SELECT
oi.order_id as order_number,
oi.pid::bigint as pid,
oi.sku,
om.date,
oi.price,
oi.quantity,
-- Discount = prorated order-level subtotal discount + item-level promo
-- discounts, clamped so a sale line can never be discounted below free.
(
-- Prorated Points Discount (e.g. loyalty points applied at order level)
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
CASE WHEN oi.quantity > 0 THEN
LEAST(
(
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
END
+ COALESCE(ot.promo_discount_sum, 0)
),
oi.price * oi.quantity
)
ELSE
(
CASE
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
ELSE 0
END
+ COALESCE(ot.promo_discount_sum, 0)
)
END
+
-- Specific Item-Level Promo Discount (coupon codes, etc.)
COALESCE(ot.promo_discount_sum, 0)
)::NUMERIC(14, 4) as discount,
COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
false as tax_included,
@@ -765,34 +719,83 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
}
}
// Start a transaction for updating sync status and dropping temp tables
// Reconciliation 2 prep: fetch canceled (15) / combined (16) orders from MySQL
// WITHOUT a date_placed filter — combine_orders zeroes date_placed on the source
// orders, so the main item query can never re-fetch them. Done before opening
// the PG transaction so we don't hold it across a MySQL round-trip.
const [statusSweepRows] = await prodConnection.query(`
SELECT order_id, order_status
FROM _order
WHERE order_status IN (15, 16)
${incrementalUpdate ? 'AND stamp > ?' : ''}
`, incrementalUpdate ? [mysqlSyncTime] : []);
let staleItemsDeleted = 0;
let sweepUpdated = 0;
// Final transaction: reconcile deletions, sweep statuses, update sync status, drop temps
await localConnection.beginTransaction();
try {
// Update sync status
// Reconciliation 1: delete PG item rows that no longer exist in MySQL for the
// orders fetched this run. temp_order_items holds the complete current item
// set of every fetched order (staff edits and unpicked promo items DELETE
// order_items rows in MySQL, which an upsert-only import never removes).
const [reconcileResult] = await localConnection.query(`
DELETE FROM orders o
USING (SELECT DISTINCT order_id FROM temp_order_items) fetched
WHERE o.order_number = fetched.order_id::text -- orders.order_number is TEXT
AND NOT EXISTS (
SELECT 1 FROM temp_order_items t
WHERE t.order_id = fetched.order_id AND t.pid = o.pid
)
`);
staleItemsDeleted = reconcileResult.rowCount || 0;
// Reconciliation 2: mark canceled/combined orders. 'combined' source orders were
// merged into a new order that carries the same items — counting both would
// double-count, so they also get canceled = true (routes filter on canceled).
for (const [code, statusText] of [[15, 'canceled'], [16, 'combined']]) {
const ids = statusSweepRows.filter(r => r.order_status === code).map(r => r.order_id);
for (let i = 0; i < ids.length; i += 5000) {
const chunk = ids.slice(i, i + 5000);
const [sweepResult] = await localConnection.query(`
UPDATE orders
SET status = $1, canceled = true
WHERE order_number = ANY($2::text[])
AND (status IS DISTINCT FROM $1 OR canceled IS DISTINCT FROM true)
`, [statusText, chunk.map(String)]);
sweepUpdated += sweepResult.rowCount || 0;
}
}
// Update sync status with the watermark captured from MySQL BEFORE the
// source queries ran (see sourceNow above).
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('orders', NOW())
VALUES ('orders', $1)
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
last_sync_timestamp = $1
`, [sourceNow]);
// Cleanup temporary tables
await localConnection.query(`
DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS temp_order_costs;
DROP TABLE IF EXISTS temp_main_discounts;
DROP TABLE IF EXISTS temp_item_discounts;
DROP TABLE IF EXISTS temp_product_costs;
`);
// Commit final transaction
await localConnection.commit();
} catch (error) {
await localConnection.rollback();
throw error;
throw error;
}
if (staleItemsDeleted > 0 || sweepUpdated > 0) {
console.log(`Orders: reconciliation removed ${staleItemsDeleted} stale item rows, swept ${sweepUpdated} canceled/combined rows`);
}
return {
@@ -800,6 +803,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
totalImported: Math.floor(importedCount) || 0,
recordsAdded: parseInt(recordsAdded) || 0,
recordsUpdated: parseInt(recordsUpdated) || 0,
recordsDeleted: staleItemsDeleted,
statusSweepUpdated: sweepUpdated,
totalSkipped: skippedOrders.size || 0,
missingProducts: missingProducts.size || 0,
totalProcessed: orderItems.length, // Total order items in source