matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import/calculations improvements

Browse Source
This commit is contained in:
Matt
2026-06-11 19:32:20 -04:00
parent 3b2f51e6b8
commit 069a44bd54
19 changed files with 1175 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inventory-server/scripts/import/purchase-orders.js
+54 -35
View File
@@ -72,6 +72,11 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
console.log('Purchase Orders: Using last sync time:', lastSyncTime, '(adjusted:', mysqlSyncTime, ')');
// Capture the next watermark from MySQL's own clock BEFORE querying any data.
// Rows modified while the import runs stay above this watermark for the next
// incremental run (overlap re-imports are harmless upserts).
const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');
// Create temp tables for processing
await localConnection.query(`
DROP TABLE IF EXISTS temp_purchase_orders;
@@ -267,13 +272,16 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
if (totalPOs === 0) {
console.log('No purchase orders to process, skipping PO import step');
} else {
// Fetch and process POs in batches
let offset = 0;
// Fetch and process POs in batches using keyset pagination on po_id.
// LIMIT/OFFSET over a date_updated predicate silently skips rows when
// concurrent updates shift rows between pages.
let processedPOCount = 0;
let lastPoId = 0;
let allPOsProcessed = false;
while (!allPOsProcessed) {
const [poList] = await prodConnection.query(`
SELECT
SELECT
p.po_id,
p.supplier_id,
s.companyname AS vendor,
@@ -286,21 +294,23 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
FROM po p
LEFT JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
AND p.po_id > ?
${incrementalUpdate ? `
AND (
p.date_updated > ?
OR p.date_ordered > ?
p.date_updated > ?
OR p.date_ordered > ?
OR p.date_estin > ?
)
` : ''}
ORDER BY p.po_id
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
`, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : []);
LIMIT ${PO_BATCH_SIZE}
`, incrementalUpdate ? [lastPoId, mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : [lastPoId]);
if (poList.length === 0) {
allPOsProcessed = true;
break;
}
lastPoId = poList[poList.length - 1].po_id;
// Get products for these POs
const poIds = poList.map(po => po.po_id);
@@ -332,7 +342,11 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
vendor: po.vendor || 'Unknown Vendor',
date: validateDate(po.date_ordered) || validateDate(po.date_created),
expected_date: validateDate(po.date_estin),
status: poStatusMap[po.status] || 'created',
// Unknown codes get a sentinel rather than 'created': defaulting an
// unknown cancel-like code to an OPEN status would inflate on-order
// FIFO (the metrics CTEs whitelist known-open statuses, so a sentinel
// is simply ignored there).
status: poStatusMap[po.status] || `unknown_${po.status}`,
notes: po.notes || '',
long_note: po.long_note || '',
ordered: product.qty_each,
@@ -393,20 +407,20 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
`, values);
}
offset += poList.length;
processedPOCount += poList.length;
totalProcessed += completePOs.length;
outputProgress({
status: "running",
operation: "Purchase orders import",
message: `Processed ${offset} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
current: offset,
message: `Processed ${processedPOCount} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
current: processedPOCount,
total: totalPOs,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, offset, totalPOs),
rate: calculateRate(startTime, offset)
remaining: estimateRemaining(startTime, processedPOCount, totalPOs),
rate: calculateRate(startTime, processedPOCount)
});
if (poList.length < PO_BATCH_SIZE) {
allPOsProcessed = true;
}
@@ -439,13 +453,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
if (totalReceivings === 0) {
console.log('No receivings to process, skipping receivings import step');
} else {
// Fetch and process receivings in batches
offset = 0; // Reset offset for receivings
// Fetch and process receivings in batches (keyset pagination, see POs above)
let processedReceivingCount = 0;
let lastReceivingId = 0;
let allReceivingsProcessed = false;
while (!allReceivingsProcessed) {
const [receivingList] = await prodConnection.query(`
SELECT
SELECT
r.receiving_id,
r.supplier_id,
r.status,
@@ -459,6 +474,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
r.date_checked
FROM receivings r
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
AND r.receiving_id > ?
${incrementalUpdate ? `
AND (
r.date_updated > ?
@@ -466,13 +482,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
)
` : ''}
ORDER BY r.receiving_id
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
`, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime] : []);
LIMIT ${PO_BATCH_SIZE}
`, incrementalUpdate ? [lastReceivingId, mysqlSyncTime, mysqlSyncTime] : [lastReceivingId]);
if (receivingList.length === 0) {
allReceivingsProcessed = true;
break;
}
lastReceivingId = receivingList[receivingList.length - 1].receiving_id;
// Get products for these receivings
const receivingIds = receivingList.map(r => r.receiving_id);
@@ -545,7 +562,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
receiving_created_date: validateDate(product.receiving_created_date),
supplier_id: receiving.supplier_id,
status: receivingStatusMap[receiving.status] || 'created'
// Sentinel for unknown codes — see PO status mapping note above
status: receivingStatusMap[receiving.status] || `unknown_${receiving.status}`
});
}
@@ -600,18 +618,18 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
`, values);
}
offset += receivingList.length;
processedReceivingCount += receivingList.length;
totalProcessed += completeReceivings.length;
outputProgress({
status: "running",
operation: "Purchase orders import",
message: `Processed ${offset} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
current: offset,
message: `Processed ${processedReceivingCount} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
current: processedReceivingCount,
total: totalReceivings,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, offset, totalReceivings),
rate: calculateRate(startTime, offset)
remaining: estimateRemaining(startTime, processedReceivingCount, totalReceivings),
rate: calculateRate(startTime, processedReceivingCount)
});
if (receivingList.length < PO_BATCH_SIZE) {
@@ -829,13 +847,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
receivingRecordsAdded = receivingsResult.rows.filter(r => r.inserted).length;
receivingRecordsUpdated = receivingsResult.rows.filter(r => !r.inserted).length;
// Update sync status
// Update sync status with the watermark captured from MySQL BEFORE the
// source queries ran (see sourceNow above).
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW())
VALUES ('purchase_orders', $1)
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
last_sync_timestamp = $1
`, [sourceNow]);
// Clean up temporary tables
await localConnection.query(`