matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Modify import scripts to handle edge cases with empty arrays and null conditions

Browse Source 
- Improve parameter handling in incremental update queries for purchase orders and products
This commit is contained in:
Matt
2025-01-31 01:50:21 -05:00
parent 5e4d1c3bd8
commit d0abe9d9a2
4 changed files with 40 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
inventory-server/scripts/import-from-prod.js
View File

@@ -10,8 +10,8 @@ const importPurchaseOrders = require('./import/purchase-orders');
dotenv.config({ path: path.join(__dirname, "../.env") }); dotenv.config({ path: path.join(__dirname, "../.env") });
// Constants to control which imports run // Constants to control which imports run
const IMPORT_CATEGORIES = true; const IMPORT_CATEGORIES = false;
const IMPORT_PRODUCTS = true; const IMPORT_PRODUCTS = false;
const IMPORT_ORDERS = true; const IMPORT_ORDERS = true;
const IMPORT_PURCHASE_ORDERS = true; const IMPORT_PURCHASE_ORDERS = true;

18
inventory-server/scripts/import/orders.js
View File

@@ -222,10 +222,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
// Pre-check all products at once instead of per batch // Pre-check all products at once instead of per batch
const allOrderPids = [...new Set(orderItems.map(item => item.pid))]; const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
const [existingProducts] = await localConnection.query( const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
"SELECT pid FROM products WHERE pid IN (?)", "SELECT pid FROM products WHERE pid IN (?)",
[allOrderPids] [allOrderPids]
); ) : [[]];
const existingPids = new Set(existingProducts.map(p => p.pid)); const existingPids = new Set(existingProducts.map(p => p.pid));
// Process in larger batches // Process in larger batches
@@ -312,8 +312,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
// Import missing products if any // Import missing products if any
if (missingProducts.size > 0) { if (missingProducts.size > 0) {
try { try {
// Setup temporary tables again since they were dropped
await setupTemporaryTables(localConnection);
await materializeCalculations(prodConnection, localConnection);
await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts)); await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts));
// Clean up temporary tables after missing products import
await cleanupTemporaryTables(localConnection);
// Retry skipped orders after importing products // Retry skipped orders after importing products
if (skippedOrders.size > 0) { if (skippedOrders.size > 0) {
outputProgress({ outputProgress({
@@ -322,7 +329,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
message: `Retrying import of ${skippedOrders.size} orders with previously missing products` message: `Retrying import of ${skippedOrders.size} orders with previously missing products`
}); });
const [skippedProdOrders] = await prodConnection.query(` const skippedOrdersArray = Array.from(skippedOrders);
const [skippedProdOrders] = skippedOrdersArray.length > 0 ? await prodConnection.query(`
SELECT SELECT
o.order_id, o.order_id,
CASE CASE
@@ -353,7 +361,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
FROM order_items oi FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id JOIN _order o ON oi.order_id = o.order_id
WHERE o.order_id IN (?) WHERE o.order_id IN (?)
`, [Array.from(skippedOrders)]); `, [skippedOrdersArray]) : [[]];
// Prepare values for insertion // Prepare values for insertion
const skippedOrderValues = skippedProdOrders.flatMap(order => { const skippedOrderValues = skippedProdOrders.flatMap(order => {
@@ -420,7 +428,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
} }
} }
// Update sync status - do this even if missing products import fails // Only update sync status if we get here (no errors thrown)
await localConnection.query(` await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp) INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('orders', NOW()) VALUES ('orders', NOW())

7
inventory-server/scripts/import/products.js
View File

@@ -331,8 +331,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
pcp.date_active > ? OR pcp.date_active > ? OR
sid.stamp > ? OR sid.stamp > ? OR
pnb.date_updated > ? OR pnb.date_updated > ? OR
pls.date_sold > ? OR pls.date_sold > ?
si.stamp > ?
` : 'TRUE'} ` : 'TRUE'}
THEN 1 ELSE 0 END as needs_update THEN 1 ELSE 0 END as needs_update
FROM products p FROM products p
@@ -349,7 +348,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1 LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
GROUP BY p.pid GROUP BY p.pid
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []); `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
// Insert production data in batches, but only for products that need updates // Insert production data in batches, but only for products that need updates
for (let i = 0; i < prodData.length; i += 1000) { for (let i = 0; i < prodData.length; i += 1000) {
@@ -500,7 +499,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
// Drop temporary tables // Drop temporary tables
await cleanupTemporaryTables(localConnection); await cleanupTemporaryTables(localConnection);
// After successful import, update the sync status // Only update sync status if we get here (no errors thrown)
await localConnection.query(` await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp) INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('products', NOW()) VALUES ('products', NOW())

29
inventory-server/scripts/import/purchase-orders.js
View File

@@ -31,8 +31,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
// Build incremental conditions // Build incremental conditions
const incrementalWhereClause = incrementalUpdate const incrementalWhereClause = incrementalUpdate
? `AND ( ? `AND (
p.stamp > ? p.date_updated > ?
OR p.date_updated > ?
OR p.date_ordered > ? OR p.date_ordered > ?
OR p.date_estin > ? OR p.date_estin > ?
OR r.date_updated > ? OR r.date_updated > ?
@@ -43,7 +42,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
)` )`
: ""; : "";
const incrementalParams = incrementalUpdate const incrementalParams = incrementalUpdate
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
: []; : [];
// First get all relevant PO IDs with basic info // First get all relevant PO IDs with basic info
@@ -56,16 +55,32 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
JOIN po_products pop ON p.po_id = pop.po_id JOIN po_products pop ON p.po_id = pop.po_id
JOIN suppliers s ON p.supplier_id = s.supplierid JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
${incrementalWhereClause} ${incrementalUpdate ? `
AND (
p.date_updated > ?
OR p.date_ordered > ?
OR p.date_estin > ?
)
` : ''}
UNION UNION
SELECT DISTINCT r.receiving_id as po_id, rp.pid SELECT DISTINCT r.receiving_id as po_id, rp.pid
FROM receivings_products rp FROM receivings_products rp
USE INDEX (received_date) USE INDEX (received_date)
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
${incrementalWhereClause} ${incrementalUpdate ? `
AND (
r.date_created > ?
OR r.date_checked > ?
OR rp.stamp > ?
OR rp.received_date > ?
)
` : ''}
) all_items ) all_items
`, [...incrementalParams, ...incrementalParams]); `, incrementalUpdate ? [
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
] : []);
const [poList] = await prodConnection.query(` const [poList] = await prodConnection.query(`
SELECT DISTINCT SELECT DISTINCT
@@ -337,7 +352,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
} }
} }
// Update sync status with proper incrementing of last_sync_id // Only update sync status if we get here (no errors thrown)
await localConnection.query(` await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp) INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW()) VALUES ('purchase_orders', NOW())