From d0abe9d9a2e3cd5267ab2b3662be41760df4b2f0 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 31 Jan 2025 01:50:21 -0500 Subject: [PATCH] - Modify import scripts to handle edge cases with empty arrays and null conditions - Improve parameter handling in incremental update queries for purchase orders and products --- inventory-server/scripts/import-from-prod.js | 4 +-- inventory-server/scripts/import/orders.js | 18 ++++++++---- inventory-server/scripts/import/products.js | 7 ++--- .../scripts/import/purchase-orders.js | 29 ++++++++++++++----- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/inventory-server/scripts/import-from-prod.js b/inventory-server/scripts/import-from-prod.js index 7395293..2f054da 100644 --- a/inventory-server/scripts/import-from-prod.js +++ b/inventory-server/scripts/import-from-prod.js @@ -10,8 +10,8 @@ const importPurchaseOrders = require('./import/purchase-orders'); dotenv.config({ path: path.join(__dirname, "../.env") }); // Constants to control which imports run -const IMPORT_CATEGORIES = true; -const IMPORT_PRODUCTS = true; +const IMPORT_CATEGORIES = false; +const IMPORT_PRODUCTS = false; const IMPORT_ORDERS = true; const IMPORT_PURCHASE_ORDERS = true; diff --git a/inventory-server/scripts/import/orders.js b/inventory-server/scripts/import/orders.js index 38b0eae..c000555 100644 --- a/inventory-server/scripts/import/orders.js +++ b/inventory-server/scripts/import/orders.js @@ -222,10 +222,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = // Pre-check all products at once instead of per batch const allOrderPids = [...new Set(orderItems.map(item => item.pid))]; - const [existingProducts] = await localConnection.query( + const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query( "SELECT pid FROM products WHERE pid IN (?)", [allOrderPids] - ); + ) : [[]]; const existingPids = new Set(existingProducts.map(p => p.pid)); // Process in larger batches @@ -312,8 +312,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = // Import missing products if any if (missingProducts.size > 0) { try { + // Setup temporary tables again since they were dropped + await setupTemporaryTables(localConnection); + await materializeCalculations(prodConnection, localConnection); + await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts)); + // Clean up temporary tables after missing products import + await cleanupTemporaryTables(localConnection); + // Retry skipped orders after importing products if (skippedOrders.size > 0) { outputProgress({ @@ -322,7 +329,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = message: `Retrying import of ${skippedOrders.size} orders with previously missing products` }); - const [skippedProdOrders] = await prodConnection.query(` + const skippedOrdersArray = Array.from(skippedOrders); + const [skippedProdOrders] = skippedOrdersArray.length > 0 ? await prodConnection.query(` SELECT o.order_id, CASE @@ -353,7 +361,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = FROM order_items oi JOIN _order o ON oi.order_id = o.order_id WHERE o.order_id IN (?) - `, [Array.from(skippedOrders)]); + `, [skippedOrdersArray]) : [[]]; // Prepare values for insertion const skippedOrderValues = skippedProdOrders.flatMap(order => { @@ -420,7 +428,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = } } - // Update sync status - do this even if missing products import fails + // Only update sync status if we get here (no errors thrown) await localConnection.query(` INSERT INTO sync_status (table_name, last_sync_timestamp) VALUES ('orders', NOW()) diff --git a/inventory-server/scripts/import/products.js b/inventory-server/scripts/import/products.js index 41bbbaa..cbd67a1 100644 --- a/inventory-server/scripts/import/products.js +++ b/inventory-server/scripts/import/products.js @@ -331,8 +331,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate pcp.date_active > ? OR sid.stamp > ? OR pnb.date_updated > ? OR - pls.date_sold > ? OR - si.stamp > ? + pls.date_sold > ? ` : 'TRUE'} THEN 1 ELSE 0 END as needs_update FROM products p @@ -349,7 +348,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1 LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid GROUP BY p.pid - `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []); + `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []); // Insert production data in batches, but only for products that need updates for (let i = 0; i < prodData.length; i += 1000) { @@ -500,7 +499,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate // Drop temporary tables await cleanupTemporaryTables(localConnection); - // After successful import, update the sync status + // Only update sync status if we get here (no errors thrown) await localConnection.query(` INSERT INTO sync_status (table_name, last_sync_timestamp) VALUES ('products', NOW()) diff --git a/inventory-server/scripts/import/purchase-orders.js b/inventory-server/scripts/import/purchase-orders.js index d05c231..369e369 100644 --- a/inventory-server/scripts/import/purchase-orders.js +++ b/inventory-server/scripts/import/purchase-orders.js @@ -31,8 +31,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental // Build incremental conditions const incrementalWhereClause = incrementalUpdate ? `AND ( - p.stamp > ? - OR p.date_updated > ? + p.date_updated > ? OR p.date_ordered > ? OR p.date_estin > ? OR r.date_updated > ? @@ -43,7 +42,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental )` : ""; const incrementalParams = incrementalUpdate - ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] + ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []; // First get all relevant PO IDs with basic info @@ -56,16 +55,32 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental JOIN po_products pop ON p.po_id = pop.po_id JOIN suppliers s ON p.supplier_id = s.supplierid WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) - ${incrementalWhereClause} + ${incrementalUpdate ? ` + AND ( + p.date_updated > ? + OR p.date_ordered > ? + OR p.date_estin > ? + ) + ` : ''} UNION SELECT DISTINCT r.receiving_id as po_id, rp.pid FROM receivings_products rp USE INDEX (received_date) LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) - ${incrementalWhereClause} + ${incrementalUpdate ? ` + AND ( + r.date_created > ? + OR r.date_checked > ? + OR rp.stamp > ? + OR rp.received_date > ? + ) + ` : ''} ) all_items - `, [...incrementalParams, ...incrementalParams]); + `, incrementalUpdate ? [ + lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions + lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions + ] : []); const [poList] = await prodConnection.query(` SELECT DISTINCT @@ -337,7 +352,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental } } - // Update sync status with proper incrementing of last_sync_id + // Only update sync status if we get here (no errors thrown) await localConnection.query(` INSERT INTO sync_status (table_name, last_sync_timestamp) VALUES ('purchase_orders', NOW())