- Modify import scripts to handle edge cases with empty arrays and null conditions

- Improve parameter handling in incremental update queries for purchase orders and products
2025-01-31 01:50:21 -05:00
parent 5e4d1c3bd8
commit d0abe9d9a2
4 changed files with 40 additions and 18 deletions
--- a/inventory-server/scripts/import-from-prod.js
+++ b/inventory-server/scripts/import-from-prod.js
@@ -10,8 +10,8 @@ const importPurchaseOrders = require('./import/purchase-orders');
 dotenv.config({ path: path.join(__dirname, "../.env") });

 // Constants to control which imports run
-const IMPORT_CATEGORIES = true;
-const IMPORT_PRODUCTS = true;
+const IMPORT_CATEGORIES = false;
+const IMPORT_PRODUCTS = false;
 const IMPORT_ORDERS = true;
 const IMPORT_PURCHASE_ORDERS = true;

--- a/inventory-server/scripts/import/orders.js
+++ b/inventory-server/scripts/import/orders.js
@@ -222,10 +222,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =

    // Pre-check all products at once instead of per batch
    const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
-    const [existingProducts] = await localConnection.query(
+    const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
      "SELECT pid FROM products WHERE pid IN (?)",
      [allOrderPids]
-    );
+    ) : [[]];
    const existingPids = new Set(existingProducts.map(p => p.pid));

    // Process in larger batches
@@ -312,8 +312,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
    // Import missing products if any
    if (missingProducts.size > 0) {
      try {
+        // Setup temporary tables again since they were dropped
+        await setupTemporaryTables(localConnection);
+        await materializeCalculations(prodConnection, localConnection);
+        
        await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts));
        
+        // Clean up temporary tables after missing products import
+        await cleanupTemporaryTables(localConnection);
+        
        // Retry skipped orders after importing products
        if (skippedOrders.size > 0) {
          outputProgress({
@@ -322,7 +329,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
            message: `Retrying import of ${skippedOrders.size} orders with previously missing products`
          });

-          const [skippedProdOrders] = await prodConnection.query(`
+          const skippedOrdersArray = Array.from(skippedOrders);
+          const [skippedProdOrders] = skippedOrdersArray.length > 0 ? await prodConnection.query(`
            SELECT
              o.order_id,
              CASE 
@@ -353,7 +361,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
            FROM order_items oi
            JOIN _order o ON oi.order_id = o.order_id
            WHERE o.order_id IN (?)
-          `, [Array.from(skippedOrders)]);
+          `, [skippedOrdersArray]) : [[]];

          // Prepare values for insertion
          const skippedOrderValues = skippedProdOrders.flatMap(order => {
@@ -420,7 +428,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
      }
    }

-    // Update sync status - do this even if missing products import fails
+    // Only update sync status if we get here (no errors thrown)
    await localConnection.query(`
      INSERT INTO sync_status (table_name, last_sync_timestamp)
      VALUES ('orders', NOW())
--- a/inventory-server/scripts/import/products.js
+++ b/inventory-server/scripts/import/products.js
@@ -331,8 +331,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
            pcp.date_active > ? OR
            sid.stamp > ? OR
            pnb.date_updated > ? OR
-            pls.date_sold > ? OR
-            si.stamp > ?
+            pls.date_sold > ?
          ` : 'TRUE'}
        THEN 1 ELSE 0 END as needs_update
      FROM products p
@@ -349,7 +348,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
        LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
        LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
      GROUP BY p.pid
-    `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
+    `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);

    // Insert production data in batches, but only for products that need updates
    for (let i = 0; i < prodData.length; i += 1000) {
@@ -500,7 +499,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
    // Drop temporary tables
    await cleanupTemporaryTables(localConnection);

-    // After successful import, update the sync status
+    // Only update sync status if we get here (no errors thrown)
    await localConnection.query(`
      INSERT INTO sync_status (table_name, last_sync_timestamp)
      VALUES ('products', NOW())
--- a/inventory-server/scripts/import/purchase-orders.js
+++ b/inventory-server/scripts/import/purchase-orders.js
@@ -31,8 +31,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
    // Build incremental conditions
    const incrementalWhereClause = incrementalUpdate
      ? `AND (
-          p.stamp > ? 
-          OR p.date_updated > ?
+          p.date_updated > ? 
          OR p.date_ordered > ?
          OR p.date_estin > ?
          OR r.date_updated > ?
@@ -43,7 +42,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
        )`
      : "";
    const incrementalParams = incrementalUpdate
-      ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
+      ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
      : [];

    // First get all relevant PO IDs with basic info
@@ -56,16 +55,32 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
        JOIN po_products pop ON p.po_id = pop.po_id
        JOIN suppliers s ON p.supplier_id = s.supplierid
        WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
-          ${incrementalWhereClause}
+          ${incrementalUpdate ? `
+            AND (
+              p.date_updated > ? 
+              OR p.date_ordered > ?
+              OR p.date_estin > ?
+            )
+          ` : ''}
        UNION
        SELECT DISTINCT r.receiving_id as po_id, rp.pid
        FROM receivings_products rp
        USE INDEX (received_date)
        LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
        WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
-          ${incrementalWhereClause}
+          ${incrementalUpdate ? `
+            AND (
+              r.date_created > ?
+              OR r.date_checked > ?
+              OR rp.stamp > ?
+              OR rp.received_date > ?
+            )
+          ` : ''}
      ) all_items
-    `, [...incrementalParams, ...incrementalParams]);
+    `, incrementalUpdate ? [
+      lastSyncTime, lastSyncTime, lastSyncTime,  // PO conditions
+      lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime  // Receiving conditions
+    ] : []);

    const [poList] = await prodConnection.query(`
      SELECT DISTINCT 
@@ -337,7 +352,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
      }
    }

-    // Update sync status with proper incrementing of last_sync_id
+    // Only update sync status if we get here (no errors thrown)
    await localConnection.query(`
      INSERT INTO sync_status (table_name, last_sync_timestamp)
      VALUES ('purchase_orders', NOW())