Finish up import script incremental and reliability updates

inventory-server/scripts/import/purchase-orders.js

@@ -12,6 +12,22 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
     );
     const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
 
+    console.log('Purchase Orders: Using last sync time:', lastSyncTime);
+
+    // Insert temporary table creation query for purchase orders
+    await localConnection.query(`
+      CREATE TABLE IF NOT EXISTS temp_purchase_orders (
+        po_id INT UNSIGNED NOT NULL,
+        pid INT UNSIGNED NOT NULL,
+        vendor VARCHAR(255),
+        date DATE,
+        expected_date DATE,
+        status INT,
+        notes TEXT,
+        PRIMARY KEY (po_id, pid)
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+    `);
+
     outputProgress({
       operation: `Starting ${incrementalUpdate ? 'incremental' : 'full'} purchase orders import`,
       status: "running",
@@ -82,6 +98,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
       lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime  // Receiving conditions
     ] : []);
 
+    console.log('Purchase Orders: Found changes:', total);
+
     const [poList] = await prodConnection.query(`
       SELECT DISTINCT 
         COALESCE(p.po_id, r.receiving_id) as po_id,
@@ -221,6 +239,22 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
         const values = [];
         let batchProcessed = 0;
 
+        // First check which PO lines already exist and get their current values
+        const poLines = Array.from(poProductMap.values())
+          .filter(p => validPids.has(p.pid))
+          .map(p => [p.po_id, p.pid]);
+
+        const [existingPOs] = await localConnection.query(
+          `SELECT ${columnNames.join(',')} FROM purchase_orders WHERE (po_id, pid) IN (${poLines.map(() => "(?,?)").join(",")})`,
+          poLines.flat()
+        );
+        const existingPOMap = new Map(
+          existingPOs.map(po => [`${po.po_id}-${po.pid}`, po])
+        );
+
+        // Split into inserts and updates
+        const insertsAndUpdates = { inserts: [], updates: [] };
+
         for (const po of batch) {
           const poProducts = Array.from(poProductMap.values())
             .filter(p => p.po_id === po.po_id && validPids.has(p.pid));
@@ -280,7 +314,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
             const firstReceiving = allReceivings[0] || {};
             const lastReceiving = allReceivings[allReceivings.length - 1] || {};
 
-            values.push(columnNames.map(col => {
+            const rowValues = columnNames.map(col => {
               switch (col) {
                 case 'po_id': return po.po_id;
                 case 'vendor': return po.vendor;
@@ -309,28 +343,75 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
                 });
                 default: return null;
               }
-            }));
+            });
+
+            if (existingPOMap.has(key)) {
+              const existing = existingPOMap.get(key);
+              // Check if any values are different
+              const hasChanges = columnNames.some(col => {
+                const newVal = rowValues[columnNames.indexOf(col)];
+                const oldVal = existing[col] ?? null;
+                // Special handling for numbers to avoid type coercion issues
+                if (typeof newVal === 'number' && typeof oldVal === 'number') {
+                  return Math.abs(newVal - oldVal) > 0.00001; // Allow for tiny floating point differences
+                }
+                // Special handling for receiving_history - parse and compare
+                if (col === 'receiving_history') {
+                  const newHistory = JSON.parse(newVal || '{}');
+                  const oldHistory = JSON.parse(oldVal || '{}');
+                  return JSON.stringify(newHistory) !== JSON.stringify(oldHistory);
+                }
+                return newVal !== oldVal;
+              });
+
+              if (hasChanges) {
+                insertsAndUpdates.updates.push({
+                  po_id: po.po_id,
+                  pid: product.pid,
+                  values: rowValues
+                });
+              }
+            } else {
+              insertsAndUpdates.inserts.push({
+                po_id: po.po_id,
+                pid: product.pid,
+                values: rowValues
+              });
+            }
             batchProcessed++;
           }
         }
 
-        if (values.length > 0) {
-          const placeholders = values.map(() => 
-            `(${Array(columnNames.length).fill("?").join(",")})`
-          ).join(",");
+        // Handle inserts
+        if (insertsAndUpdates.inserts.length > 0) {
+          const insertPlaceholders = insertsAndUpdates.inserts
+            .map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
+            .join(",");
 
-          const query = `
+          const insertResult = await localConnection.query(`
             INSERT INTO purchase_orders (${columnNames.join(",")})
-            VALUES ${placeholders}
+            VALUES ${insertPlaceholders}
+          `, insertsAndUpdates.inserts.map(i => i.values).flat());
+
+          recordsAdded += insertResult[0].affectedRows;
+        }
+
+        // Handle updates - now we know these actually have changes
+        if (insertsAndUpdates.updates.length > 0) {
+          const updatePlaceholders = insertsAndUpdates.updates
+            .map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
+            .join(",");
+
+          const updateResult = await localConnection.query(`
+            INSERT INTO purchase_orders (${columnNames.join(",")})
+            VALUES ${updatePlaceholders}
             ON DUPLICATE KEY UPDATE ${columnNames
               .filter((col) => col !== "po_id" && col !== "pid")
               .map((col) => `${col} = VALUES(${col})`)
               .join(",")};
-          `;
+          `, insertsAndUpdates.updates.map(u => u.values).flat());
 
-          const result = await localConnection.query(query, values.flat());
-          recordsAdded += result.affectedRows - (2 * result.changedRows);
-          recordsUpdated += result.changedRows;
+          recordsUpdated += updateResult[0].affectedRows / 2;  // Each update counts as 2 in affectedRows
         }
 
         processed += batchProcessed;