Improve import scripts with enhanced incremental update tracking and performance

- Add record tracking for added and updated records in import scripts
- Modify products import to use a dynamic 'needs_update' flag for selective updates
- Enhance order import with more comprehensive timestamp checks
- Update import-from-prod.js to handle and clean up previously running imports
- Improve error handling and connection management in import processes

inventory-server/scripts/import/orders.js

@@ -17,6 +17,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
   const startTime = Date.now();
   const skippedOrders = new Set();
   const missingProducts = new Set();
+  let recordsAdded = 0;
+  let recordsUpdated = 0;
 
   try {
     // Get column names from the local table
@@ -88,12 +90,14 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
         ${incrementalUpdate ? `
           AND (
             o.stamp > ?
+            OR oi.stamp > ?
             OR o.date_placed > ?
             OR o.date_shipped > ?
-            OR oi.stamp > ?
+            OR o.date_cancelled > ?
+            OR o.date_updated > ?
           )
         ` : ''}
-    `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
+    `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
 
     const totalOrders = orderItems.length;
     let processed = 0;
@@ -271,12 +275,16 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
         const singlePlaceholder = `(${columnNames.map(() => "?").join(",")})`;
         const placeholders = Array(validOrders.length).fill(singlePlaceholder).join(",");
 
-        await localConnection.query(`
+        const query = `
           INSERT INTO orders (${columnNames.join(",")})
           VALUES ${placeholders}
           ON DUPLICATE KEY UPDATE
             ${columnNames.map(col => `${col} = VALUES(${col})`).join(",")}
-        `, values);
+        `;
+
+        const result = await localConnection.query(query, values.flat());
+        recordsAdded += result.affectedRows - result.changedRows;
+        recordsUpdated += result.changedRows;
 
         importedCount += validOrders.length;
       }
@@ -422,6 +430,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
     return {
       status: "complete",
       totalImported: importedCount,
+      recordsAdded,
+      recordsUpdated,
       totalSkipped: skippedOrders.size,
       missingProducts: missingProducts.size,
       incrementalUpdate,