Import script incremental fixes

inventory-server/scripts/import-from-prod.js

@@ -161,32 +161,32 @@ async function main() {
       results.categories = await importCategories(prodConnection, localConnection);
       if (isImportCancelled) throw new Error("Import cancelled");
       completedSteps++;
-      if (results.categories?.recordsAdded) totalRecordsAdded += results.categories.recordsAdded;
+      totalRecordsAdded += results.categories?.recordsAdded || 0;
-      if (results.categories?.recordsUpdated) totalRecordsUpdated += results.categories.recordsUpdated;
+      totalRecordsUpdated += results.categories?.recordsUpdated || 0;
     }
 
     if (IMPORT_PRODUCTS) {
       results.products = await importProducts(prodConnection, localConnection, INCREMENTAL_UPDATE);
       if (isImportCancelled) throw new Error("Import cancelled");
       completedSteps++;
-      if (results.products?.recordsAdded) totalRecordsAdded += results.products.recordsAdded;
+      totalRecordsAdded += results.products?.recordsAdded || 0;
-      if (results.products?.recordsUpdated) totalRecordsUpdated += results.products.recordsUpdated;
+      totalRecordsUpdated += results.products?.recordsUpdated || 0;
     }
 
     if (IMPORT_ORDERS) {
       results.orders = await importOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
       if (isImportCancelled) throw new Error("Import cancelled");
       completedSteps++;
-      if (results.orders?.recordsAdded) totalRecordsAdded += results.orders.recordsAdded;
+      totalRecordsAdded += results.orders?.recordsAdded || 0;
-      if (results.orders?.recordsUpdated) totalRecordsUpdated += results.orders.recordsUpdated;
+      totalRecordsUpdated += results.orders?.recordsUpdated || 0;
     }
 
     if (IMPORT_PURCHASE_ORDERS) {
       results.purchaseOrders = await importPurchaseOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
       if (isImportCancelled) throw new Error("Import cancelled");
       completedSteps++;
-      if (results.purchaseOrders?.recordsAdded) totalRecordsAdded += results.purchaseOrders.recordsAdded;
+      totalRecordsAdded += results.purchaseOrders?.recordsAdded || 0;
-      if (results.purchaseOrders?.recordsUpdated) totalRecordsUpdated += results.purchaseOrders.recordsUpdated;
+      totalRecordsUpdated += results.purchaseOrders?.recordsUpdated || 0;
     }
 
     const endTime = Date.now();

inventory-server/scripts/import/orders.js

@@ -432,8 +432,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
     return {
       status: "complete",
       totalImported: importedCount,
-      recordsAdded,
+      recordsAdded: recordsAdded || 0,
-      recordsUpdated,
+      recordsUpdated: recordsUpdated || 0,
       totalSkipped: skippedOrders.size,
       missingProducts: missingProducts.size,
       incrementalUpdate,

inventory-server/scripts/import/products.js

@@ -357,11 +357,46 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
     // Insert production data in batches, but only for products that need updates
     for (let i = 0; i < prodData.length; i += 1000) {
       const batch = prodData.slice(i, i + 1000);
-      const placeholders = batch.map(() => "(?)").join(",");
+      const placeholders = batch.map(() => `(${Array(31).fill("?").join(",")})`).join(",");
+      
+      // Map each row to exactly match our temp table columns
+      const values = batch.flatMap(row => [
+        row.pid,
+        row.title,
+        row.description,
+        row.SKU,
+        row.date_created,
+        row.first_received,
+        row.location,
+        row.barcode,
+        row.harmonized_tariff_code,
+        row.updated_at,
+        row.visible,
+        row.replenishable,
+        row.vendor,
+        row.vendor_reference,
+        row.notions_reference,
+        row.brand,
+        row.line,
+        row.subline,
+        row.artist,
+        row.moq,
+        row.rating,
+        row.reviews,
+        row.weight,
+        row.length,
+        row.width,
+        row.height,
+        row.total_sold,
+        row.country_of_origin,
+        row.date_last_sold,
+        row.category_ids,
+        true  // needs_update
+      ]);
 
       await localConnection.query(`
         INSERT INTO temp_prod_data VALUES ${placeholders}
-      `, batch.map(row => Object.values(row)));
+      `, values);
 
       outputProgress({
         status: "running",
@@ -378,7 +413,12 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
     let recordsAdded = 0;
     let recordsUpdated = 0;
     
-    while (processed < totalProducts) {
+    // Get actual count from temp table
+    const [[{ actualTotal }]] = await localConnection.query(
+      "SELECT COUNT(*) as actualTotal FROM temp_prod_data WHERE needs_update = 1"
+    );
+    
+    while (processed < actualTotal) {
       const [batch] = await localConnection.query(`
         SELECT 
           p.*,
@@ -395,6 +435,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
         LIMIT ? OFFSET ?
       `, [BATCH_SIZE, processed]);
 
+      if (!batch || batch.length === 0) break;  // Exit if no more records
+      
       // Add image URLs
       batch.forEach(row => {
         const urls = getImageUrls(row.pid);
@@ -413,24 +455,25 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
         })
       );
 
-      // MySQL 8.0 optimized insert
+      if (productValues.length > 0) {
+        // MySQL 8.0 optimized insert with proper placeholders
         const placeholderGroup = `(${Array(columnNames.length).fill("?").join(",")})`;
         const productPlaceholders = Array(batch.length).fill(placeholderGroup).join(",");
         
         const insertQuery = `
           INSERT INTO products (${columnNames.join(",")})
           VALUES ${productPlaceholders}
-        AS new_products
           ON DUPLICATE KEY UPDATE
             ${columnNames
               .filter(col => col !== "pid")
-            .map(col => `${col} = new_products.${col}`)
+              .map(col => `${col} = VALUES(${col})`)
               .join(",")};
         `;
 
         const result = await localConnection.query(insertQuery, productValues);
         recordsAdded += result.affectedRows - result.changedRows;
         recordsUpdated += result.changedRows;
+      }
 
       // Insert category relationships
       const categoryRelationships = [];
@@ -482,15 +525,16 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
         }
       }
 
-      processed += batch.length;
+      processed += batch.length;  // Only increment by actual records processed
+
       outputProgress({
         status: "running",
         operation: "Products import",
-        message: `Processed ${processed} of ${totalProducts} products`,
+        message: `Processed ${processed} of ${actualTotal} products`,
         current: processed,
-        total: totalProducts,
+        total: actualTotal,
         elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
-        remaining: estimateRemaining(startTime, processed, totalProducts),
+        remaining: estimateRemaining(startTime, processed, actualTotal),
         rate: calculateRate(startTime, processed)
       });
 
@@ -512,10 +556,10 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
 
     return {
       status: "complete",
-      totalImported: totalProducts,
+      totalImported: actualTotal,
-      recordsAdded,
+      recordsAdded: recordsAdded || 0,
-      recordsUpdated,
+      recordsUpdated: recordsUpdated || 0,
-      incrementalUpdate: true,
+      incrementalUpdate,
       lastSyncTime
     };
   } catch (error) {

inventory-server/scripts/import/purchase-orders.js

@@ -364,8 +364,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
     return {
       status: "complete",
       totalImported: totalItems,
-      recordsAdded,
+      recordsAdded: recordsAdded || 0,
-      recordsUpdated,
+      recordsUpdated: recordsUpdated || 0,
       incrementalUpdate,
       lastSyncTime
     };