diff --git a/inventory-server/scripts/import-from-prod.js b/inventory-server/scripts/import-from-prod.js index 1fb63f1..a8390e6 100644 --- a/inventory-server/scripts/import-from-prod.js +++ b/inventory-server/scripts/import-from-prod.js @@ -161,32 +161,32 @@ async function main() { results.categories = await importCategories(prodConnection, localConnection); if (isImportCancelled) throw new Error("Import cancelled"); completedSteps++; - if (results.categories?.recordsAdded) totalRecordsAdded += results.categories.recordsAdded; - if (results.categories?.recordsUpdated) totalRecordsUpdated += results.categories.recordsUpdated; + totalRecordsAdded += results.categories?.recordsAdded || 0; + totalRecordsUpdated += results.categories?.recordsUpdated || 0; } if (IMPORT_PRODUCTS) { results.products = await importProducts(prodConnection, localConnection, INCREMENTAL_UPDATE); if (isImportCancelled) throw new Error("Import cancelled"); completedSteps++; - if (results.products?.recordsAdded) totalRecordsAdded += results.products.recordsAdded; - if (results.products?.recordsUpdated) totalRecordsUpdated += results.products.recordsUpdated; + totalRecordsAdded += results.products?.recordsAdded || 0; + totalRecordsUpdated += results.products?.recordsUpdated || 0; } if (IMPORT_ORDERS) { results.orders = await importOrders(prodConnection, localConnection, INCREMENTAL_UPDATE); if (isImportCancelled) throw new Error("Import cancelled"); completedSteps++; - if (results.orders?.recordsAdded) totalRecordsAdded += results.orders.recordsAdded; - if (results.orders?.recordsUpdated) totalRecordsUpdated += results.orders.recordsUpdated; + totalRecordsAdded += results.orders?.recordsAdded || 0; + totalRecordsUpdated += results.orders?.recordsUpdated || 0; } if (IMPORT_PURCHASE_ORDERS) { results.purchaseOrders = await importPurchaseOrders(prodConnection, localConnection, INCREMENTAL_UPDATE); if (isImportCancelled) throw new Error("Import cancelled"); completedSteps++; - if (results.purchaseOrders?.recordsAdded) totalRecordsAdded += results.purchaseOrders.recordsAdded; - if (results.purchaseOrders?.recordsUpdated) totalRecordsUpdated += results.purchaseOrders.recordsUpdated; + totalRecordsAdded += results.purchaseOrders?.recordsAdded || 0; + totalRecordsUpdated += results.purchaseOrders?.recordsUpdated || 0; } const endTime = Date.now(); diff --git a/inventory-server/scripts/import/orders.js b/inventory-server/scripts/import/orders.js index 1cfdea8..e739e7a 100644 --- a/inventory-server/scripts/import/orders.js +++ b/inventory-server/scripts/import/orders.js @@ -432,8 +432,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = return { status: "complete", totalImported: importedCount, - recordsAdded, - recordsUpdated, + recordsAdded: recordsAdded || 0, + recordsUpdated: recordsUpdated || 0, totalSkipped: skippedOrders.size, missingProducts: missingProducts.size, incrementalUpdate, diff --git a/inventory-server/scripts/import/products.js b/inventory-server/scripts/import/products.js index fa1f2ce..99dccc4 100644 --- a/inventory-server/scripts/import/products.js +++ b/inventory-server/scripts/import/products.js @@ -357,11 +357,46 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate // Insert production data in batches, but only for products that need updates for (let i = 0; i < prodData.length; i += 1000) { const batch = prodData.slice(i, i + 1000); - const placeholders = batch.map(() => "(?)").join(","); + const placeholders = batch.map(() => `(${Array(31).fill("?").join(",")})`).join(","); + // Map each row to exactly match our temp table columns + const values = batch.flatMap(row => [ + row.pid, + row.title, + row.description, + row.SKU, + row.date_created, + row.first_received, + row.location, + row.barcode, + row.harmonized_tariff_code, + row.updated_at, + row.visible, + row.replenishable, + row.vendor, + row.vendor_reference, + row.notions_reference, + row.brand, + row.line, + row.subline, + row.artist, + row.moq, + row.rating, + row.reviews, + row.weight, + row.length, + row.width, + row.height, + row.total_sold, + row.country_of_origin, + row.date_last_sold, + row.category_ids, + true // needs_update + ]); + await localConnection.query(` INSERT INTO temp_prod_data VALUES ${placeholders} - `, batch.map(row => Object.values(row))); + `, values); outputProgress({ status: "running", @@ -378,7 +413,12 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate let recordsAdded = 0; let recordsUpdated = 0; - while (processed < totalProducts) { + // Get actual count from temp table + const [[{ actualTotal }]] = await localConnection.query( + "SELECT COUNT(*) as actualTotal FROM temp_prod_data WHERE needs_update = 1" + ); + + while (processed < actualTotal) { const [batch] = await localConnection.query(` SELECT p.*, @@ -394,6 +434,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate WHERE p.needs_update = 1 LIMIT ? OFFSET ? `, [BATCH_SIZE, processed]); + + if (!batch || batch.length === 0) break; // Exit if no more records // Add image URLs batch.forEach(row => { @@ -413,24 +455,25 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate }) ); - // MySQL 8.0 optimized insert - const placeholderGroup = `(${Array(columnNames.length).fill("?").join(",")})`; - const productPlaceholders = Array(batch.length).fill(placeholderGroup).join(","); - - const insertQuery = ` - INSERT INTO products (${columnNames.join(",")}) - VALUES ${productPlaceholders} - AS new_products - ON DUPLICATE KEY UPDATE - ${columnNames - .filter(col => col !== "pid") - .map(col => `${col} = new_products.${col}`) - .join(",")}; - `; + if (productValues.length > 0) { + // MySQL 8.0 optimized insert with proper placeholders + const placeholderGroup = `(${Array(columnNames.length).fill("?").join(",")})`; + const productPlaceholders = Array(batch.length).fill(placeholderGroup).join(","); + + const insertQuery = ` + INSERT INTO products (${columnNames.join(",")}) + VALUES ${productPlaceholders} + ON DUPLICATE KEY UPDATE + ${columnNames + .filter(col => col !== "pid") + .map(col => `${col} = VALUES(${col})`) + .join(",")}; + `; - const result = await localConnection.query(insertQuery, productValues); - recordsAdded += result.affectedRows - result.changedRows; - recordsUpdated += result.changedRows; + const result = await localConnection.query(insertQuery, productValues); + recordsAdded += result.affectedRows - result.changedRows; + recordsUpdated += result.changedRows; + } // Insert category relationships const categoryRelationships = []; @@ -482,15 +525,16 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate } } - processed += batch.length; + processed += batch.length; // Only increment by actual records processed + outputProgress({ status: "running", operation: "Products import", - message: `Processed ${processed} of ${totalProducts} products`, + message: `Processed ${processed} of ${actualTotal} products`, current: processed, - total: totalProducts, + total: actualTotal, elapsed: formatElapsedTime((Date.now() - startTime) / 1000), - remaining: estimateRemaining(startTime, processed, totalProducts), + remaining: estimateRemaining(startTime, processed, actualTotal), rate: calculateRate(startTime, processed) }); @@ -512,10 +556,10 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate return { status: "complete", - totalImported: totalProducts, - recordsAdded, - recordsUpdated, - incrementalUpdate: true, + totalImported: actualTotal, + recordsAdded: recordsAdded || 0, + recordsUpdated: recordsUpdated || 0, + incrementalUpdate, lastSyncTime }; } catch (error) { diff --git a/inventory-server/scripts/import/purchase-orders.js b/inventory-server/scripts/import/purchase-orders.js index 369e369..f1add71 100644 --- a/inventory-server/scripts/import/purchase-orders.js +++ b/inventory-server/scripts/import/purchase-orders.js @@ -364,8 +364,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental return { status: "complete", totalImported: totalItems, - recordsAdded, - recordsUpdated, + recordsAdded: recordsAdded || 0, + recordsUpdated: recordsUpdated || 0, incrementalUpdate, lastSyncTime };