diff --git a/inventory-server/scripts/import/orders.js b/inventory-server/scripts/import/orders.js index 58d9329..af7f307 100644 --- a/inventory-server/scripts/import/orders.js +++ b/inventory-server/scripts/import/orders.js @@ -19,6 +19,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = const missingProducts = new Set(); let recordsAdded = 0; let recordsUpdated = 0; + let processedCount = 0; + let importedCount = 0; + let totalOrderItems = 0; + let totalUniqueOrders = 0; try { // Insert temporary table creation queries @@ -86,7 +90,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = console.log('Orders: Using last sync time:', lastSyncTime); - // First get all relevant order items with basic info + // First get count of order items const [[{ total }]] = await prodConnection.query(` SELECT COUNT(*) as total FROM order_items oi @@ -115,7 +119,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = ` : ''} `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []); - console.log('Orders: Found changes:', total); + totalOrderItems = total; + console.log('Orders: Found changes:', totalOrderItems); // Get order items in batches const [orderItems] = await prodConnection.query(` @@ -155,9 +160,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = console.log('Orders: Processing', orderItems.length, 'order items'); - const totalOrders = orderItems.length; - let processed = 0; - // Insert order items in batches for (let i = 0; i < orderItems.length; i += 5000) { const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length)); @@ -176,19 +178,23 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = base_discount = VALUES(base_discount) `, values); - processed += batch.length; + processedCount = i + batch.length; outputProgress({ status: "running", operation: "Orders import", - message: `Loading order items: ${processed} of ${totalOrders}`, - current: processed, - total: totalOrders + message: `Loading order items: ${processedCount} of ${totalOrderItems}`, + current: processedCount, + total: totalOrderItems }); } // Get unique order IDs const orderIds = [...new Set(orderItems.map(item => item.order_id))]; - console.log('Total unique order IDs:', orderIds.length); + totalUniqueOrders = orderIds.length; + console.log('Total unique order IDs:', totalUniqueOrders); + + // Reset processed count for order processing phase + processedCount = 0; // Get order metadata in batches for (let i = 0; i < orderIds.length; i += 5000) { @@ -232,15 +238,19 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = canceled = VALUES(canceled) `, values); + processedCount = i + orders.length; outputProgress({ status: "running", operation: "Orders import", - message: `Loading order metadata: ${i + orders.length} of ${orderIds.length}`, - current: i + orders.length, - total: orderIds.length + message: `Loading order metadata: ${processedCount} of ${totalUniqueOrders}`, + current: processedCount, + total: totalUniqueOrders }); } + // Reset processed count for final phase + processedCount = 0; + // Get promotional discounts in batches for (let i = 0; i < orderIds.length; i += 5000) { const batchIds = orderIds.slice(i, i + 5000); @@ -321,8 +331,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = } // Now combine all the data and insert into orders table - let importedCount = 0; - // Pre-check all products at once instead of per batch const allOrderPids = [...new Set(orderItems.map(item => item.pid))]; const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query( @@ -403,24 +411,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = return newVal !== oldVal; }); if (hasChanges) { - acc.updates.push({ - order_number: order.order_number, - pid: order.pid, - values: columnNames.map(col => order[col] ?? null) - }); - } else { - acc.inserts.push({ - order_number: order.order_number, - pid: order.pid, - values: columnNames.map(col => order[col] ?? null) - }); + acc.updates.push(order); } } else { - acc.inserts.push({ - order_number: order.order_number, - pid: order.pid, - values: columnNames.map(col => order[col] ?? null) - }); + acc.inserts.push(order); } return acc; }, { inserts: [], updates: [] }); @@ -432,9 +426,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = const insertResult = await localConnection.query(` INSERT INTO orders (${columnNames.join(",")}) VALUES ${insertPlaceholders} - `, insertsAndUpdates.inserts.map(i => i.values).flat()); + `, insertsAndUpdates.inserts.map(i => columnNames.map(col => i[col] ?? null)).flat()); recordsAdded += insertResult[0].affectedRows; + importedCount += insertResult[0].affectedRows; } // Handle updates - now we know these actually have changes @@ -456,24 +451,26 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = customer = VALUES(customer), customer_name = VALUES(customer_name), status = VALUES(status), - canceled = VALUES(canceled) - `, insertsAndUpdates.updates.map(u => u.values).flat()); + canceled = VALUES(canceled), + costeach = VALUES(costeach) + `, insertsAndUpdates.updates.map(u => columnNames.map(col => u[col] ?? null)).flat()); recordsUpdated += updateResult[0].affectedRows / 2; // Each update counts as 2 in affectedRows + importedCount += updateResult[0].affectedRows / 2; } - - importedCount += validOrders.length; } + // Update progress based on batch size - this is the number of order items we've processed + processedCount = i + batchIds.length * (totalOrderItems / totalUniqueOrders); outputProgress({ status: "running", operation: "Orders import", - message: `Imported ${importedCount} of ${totalOrders} orders`, - current: importedCount, - total: totalOrders, + message: `Imported ${Math.floor(importedCount)} orders (${Math.floor(processedCount)} of ${totalOrderItems} items processed)`, + current: Math.floor(processedCount), + total: totalOrderItems, elapsed: formatElapsedTime((Date.now() - startTime) / 1000), - remaining: estimateRemaining(startTime, importedCount, totalOrders), - rate: calculateRate(startTime, importedCount) + remaining: estimateRemaining(startTime, processedCount, totalOrderItems), + rate: calculateRate(startTime, processedCount) }); } @@ -577,21 +574,34 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = INSERT INTO orders (${columnNames.join(", ")}) VALUES ${skippedPlaceholders} ON DUPLICATE KEY UPDATE - ${columnNames.map(col => `${col} = VALUES(${col})`).join(", ")} + SKU = VALUES(SKU), + date = VALUES(date), + price = VALUES(price), + quantity = VALUES(quantity), + discount = VALUES(discount), + tax = VALUES(tax), + tax_included = VALUES(tax_included), + shipping = VALUES(shipping), + customer = VALUES(customer), + customer_name = VALUES(customer_name), + status = VALUES(status), + canceled = VALUES(canceled), + costeach = VALUES(costeach) `; // Execute the insert query if (skippedOrderValues.length > 0) { - await localConnection.query(skippedInsertQuery, skippedOrderValues.flat()); + const result = await localConnection.query(skippedInsertQuery, skippedOrderValues.flat()); + const addedOrUpdated = Math.floor(result[0].affectedRows / 2); // Round down to avoid fractional orders + importedCount += addedOrUpdated; + recordsUpdated += addedOrUpdated; + + outputProgress({ + status: "running", + operation: "Orders import", + message: `Successfully imported ${addedOrUpdated} previously skipped orders`, + }); } - - importedCount += skippedProdOrders.length; - - outputProgress({ - status: "running", - operation: "Orders import", - message: `Successfully imported ${skippedProdOrders.length} previously skipped orders`, - }); } } catch (error) { console.warn('Warning: Failed to import missing products:', error.message); @@ -608,9 +618,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate = return { status: "complete", - totalImported: importedCount, + totalImported: Math.floor(importedCount), // Round down to avoid fractional orders recordsAdded: recordsAdded || 0, - recordsUpdated: recordsUpdated || 0, + recordsUpdated: Math.floor(recordsUpdated), // Round down to avoid fractional orders totalSkipped: skippedOrders.size, missingProducts: missingProducts.size, incrementalUpdate, diff --git a/inventory-server/scripts/import/products.js b/inventory-server/scripts/import/products.js index 5986858..979cca7 100644 --- a/inventory-server/scripts/import/products.js +++ b/inventory-server/scripts/import/products.js @@ -582,7 +582,21 @@ async function importMissingProducts(prodConnection, localConnection, missingPid ELSE 1 END AS replenishable, COALESCE(si.available_local, 0) as stock_quantity, - COALESCE(pq.qty, 0) as pending_qty, + COALESCE( + (SELECT SUM(oi.qty_ordered - oi.qty_placed) + FROM order_items oi + JOIN _order o ON oi.order_id = o.order_id + WHERE oi.prod_pid = p.pid + AND o.date_placed != '0000-00-00 00:00:00' + AND o.date_shipped = '0000-00-00 00:00:00' + AND oi.pick_finished = 0 + AND oi.qty_back = 0 + AND o.order_status != 15 + AND o.order_status < 90 + AND oi.qty_ordered >= oi.qty_placed + AND oi.qty_ordered > 0 + ), 0 + ) as pending_qty, COALESCE(ci.onpreorder, 0) as preorder_count, COALESCE(pnb.inventory, 0) as notions_inv_count, COALESCE(pcp.price_each, 0) as price, diff --git a/inventory-server/scripts/import/purchase-orders.js b/inventory-server/scripts/import/purchase-orders.js index 7dacde7..cd87139 100644 --- a/inventory-server/scripts/import/purchase-orders.js +++ b/inventory-server/scripts/import/purchase-orders.js @@ -117,7 +117,12 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental WHEN r.receiving_id IS NOT NULL THEN DATE(r.date_created) END as date, - NULLIF(p.date_estin, '0000-00-00') as expected_date, + CASE + WHEN p.date_estin = '0000-00-00' THEN NULL + WHEN p.date_estin IS NULL THEN NULL + WHEN p.date_estin NOT REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' THEN NULL + ELSE p.date_estin + END as expected_date, COALESCE(p.status, 50) as status, p.short_note as notes, p.notes as long_note @@ -359,9 +364,12 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental function formatDate(dateStr) { if (!dateStr) return null; + if (dateStr === '0000-00-00' || dateStr === '0000-00-00 00:00:00') return null; + if (typeof dateStr === 'string' && !dateStr.match(/^\d{4}-\d{2}-\d{2}/)) return null; try { const date = new Date(dateStr); if (isNaN(date.getTime())) return null; + if (date.getFullYear() < 1900 || date.getFullYear() > 2100) return null; return date.toISOString().split('T')[0]; } catch (e) { return null;