Get orders import started, get all needed products imported

inventory-server/db/schema.sql

@@ -122,7 +122,6 @@ CREATE TABLE orders (
     billing_address TEXT,
     canceled BOOLEAN DEFAULT false,
     FOREIGN KEY (pid) REFERENCES products(pid),
-    FOREIGN KEY (SKU) REFERENCES products(SKU),
     INDEX idx_order_number (order_number),
     INDEX idx_customer (customer),
     INDEX idx_date (date),

inventory-server/scripts/import-from-prod.js

@@ -238,7 +238,7 @@ async function importCategories(prodConnection, localConnection) {
 
 async function importProducts(prodConnection, localConnection) {
   outputProgress({
-    operation: 'Starting products import',
+    operation: 'Starting products import - Getting schema',
     status: 'running'
   });
 
@@ -255,6 +255,27 @@ async function importProducts(prodConnection, localConnection) {
     
     const columnNames = columns.map(col => col.COLUMN_NAME);
 
+    // Get total count first for progress indication
+    outputProgress({
+      operation: 'Starting products import - Getting total count',
+      status: 'running'
+    });
+
+    const [countResult] = await prodConnection.query(`
+      SELECT COUNT(*) as total
+      FROM products p
+      LEFT JOIN product_last_sold pls ON p.pid = pls.pid
+      WHERE pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+        OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+        OR p.itemnumber LIKE 'chbx%'
+    `);
+    const totalProducts = countResult[0].total;
+
+    outputProgress({
+      operation: `Starting products import - Fetching ${totalProducts} products from production`,
+      status: 'running'
+    });
+
     // Get products from production with optimized query
     const [rows] = await prodConnection.query(`
       SELECT 
@@ -353,14 +374,16 @@ async function importProducts(prodConnection, localConnection) {
         WHERE active = 1
         GROUP BY pid
       ) pcp ON p.pid = pcp.pid
-      WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+      WHERE (pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+        OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+        OR p.itemnumber LIKE 'chbx%')
       GROUP BY p.pid
     `);
 
     let current = 0;
     const total = rows.length;
 
-    // Process products in larger batches
+    // Process products in batches
     const BATCH_SIZE = 100;
     for (let i = 0; i < rows.length; i += BATCH_SIZE) {
       const batch = rows.slice(i, i + BATCH_SIZE);
@@ -478,20 +501,61 @@ async function importProducts(prodConnection, localConnection) {
   }
 }
 
+// Helper function to get date ranges for chunked queries
+async function getDateRanges(prodConnection, table, dateField, startYearsAgo = 2, chunkMonths = 3) {
+  const ranges = [];
+  const [result] = await prodConnection.query(`
+    SELECT 
+      DATE_SUB(CURRENT_DATE, INTERVAL ? YEAR) as start_date,
+      CURRENT_DATE as end_date
+  `, [startYearsAgo]);
+  
+  let currentDate = new Date(result[0].end_date);
+  const startDate = new Date(result[0].start_date);
+  
+  while (currentDate > startDate) {
+    const rangeEnd = new Date(currentDate);
+    currentDate.setMonth(currentDate.getMonth() - chunkMonths);
+    const rangeStart = new Date(Math.max(currentDate, startDate));
+    
+    ranges.push({
+      start: rangeStart.toISOString().split('T')[0],
+      end: rangeEnd.toISOString().split('T')[0]
+    });
+  }
+  
+  return ranges;
+}
+
 async function importOrders(prodConnection, localConnection) {
   outputProgress({
-    operation: 'Starting orders import',
+    operation: 'Starting orders import - Getting total count',
     status: 'running'
   });
 
   const startTime = Date.now();
 
   try {
+    // Get total count first for progress indication
+    const [countResult] = await prodConnection.query(`
+      SELECT COUNT(*) as total
+      FROM _order o
+      JOIN order_items oi ON o.order_id = oi.order_id
+      WHERE o.order_status >= 15
+        AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+    `);
+    const totalOrders = countResult[0].total;
+
+    outputProgress({
+      operation: `Starting orders import - Fetching ${totalOrders} orders from production`,
+      status: 'running'
+    });
+
     // Get orders from production
     const [rows] = await prodConnection.query(`
       SELECT
         oi.order_id AS order_number,
-        oi.prod_pid AS product_id,
+        oi.prod_pid AS pid,
         oi.prod_itemnumber AS SKU,
         o.date_placed_onlydate AS date,
         oi.prod_price_reg AS price,
@@ -535,11 +599,31 @@ async function importOrders(prodConnection, localConnection) {
     let current = 0;
     const total = rows.length;
 
+    // Start transaction
+    await localConnection.query('START TRANSACTION');
+
     // Process in batches
     const BATCH_SIZE = 500;
     for (let i = 0; i < rows.length; i += BATCH_SIZE) {
       const batch = rows.slice(i, i + BATCH_SIZE);
       
+      // Get unique product IDs from this batch
+      const productIds = [...new Set(batch.map(row => row.pid))];
+      
+      // Verify all products exist
+      const [existingProducts] = await localConnection.query(
+        'SELECT pid FROM products WHERE pid IN (?)',
+        [productIds]
+      );
+      
+      const existingProductIds = new Set(existingProducts.map(p => p.pid));
+      const missingProducts = productIds.filter(pid => !existingProductIds.has(pid));
+      
+      if (missingProducts.length > 0) {
+        console.error('Missing products:', missingProducts);
+        throw new Error(`Products missing from database: ${missingProducts.join(', ')}`);
+      }
+
       // Create placeholders for batch insert
       const placeholders = batch.map(() => 
         '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
@@ -548,7 +632,7 @@ async function importOrders(prodConnection, localConnection) {
       // Flatten values for batch insert
       const values = batch.flatMap(row => [
         row.order_number,
-        row.product_id,
+        row.pid,
         row.SKU,
         row.date,
         row.price,
@@ -562,27 +646,35 @@ async function importOrders(prodConnection, localConnection) {
         row.canceled
       ]);
 
-      await localConnection.query(`
+      try {
-        INSERT INTO orders (
+        await localConnection.query(`
-          order_number, product_id, SKU, date, price, quantity, discount,
+          INSERT INTO orders (
-          tax, tax_included, shipping, customer, customer_name, canceled
+            order_number, pid, SKU, date, price, quantity, discount,
-        )
+            tax, tax_included, shipping, customer, customer_name, canceled
-        VALUES ${placeholders}
+          )
-        ON DUPLICATE KEY UPDATE
+          VALUES ${placeholders}
-          price = VALUES(price),
+          ON DUPLICATE KEY UPDATE
-          quantity = VALUES(quantity),
+            price = VALUES(price),
-          discount = VALUES(discount),
+            quantity = VALUES(quantity),
-          tax = VALUES(tax),
+            discount = VALUES(discount),
-          tax_included = VALUES(tax_included),
+            tax = VALUES(tax),
-          shipping = VALUES(shipping),
+            tax_included = VALUES(tax_included),
-          customer_name = VALUES(customer_name),
+            shipping = VALUES(shipping),
-          canceled = VALUES(canceled)
+            customer_name = VALUES(customer_name),
-      `, values);
+            canceled = VALUES(canceled)
+        `, values);
+      } catch (insertError) {
+        console.error('Error inserting batch:', insertError);
+        throw insertError;
+      }
 
       current += batch.length;
       updateProgress(current, total, 'Orders import', startTime);
     }
 
+    // If we get here, commit the transaction
+    await localConnection.query('COMMIT');
+
     outputProgress({
       status: 'complete',
       operation: 'Orders import completed',
@@ -591,6 +683,8 @@ async function importOrders(prodConnection, localConnection) {
       duration: formatDuration((Date.now() - startTime) / 1000)
     });
   } catch (error) {
+    // Rollback on error
+    await localConnection.query('ROLLBACK');
     console.error('Error importing orders:', error);
     throw error;
   }
@@ -598,13 +692,18 @@ async function importOrders(prodConnection, localConnection) {
 
 async function importPurchaseOrders(prodConnection, localConnection) {
   outputProgress({
-    operation: 'Starting purchase orders import',
+    operation: 'Starting purchase orders import - Initializing',
     status: 'running'
   });
 
   const startTime = Date.now();
 
   try {
+    outputProgress({
+      operation: 'Starting purchase orders import - Fetching POs from production',
+      status: 'running'
+    });
+
     // Get purchase orders from production
     const [rows] = await prodConnection.query(`
       SELECT 
@@ -710,6 +809,12 @@ async function importPurchaseOrders(prodConnection, localConnection) {
 }
 
 // Modify main function to handle cancellation and avoid process.exit
+// Constants to control which imports run
+const IMPORT_CATEGORIES = true;
+const IMPORT_PRODUCTS = true; 
+const IMPORT_ORDERS = true;
+const IMPORT_PURCHASE_ORDERS = true;
+
 async function main() {
   let ssh;
   let prodConnection;
@@ -718,7 +823,7 @@ async function main() {
   try {
     outputProgress({
       status: 'running',
-      operation: 'Starting import process',
+      operation: 'Starting import process', 
       message: 'Setting up connections...'
     });
 
@@ -735,19 +840,26 @@ async function main() {
 
     if (isImportCancelled) throw new Error('Import cancelled');
 
-    // First import all categories
+    // Run each import based on constants
-    await importCategories(prodConnection, localConnection);
+    if (IMPORT_CATEGORIES) {
-    if (isImportCancelled) throw new Error('Import cancelled');
+      await importCategories(prodConnection, localConnection);
+      if (isImportCancelled) throw new Error('Import cancelled');
+    }
     
-    // Then import products
+    if (IMPORT_PRODUCTS) {
-    await importProducts(prodConnection, localConnection);
+      await importProducts(prodConnection, localConnection);
-    if (isImportCancelled) throw new Error('Import cancelled');
+      if (isImportCancelled) throw new Error('Import cancelled');
+    }
     
-    await importOrders(prodConnection, localConnection);
+    if (IMPORT_ORDERS) {
-    if (isImportCancelled) throw new Error('Import cancelled');
+      await importOrders(prodConnection, localConnection);
+      if (isImportCancelled) throw new Error('Import cancelled');
+    }
     
-    await importPurchaseOrders(prodConnection, localConnection);
+    if (IMPORT_PURCHASE_ORDERS) {
-    if (isImportCancelled) throw new Error('Import cancelled');
+      await importPurchaseOrders(prodConnection, localConnection);
+      if (isImportCancelled) throw new Error('Import cancelled');
+    }
 
     outputProgress({
       status: 'complete',