Finish getting POs working

inventory-server/db/schema.sql

@@ -137,6 +137,7 @@ CREATE TABLE purchase_orders (
     cost_price DECIMAL(10, 3) NOT NULL,
     status VARCHAR(20) DEFAULT 'pending' COMMENT 'canceled,created,electronically_ready_send,ordered,preordered,electronically_sent,receiving_started,closed',
     notes TEXT,
+    long_note TEXT,
     ordered INT NOT NULL,
     received INT DEFAULT 0,
     received_date DATE,

inventory-server/scripts/import-from-prod.js

@@ -1118,67 +1118,92 @@ async function importPurchaseOrders(prodConnection, localConnection) {
       .map((col) => col.COLUMN_NAME)
       .filter((name) => name !== "id"); // Skip auto-increment ID
 
+    // Get total count first for progress indication
+    const [countResult] = await prodConnection.query(`
+      SELECT COUNT(*) as total
+      FROM po_products pop
+      JOIN po ON pop.po_id = po.po_id
+      WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+    `);
+    const total = countResult[0].total;
+
     outputProgress({
-      operation: "Starting purchase orders import - Fetching POs from production",
+      operation: `Starting purchase orders import - Fetching ${total} PO items from production`,
       status: "running",
     });
 
-    const [purchaseOrders] = await prodConnection.query(`
+    // Process in batches
-      SELECT 
+    const batchSize = 1000;
-        po.po_id as po_number,
+    let processed = 0;
-        s.companyname as vendor,
+    let offset = 0;
-        po.date_ordered as date,
-        po.date_estin as expected_date,
-        pop.pid as product_id,
-        p.itemnumber as sku,
-        COALESCE(rp.cost_each, pop.cost_each) as cost_price,
-        po.status,
-        CONCAT(COALESCE(po.short_note, ''), ' ', COALESCE(po.notes, '')) as notes,
-        pop.qty_each as ordered,
-        rp.received_by,
-        rp.received_date
-      FROM po
-      JOIN po_products pop ON po.po_id = pop.po_id
-      JOIN products p ON pop.pid = p.pid
-      JOIN suppliers s ON po.supplier_id = s.supplierid
-      LEFT JOIN receivings r ON po.po_id = r.po_id
-      LEFT JOIN receivings_products rp ON r.receiving_id = rp.receiving_id 
-          AND pop.pid = rp.pid
-      WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
-      ORDER BY po.po_id
-    `);
 
-    if (purchaseOrders.length > 0) {
+    while (offset < total) {
-      const placeholders = purchaseOrders
+      const [purchaseOrders] = await prodConnection.query(`
-        .map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
+        SELECT 
-        .join(",");
+          pop.po_id,
-      const updateClauses = columnNames
+          s.companyname as vendor,
-        .filter((col) => col !== "po_number") // Don't update primary key
+          DATE(po.date_ordered) as date,
-        .map((col) => `${col} = VALUES(${col})`)
+          DATE(po.date_estin) as expected_date,
-        .join(",");
+          pop.pid,
+          p.itemnumber as sku,
+          COALESCE(rp.cost_each, pop.cost_each) as cost_price,
+          po.status,
+          COALESCE(po.short_note, '') as notes,
+          COALESCE(po.notes, '') as long_note, 
+          pop.qty_each as ordered,
+          COALESCE(rp.qty_each, 0) as received,
+          DATE(rp.received_date) as received_date,
+          CAST(NULLIF(rp.received_by, '') AS SIGNED) as received_by
+        FROM po_products pop
+        JOIN po ON pop.po_id = po.po_id
+        JOIN products p ON pop.pid = p.pid
+        JOIN suppliers s ON po.supplier_id = s.supplierid
+        LEFT JOIN receivings r ON po.po_id = r.po_id
+        LEFT JOIN receivings_products rp ON r.receiving_id = rp.receiving_id 
+            AND pop.pid = rp.pid
+        WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
+        LIMIT ? OFFSET ?
+      `, [batchSize, offset]);
 
-      const query = `
+      if (purchaseOrders.length > 0) {
-        INSERT INTO purchase_orders (${columnNames.join(",")})
+        const placeholders = purchaseOrders
-        VALUES ${placeholders}
+          .map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
-        ON DUPLICATE KEY UPDATE ${updateClauses}
+          .join(",");
-      `;
+        const updateClauses = columnNames
+          .filter((col) => col !== "po_id") // Don't update primary key
+          .map((col) => `${col} = VALUES(${col})`)
+          .join(",");
 
-      await localConnection.query(
+        const query = `
-        query,
+          INSERT INTO purchase_orders (${columnNames.join(",")})
-        purchaseOrders.flatMap(po => columnNames.map(col => po[col]))
+          VALUES ${placeholders}
-      );
+          ON DUPLICATE KEY UPDATE ${updateClauses}
+        `;
 
-      outputProgress({
+        await localConnection.query(
-        operation: `Imported ${purchaseOrders.length} purchase order items`,
+          query,
-        status: "running",
+          purchaseOrders.flatMap(po => columnNames.map(col => po[col]))
-      });
+        );
+
+        processed += purchaseOrders.length;
+        offset += batchSize;
+
+        updateProgress(
+          processed,
+          total,
+          "Purchase orders import",
+          startTime
+        );
+      } else {
+        break;
+      }
     }
 
     const endTime = Date.now();
     outputProgress({
       operation: `Purchase orders import complete in ${Math.round(
         (endTime - startTime) / 1000
-      )}s - Imported ${purchaseOrders.length} PO items`,
+      )}s - Imported ${processed} PO items`,
       status: "complete",
     });
   } catch (error) {