diff --git a/inventory-server/db/schema.sql b/inventory-server/db/schema.sql index ba2b343..9835edf 100644 --- a/inventory-server/db/schema.sql +++ b/inventory-server/db/schema.sql @@ -137,6 +137,7 @@ CREATE TABLE purchase_orders ( cost_price DECIMAL(10, 3) NOT NULL, status VARCHAR(20) DEFAULT 'pending' COMMENT 'canceled,created,electronically_ready_send,ordered,preordered,electronically_sent,receiving_started,closed', notes TEXT, + long_note TEXT, ordered INT NOT NULL, received INT DEFAULT 0, received_date DATE, diff --git a/inventory-server/scripts/import-from-prod.js b/inventory-server/scripts/import-from-prod.js index 2c370ae..06aea98 100644 --- a/inventory-server/scripts/import-from-prod.js +++ b/inventory-server/scripts/import-from-prod.js @@ -1118,67 +1118,92 @@ async function importPurchaseOrders(prodConnection, localConnection) { .map((col) => col.COLUMN_NAME) .filter((name) => name !== "id"); // Skip auto-increment ID + // Get total count first for progress indication + const [countResult] = await prodConnection.query(` + SELECT COUNT(*) as total + FROM po_products pop + JOIN po ON pop.po_id = po.po_id + WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + `); + const total = countResult[0].total; + outputProgress({ - operation: "Starting purchase orders import - Fetching POs from production", + operation: `Starting purchase orders import - Fetching ${total} PO items from production`, status: "running", }); - const [purchaseOrders] = await prodConnection.query(` - SELECT - po.po_id as po_number, - s.companyname as vendor, - po.date_ordered as date, - po.date_estin as expected_date, - pop.pid as product_id, - p.itemnumber as sku, - COALESCE(rp.cost_each, pop.cost_each) as cost_price, - po.status, - CONCAT(COALESCE(po.short_note, ''), ' ', COALESCE(po.notes, '')) as notes, - pop.qty_each as ordered, - rp.received_by, - rp.received_date - FROM po - JOIN po_products pop ON po.po_id = pop.po_id - JOIN products p ON pop.pid = p.pid - JOIN suppliers s ON po.supplier_id = s.supplierid - LEFT JOIN receivings r ON po.po_id = r.po_id - LEFT JOIN receivings_products rp ON r.receiving_id = rp.receiving_id - AND pop.pid = rp.pid - WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) - ORDER BY po.po_id - `); + // Process in batches + const batchSize = 1000; + let processed = 0; + let offset = 0; - if (purchaseOrders.length > 0) { - const placeholders = purchaseOrders - .map(() => `(${Array(columnNames.length).fill("?").join(",")})`) - .join(","); - const updateClauses = columnNames - .filter((col) => col !== "po_number") // Don't update primary key - .map((col) => `${col} = VALUES(${col})`) - .join(","); + while (offset < total) { + const [purchaseOrders] = await prodConnection.query(` + SELECT + pop.po_id, + s.companyname as vendor, + DATE(po.date_ordered) as date, + DATE(po.date_estin) as expected_date, + pop.pid, + p.itemnumber as sku, + COALESCE(rp.cost_each, pop.cost_each) as cost_price, + po.status, + COALESCE(po.short_note, '') as notes, + COALESCE(po.notes, '') as long_note, + pop.qty_each as ordered, + COALESCE(rp.qty_each, 0) as received, + DATE(rp.received_date) as received_date, + CAST(NULLIF(rp.received_by, '') AS SIGNED) as received_by + FROM po_products pop + JOIN po ON pop.po_id = po.po_id + JOIN products p ON pop.pid = p.pid + JOIN suppliers s ON po.supplier_id = s.supplierid + LEFT JOIN receivings r ON po.po_id = r.po_id + LEFT JOIN receivings_products rp ON r.receiving_id = rp.receiving_id + AND pop.pid = rp.pid + WHERE po.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + LIMIT ? OFFSET ? + `, [batchSize, offset]); - const query = ` - INSERT INTO purchase_orders (${columnNames.join(",")}) - VALUES ${placeholders} - ON DUPLICATE KEY UPDATE ${updateClauses} - `; + if (purchaseOrders.length > 0) { + const placeholders = purchaseOrders + .map(() => `(${Array(columnNames.length).fill("?").join(",")})`) + .join(","); + const updateClauses = columnNames + .filter((col) => col !== "po_id") // Don't update primary key + .map((col) => `${col} = VALUES(${col})`) + .join(","); - await localConnection.query( - query, - purchaseOrders.flatMap(po => columnNames.map(col => po[col])) - ); + const query = ` + INSERT INTO purchase_orders (${columnNames.join(",")}) + VALUES ${placeholders} + ON DUPLICATE KEY UPDATE ${updateClauses} + `; - outputProgress({ - operation: `Imported ${purchaseOrders.length} purchase order items`, - status: "running", - }); + await localConnection.query( + query, + purchaseOrders.flatMap(po => columnNames.map(col => po[col])) + ); + + processed += purchaseOrders.length; + offset += batchSize; + + updateProgress( + processed, + total, + "Purchase orders import", + startTime + ); + } else { + break; + } } const endTime = Date.now(); outputProgress({ operation: `Purchase orders import complete in ${Math.round( (endTime - startTime) / 1000 - )}s - Imported ${purchaseOrders.length} PO items`, + )}s - Imported ${processed} PO items`, status: "complete", }); } catch (error) {