From cdce12e9fbfca1859f4dad2781991152d45b36f2 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 25 Jan 2025 21:43:31 -0500 Subject: [PATCH] Get orders import started, get all needed products imported --- inventory-server/db/schema.sql | 1 - inventory-server/scripts/import-from-prod.js | 180 +++++++++++++++---- 2 files changed, 146 insertions(+), 35 deletions(-) diff --git a/inventory-server/db/schema.sql b/inventory-server/db/schema.sql index 0d4af39..0ebf51a 100644 --- a/inventory-server/db/schema.sql +++ b/inventory-server/db/schema.sql @@ -122,7 +122,6 @@ CREATE TABLE orders ( billing_address TEXT, canceled BOOLEAN DEFAULT false, FOREIGN KEY (pid) REFERENCES products(pid), - FOREIGN KEY (SKU) REFERENCES products(SKU), INDEX idx_order_number (order_number), INDEX idx_customer (customer), INDEX idx_date (date), diff --git a/inventory-server/scripts/import-from-prod.js b/inventory-server/scripts/import-from-prod.js index b5b029a..557de83 100644 --- a/inventory-server/scripts/import-from-prod.js +++ b/inventory-server/scripts/import-from-prod.js @@ -238,7 +238,7 @@ async function importCategories(prodConnection, localConnection) { async function importProducts(prodConnection, localConnection) { outputProgress({ - operation: 'Starting products import', + operation: 'Starting products import - Getting schema', status: 'running' }); @@ -255,6 +255,27 @@ async function importProducts(prodConnection, localConnection) { const columnNames = columns.map(col => col.COLUMN_NAME); + // Get total count first for progress indication + outputProgress({ + operation: 'Starting products import - Getting total count', + status: 'running' + }); + + const [countResult] = await prodConnection.query(` + SELECT COUNT(*) as total + FROM products p + LEFT JOIN product_last_sold pls ON p.pid = pls.pid + WHERE pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + OR p.itemnumber LIKE 'chbx%' + `); + const totalProducts = countResult[0].total; + + outputProgress({ + operation: `Starting products import - Fetching ${totalProducts} products from production`, + status: 'running' + }); + // Get products from production with optimized query const [rows] = await prodConnection.query(` SELECT @@ -353,14 +374,16 @@ async function importProducts(prodConnection, localConnection) { WHERE active = 1 GROUP BY pid ) pcp ON p.pid = pcp.pid - WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + WHERE (pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + OR p.itemnumber LIKE 'chbx%') GROUP BY p.pid `); let current = 0; const total = rows.length; - // Process products in larger batches + // Process products in batches const BATCH_SIZE = 100; for (let i = 0; i < rows.length; i += BATCH_SIZE) { const batch = rows.slice(i, i + BATCH_SIZE); @@ -478,20 +501,61 @@ async function importProducts(prodConnection, localConnection) { } } +// Helper function to get date ranges for chunked queries +async function getDateRanges(prodConnection, table, dateField, startYearsAgo = 2, chunkMonths = 3) { + const ranges = []; + const [result] = await prodConnection.query(` + SELECT + DATE_SUB(CURRENT_DATE, INTERVAL ? YEAR) as start_date, + CURRENT_DATE as end_date + `, [startYearsAgo]); + + let currentDate = new Date(result[0].end_date); + const startDate = new Date(result[0].start_date); + + while (currentDate > startDate) { + const rangeEnd = new Date(currentDate); + currentDate.setMonth(currentDate.getMonth() - chunkMonths); + const rangeStart = new Date(Math.max(currentDate, startDate)); + + ranges.push({ + start: rangeStart.toISOString().split('T')[0], + end: rangeEnd.toISOString().split('T')[0] + }); + } + + return ranges; +} + async function importOrders(prodConnection, localConnection) { outputProgress({ - operation: 'Starting orders import', + operation: 'Starting orders import - Getting total count', status: 'running' }); const startTime = Date.now(); try { + // Get total count first for progress indication + const [countResult] = await prodConnection.query(` + SELECT COUNT(*) as total + FROM _order o + JOIN order_items oi ON o.order_id = oi.order_id + WHERE o.order_status >= 15 + AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR) + `); + const totalOrders = countResult[0].total; + + outputProgress({ + operation: `Starting orders import - Fetching ${totalOrders} orders from production`, + status: 'running' + }); + // Get orders from production const [rows] = await prodConnection.query(` SELECT oi.order_id AS order_number, - oi.prod_pid AS product_id, + oi.prod_pid AS pid, oi.prod_itemnumber AS SKU, o.date_placed_onlydate AS date, oi.prod_price_reg AS price, @@ -535,11 +599,31 @@ async function importOrders(prodConnection, localConnection) { let current = 0; const total = rows.length; + // Start transaction + await localConnection.query('START TRANSACTION'); + // Process in batches const BATCH_SIZE = 500; for (let i = 0; i < rows.length; i += BATCH_SIZE) { const batch = rows.slice(i, i + BATCH_SIZE); + // Get unique product IDs from this batch + const productIds = [...new Set(batch.map(row => row.pid))]; + + // Verify all products exist + const [existingProducts] = await localConnection.query( + 'SELECT pid FROM products WHERE pid IN (?)', + [productIds] + ); + + const existingProductIds = new Set(existingProducts.map(p => p.pid)); + const missingProducts = productIds.filter(pid => !existingProductIds.has(pid)); + + if (missingProducts.length > 0) { + console.error('Missing products:', missingProducts); + throw new Error(`Products missing from database: ${missingProducts.join(', ')}`); + } + // Create placeholders for batch insert const placeholders = batch.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' @@ -548,7 +632,7 @@ async function importOrders(prodConnection, localConnection) { // Flatten values for batch insert const values = batch.flatMap(row => [ row.order_number, - row.product_id, + row.pid, row.SKU, row.date, row.price, @@ -562,27 +646,35 @@ async function importOrders(prodConnection, localConnection) { row.canceled ]); - await localConnection.query(` - INSERT INTO orders ( - order_number, product_id, SKU, date, price, quantity, discount, - tax, tax_included, shipping, customer, customer_name, canceled - ) - VALUES ${placeholders} - ON DUPLICATE KEY UPDATE - price = VALUES(price), - quantity = VALUES(quantity), - discount = VALUES(discount), - tax = VALUES(tax), - tax_included = VALUES(tax_included), - shipping = VALUES(shipping), - customer_name = VALUES(customer_name), - canceled = VALUES(canceled) - `, values); + try { + await localConnection.query(` + INSERT INTO orders ( + order_number, pid, SKU, date, price, quantity, discount, + tax, tax_included, shipping, customer, customer_name, canceled + ) + VALUES ${placeholders} + ON DUPLICATE KEY UPDATE + price = VALUES(price), + quantity = VALUES(quantity), + discount = VALUES(discount), + tax = VALUES(tax), + tax_included = VALUES(tax_included), + shipping = VALUES(shipping), + customer_name = VALUES(customer_name), + canceled = VALUES(canceled) + `, values); + } catch (insertError) { + console.error('Error inserting batch:', insertError); + throw insertError; + } current += batch.length; updateProgress(current, total, 'Orders import', startTime); } + // If we get here, commit the transaction + await localConnection.query('COMMIT'); + outputProgress({ status: 'complete', operation: 'Orders import completed', @@ -591,6 +683,8 @@ async function importOrders(prodConnection, localConnection) { duration: formatDuration((Date.now() - startTime) / 1000) }); } catch (error) { + // Rollback on error + await localConnection.query('ROLLBACK'); console.error('Error importing orders:', error); throw error; } @@ -598,13 +692,18 @@ async function importOrders(prodConnection, localConnection) { async function importPurchaseOrders(prodConnection, localConnection) { outputProgress({ - operation: 'Starting purchase orders import', + operation: 'Starting purchase orders import - Initializing', status: 'running' }); const startTime = Date.now(); try { + outputProgress({ + operation: 'Starting purchase orders import - Fetching POs from production', + status: 'running' + }); + // Get purchase orders from production const [rows] = await prodConnection.query(` SELECT @@ -710,6 +809,12 @@ async function importPurchaseOrders(prodConnection, localConnection) { } // Modify main function to handle cancellation and avoid process.exit +// Constants to control which imports run +const IMPORT_CATEGORIES = true; +const IMPORT_PRODUCTS = true; +const IMPORT_ORDERS = true; +const IMPORT_PURCHASE_ORDERS = true; + async function main() { let ssh; let prodConnection; @@ -718,7 +823,7 @@ async function main() { try { outputProgress({ status: 'running', - operation: 'Starting import process', + operation: 'Starting import process', message: 'Setting up connections...' }); @@ -735,19 +840,26 @@ async function main() { if (isImportCancelled) throw new Error('Import cancelled'); - // First import all categories - await importCategories(prodConnection, localConnection); - if (isImportCancelled) throw new Error('Import cancelled'); + // Run each import based on constants + if (IMPORT_CATEGORIES) { + await importCategories(prodConnection, localConnection); + if (isImportCancelled) throw new Error('Import cancelled'); + } - // Then import products - await importProducts(prodConnection, localConnection); - if (isImportCancelled) throw new Error('Import cancelled'); + if (IMPORT_PRODUCTS) { + await importProducts(prodConnection, localConnection); + if (isImportCancelled) throw new Error('Import cancelled'); + } - await importOrders(prodConnection, localConnection); - if (isImportCancelled) throw new Error('Import cancelled'); + if (IMPORT_ORDERS) { + await importOrders(prodConnection, localConnection); + if (isImportCancelled) throw new Error('Import cancelled'); + } - await importPurchaseOrders(prodConnection, localConnection); - if (isImportCancelled) throw new Error('Import cancelled'); + if (IMPORT_PURCHASE_ORDERS) { + await importPurchaseOrders(prodConnection, localConnection); + if (isImportCancelled) throw new Error('Import cancelled'); + } outputProgress({ status: 'complete',