matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes/improvements for import scripts

Browse Source
This commit is contained in:
Matt
2025-03-24 22:27:44 -04:00
parent 75da2c6772
commit 87d4b9e804
5 changed files with 190 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
inventory-server/scripts/import-from-prod.js
View File

@@ -10,9 +10,9 @@ const importPurchaseOrders = require('./import/purchase-orders');
dotenv.config({ path: path.join(__dirname, "../.env") }); dotenv.config({ path: path.join(__dirname, "../.env") });
// Constants to control which imports run // Constants to control which imports run
const IMPORT_CATEGORIES = false; const IMPORT_CATEGORIES = true;
const IMPORT_PRODUCTS = false; const IMPORT_PRODUCTS = true;
const IMPORT_ORDERS = false; const IMPORT_ORDERS = true;
const IMPORT_PURCHASE_ORDERS = true; const IMPORT_PURCHASE_ORDERS = true;
// Add flag for incremental updates // Add flag for incremental updates
@@ -120,27 +120,38 @@ async function main() {
`); `);
// Create import history record for the overall session // Create import history record for the overall session
const [historyResult] = await localConnection.query(` try {
INSERT INTO import_history ( const [historyResult] = await localConnection.query(`
table_name, INSERT INTO import_history (
start_time, table_name,
is_incremental, start_time,
status, is_incremental,
additional_info status,
) VALUES ( additional_info
'all_tables', ) VALUES (
NOW(), 'all_tables',
$1::boolean, NOW(),
'running', $1::boolean,
jsonb_build_object( 'running',
'categories_enabled', $2::boolean, jsonb_build_object(
'products_enabled', $3::boolean, 'categories_enabled', $2::boolean,
'orders_enabled', $4::boolean, 'products_enabled', $3::boolean,
'purchase_orders_enabled', $5::boolean 'orders_enabled', $4::boolean,
) 'purchase_orders_enabled', $5::boolean
) RETURNING id )
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]); ) RETURNING id
importHistoryId = historyResult.rows[0].id; `, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
importHistoryId = historyResult.rows[0].id;
} catch (error) {
console.error("Error creating import history record:", error);
outputProgress({
status: "error",
operation: "Import process",
message: "Failed to create import history record",
error: error.message
});
throw error;
}
const results = { const results = {
categories: null, categories: null,

41
inventory-server/scripts/import/categories.js
View File

@@ -47,42 +47,18 @@ async function importCategories(prodConnection, localConnection) {
continue; continue;
} }
console.log(`\nProcessing ${categories.length} type ${type} categories`); console.log(`Processing ${categories.length} type ${type} categories`);
if (type === 10) {
console.log("Type 10 categories:", JSON.stringify(categories, null, 2));
}
// For types that can have parents (11, 21, 12, 13), verify parent existence // For types that can have parents (11, 21, 12, 13), we'll proceed directly
// No need to check for parent existence since we process in hierarchical order
let categoriesToInsert = categories; let categoriesToInsert = categories;
if (![10, 20].includes(type)) {
// Get all parent IDs
const parentIds = [
...new Set(
categories
.filter(c => c && c.parent_id !== null)
.map(c => c.parent_id)
),
];
console.log(`Processing ${categories.length} type ${type} categories with ${parentIds.length} unique parent IDs`);
console.log('Parent IDs:', parentIds);
// No need to check for parent existence - we trust they exist since they were just inserted
categoriesToInsert = categories;
}
if (categoriesToInsert.length === 0) { if (categoriesToInsert.length === 0) {
console.log( console.log(`No valid categories of type ${type} to insert`);
`No valid categories of type ${type} to insert`
);
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`); await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
continue; continue;
} }
console.log(
`Inserting ${categoriesToInsert.length} type ${type} categories`
);
// PostgreSQL upsert query with parameterized values // PostgreSQL upsert query with parameterized values
const values = categoriesToInsert.flatMap((cat) => [ const values = categoriesToInsert.flatMap((cat) => [
cat.cat_id, cat.cat_id,
@@ -95,14 +71,10 @@ async function importCategories(prodConnection, localConnection) {
new Date() new Date()
]); ]);
console.log('Attempting to insert/update with values:', JSON.stringify(values, null, 2));
const placeholders = categoriesToInsert const placeholders = categoriesToInsert
.map((_, i) => `($${i * 8 + 1}, $${i * 8 + 2}, $${i * 8 + 3}, $${i * 8 + 4}, $${i * 8 + 5}, $${i * 8 + 6}, $${i * 8 + 7}, $${i * 8 + 8})`) .map((_, i) => `($${i * 8 + 1}, $${i * 8 + 2}, $${i * 8 + 3}, $${i * 8 + 4}, $${i * 8 + 5}, $${i * 8 + 6}, $${i * 8 + 7}, $${i * 8 + 8})`)
.join(','); .join(',');
console.log('Using placeholders:', placeholders);
// Insert categories with ON CONFLICT clause for PostgreSQL // Insert categories with ON CONFLICT clause for PostgreSQL
const query = ` const query = `
WITH inserted_categories AS ( WITH inserted_categories AS (
@@ -129,17 +101,14 @@ async function importCategories(prodConnection, localConnection) {
COUNT(*) FILTER (WHERE is_insert) as inserted, COUNT(*) FILTER (WHERE is_insert) as inserted,
COUNT(*) FILTER (WHERE NOT is_insert) as updated COUNT(*) FILTER (WHERE NOT is_insert) as updated
FROM inserted_categories`; FROM inserted_categories`;
console.log('Executing query:', query);
const result = await localConnection.query(query, values); const result = await localConnection.query(query, values);
console.log('Query result:', result);
// Get the first result since query returns an array // Get the first result since query returns an array
const queryResult = Array.isArray(result) ? result[0] : result; const queryResult = Array.isArray(result) ? result[0] : result;
if (!queryResult || !queryResult.rows || !queryResult.rows[0]) { if (!queryResult || !queryResult.rows || !queryResult.rows[0]) {
console.error('Query failed to return results. Result:', queryResult); console.error('Query failed to return results');
throw new Error('Query did not return expected results'); throw new Error('Query did not return expected results');
} }

70
inventory-server/scripts/import/orders.js
View File

@@ -26,6 +26,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
let cumulativeProcessedOrders = 0; let cumulativeProcessedOrders = 0;
try { try {
// Begin transaction
await localConnection.beginTransaction();
// Get last sync info // Get last sync info
const [syncInfo] = await localConnection.query( const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'" "SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
@@ -38,7 +41,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
const [[{ total }]] = await prodConnection.query(` const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total SELECT COUNT(*) as total
FROM order_items oi FROM order_items oi
USE INDEX (PRIMARY)
JOIN _order o ON oi.order_id = o.order_id JOIN _order o ON oi.order_id = o.order_id
WHERE o.order_status >= 15 WHERE o.order_status >= 15
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
@@ -78,7 +80,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount, COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
oi.stamp as last_modified oi.stamp as last_modified
FROM order_items oi FROM order_items oi
USE INDEX (PRIMARY)
JOIN _order o ON oi.order_id = o.order_id JOIN _order o ON oi.order_id = o.order_id
WHERE o.order_status >= 15 WHERE o.order_status >= 15
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
@@ -105,15 +106,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
console.log('Orders: Found', orderItems.length, 'order items to process'); console.log('Orders: Found', orderItems.length, 'order items to process');
// Create tables in PostgreSQL for debugging // Create tables in PostgreSQL for data processing
await localConnection.query(` await localConnection.query(`
DROP TABLE IF EXISTS debug_order_items; DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS debug_order_meta; DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS debug_order_discounts; DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS debug_order_taxes; DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS debug_order_costs; DROP TABLE IF EXISTS temp_order_costs;
CREATE TABLE debug_order_items ( CREATE TEMP TABLE temp_order_items (
order_id INTEGER NOT NULL, order_id INTEGER NOT NULL,
pid INTEGER NOT NULL, pid INTEGER NOT NULL,
SKU VARCHAR(50) NOT NULL, SKU VARCHAR(50) NOT NULL,
@@ -123,7 +124,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
PRIMARY KEY (order_id, pid) PRIMARY KEY (order_id, pid)
); );
CREATE TABLE debug_order_meta ( CREATE TEMP TABLE temp_order_meta (
order_id INTEGER NOT NULL, order_id INTEGER NOT NULL,
date DATE NOT NULL, date DATE NOT NULL,
customer VARCHAR(100) NOT NULL, customer VARCHAR(100) NOT NULL,
@@ -135,26 +136,29 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
PRIMARY KEY (order_id) PRIMARY KEY (order_id)
); );
CREATE TABLE debug_order_discounts ( CREATE TEMP TABLE temp_order_discounts (
order_id INTEGER NOT NULL, order_id INTEGER NOT NULL,
pid INTEGER NOT NULL, pid INTEGER NOT NULL,
discount DECIMAL(10,2) NOT NULL, discount DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid) PRIMARY KEY (order_id, pid)
); );
CREATE TABLE debug_order_taxes ( CREATE TEMP TABLE temp_order_taxes (
order_id INTEGER NOT NULL, order_id INTEGER NOT NULL,
pid INTEGER NOT NULL, pid INTEGER NOT NULL,
tax DECIMAL(10,2) NOT NULL, tax DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid) PRIMARY KEY (order_id, pid)
); );
CREATE TABLE debug_order_costs ( CREATE TEMP TABLE temp_order_costs (
order_id INTEGER NOT NULL, order_id INTEGER NOT NULL,
pid INTEGER NOT NULL, pid INTEGER NOT NULL,
costeach DECIMAL(10,3) DEFAULT 0.000, costeach DECIMAL(10,3) DEFAULT 0.000,
PRIMARY KEY (order_id, pid) PRIMARY KEY (order_id, pid)
); );
CREATE INDEX idx_temp_order_items_pid ON temp_order_items(pid);
CREATE INDEX idx_temp_order_meta_order_id ON temp_order_meta(order_id);
`); `);
// Insert order items in batches // Insert order items in batches
@@ -168,7 +172,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]); ]);
await localConnection.query(` await localConnection.query(`
INSERT INTO debug_order_items (order_id, pid, SKU, price, quantity, base_discount) INSERT INTO temp_order_items (order_id, pid, SKU, price, quantity, base_discount)
VALUES ${placeholders} VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET ON CONFLICT (order_id, pid) DO UPDATE SET
SKU = EXCLUDED.SKU, SKU = EXCLUDED.SKU,
@@ -239,7 +243,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]); ]);
await localConnection.query(` await localConnection.query(`
INSERT INTO debug_order_meta ( INSERT INTO temp_order_meta (
order_id, date, customer, customer_name, status, canceled, order_id, date, customer, customer_name, status, canceled,
summary_discount, summary_subtotal summary_discount, summary_subtotal
) )
@@ -281,7 +285,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]); ]);
await localConnection.query(` await localConnection.query(`
INSERT INTO debug_order_discounts (order_id, pid, discount) INSERT INTO temp_order_discounts (order_id, pid, discount)
VALUES ${placeholders} VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET ON CONFLICT (order_id, pid) DO UPDATE SET
discount = EXCLUDED.discount discount = EXCLUDED.discount
@@ -321,7 +325,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]); ]);
await localConnection.query(` await localConnection.query(`
INSERT INTO debug_order_taxes (order_id, pid, tax) INSERT INTO temp_order_taxes (order_id, pid, tax)
VALUES ${placeholders} VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET ON CONFLICT (order_id, pid) DO UPDATE SET
tax = EXCLUDED.tax tax = EXCLUDED.tax
@@ -366,7 +370,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]); ]);
await localConnection.query(` await localConnection.query(`
INSERT INTO debug_order_costs (order_id, pid, costeach) INSERT INTO temp_order_costs (order_id, pid, costeach)
VALUES ${placeholders} VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET ON CONFLICT (order_id, pid) DO UPDATE SET
costeach = EXCLUDED.costeach costeach = EXCLUDED.costeach
@@ -426,9 +430,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
SUM(COALESCE(od.discount, 0)) as promo_discount, SUM(COALESCE(od.discount, 0)) as promo_discount,
COALESCE(ot.tax, 0) as total_tax, COALESCE(ot.tax, 0) as total_tax,
COALESCE(oi.price * 0.5, 0) as costeach COALESCE(oi.price * 0.5, 0) as costeach
FROM debug_order_items oi FROM temp_order_items oi
LEFT JOIN debug_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid LEFT JOIN temp_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
LEFT JOIN debug_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
GROUP BY oi.order_id, oi.pid, ot.tax GROUP BY oi.order_id, oi.pid, ot.tax
) )
SELECT SELECT
@@ -456,11 +460,11 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
FROM ( FROM (
SELECT DISTINCT ON (order_id, pid) SELECT DISTINCT ON (order_id, pid)
order_id, pid, SKU, price, quantity, base_discount order_id, pid, SKU, price, quantity, base_discount
FROM debug_order_items FROM temp_order_items
WHERE order_id = ANY($1) WHERE order_id = ANY($1)
ORDER BY order_id, pid ORDER BY order_id, pid
) oi ) oi
JOIN debug_order_meta om ON oi.order_id = om.order_id JOIN temp_order_meta om ON oi.order_id = om.order_id
LEFT JOIN order_totals ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid LEFT JOIN order_totals ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
ORDER BY oi.order_id, oi.pid ORDER BY oi.order_id, oi.pid
`, [subBatchIds]); `, [subBatchIds]);
@@ -564,6 +568,18 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
ON CONFLICT (table_name) DO UPDATE SET ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW() last_sync_timestamp = NOW()
`); `);
// Cleanup temporary tables
await localConnection.query(`
DROP TABLE IF EXISTS temp_order_items;
DROP TABLE IF EXISTS temp_order_meta;
DROP TABLE IF EXISTS temp_order_discounts;
DROP TABLE IF EXISTS temp_order_taxes;
DROP TABLE IF EXISTS temp_order_costs;
`);
// Commit transaction
await localConnection.commit();
return { return {
status: "complete", status: "complete",
@@ -577,6 +593,14 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
}; };
} catch (error) { } catch (error) {
console.error("Error during orders import:", error); console.error("Error during orders import:", error);
// Rollback transaction
try {
await localConnection.rollback();
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
}
throw error; throw error;
} }
} }

58
inventory-server/scripts/import/products.js
View File

@@ -2,9 +2,12 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } =
const BATCH_SIZE = 100; // Smaller batch size for better progress tracking const BATCH_SIZE = 100; // Smaller batch size for better progress tracking
const MAX_RETRIES = 3; const MAX_RETRIES = 3;
const RETRY_DELAY = 5000; // 5 seconds const RETRY_DELAY = 5000; // 5 seconds
const dotenv = require("dotenv");
const path = require("path");
dotenv.config({ path: path.join(__dirname, "../../.env") });
// Utility functions // Utility functions
const imageUrlBase = 'https://sbing.com/i/products/0000/'; const imageUrlBase = process.env.PRODUCT_IMAGE_URL_BASE || 'https://sbing.com/i/products/0000/';
const getImageUrls = (pid, iid = 1) => { const getImageUrls = (pid, iid = 1) => {
const paddedPid = pid.toString().padStart(6, '0'); const paddedPid = pid.toString().padStart(6, '0');
// Use padded PID only for the first 3 digits // Use padded PID only for the first 3 digits
@@ -18,7 +21,7 @@ const getImageUrls = (pid, iid = 1) => {
}; };
}; };
// Add helper function for retrying operations // Add helper function for retrying operations with exponential backoff
async function withRetry(operation, errorMessage) { async function withRetry(operation, errorMessage) {
let lastError; let lastError;
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
@@ -28,7 +31,8 @@ async function withRetry(operation, errorMessage) {
lastError = error; lastError = error;
console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error); console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error);
if (attempt < MAX_RETRIES) { if (attempt < MAX_RETRIES) {
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); const backoffTime = RETRY_DELAY * Math.pow(2, attempt - 1);
await new Promise(resolve => setTimeout(resolve, backoffTime));
} }
} }
} }
@@ -772,32 +776,44 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
recordsAdded += parseInt(result.rows[0].inserted, 10) || 0; recordsAdded += parseInt(result.rows[0].inserted, 10) || 0;
recordsUpdated += parseInt(result.rows[0].updated, 10) || 0; recordsUpdated += parseInt(result.rows[0].updated, 10) || 0;
// Process category relationships for each product in the batch // Process category relationships in batches
const allCategories = [];
for (const row of batch) { for (const row of batch) {
if (row.categories) { if (row.categories) {
const categoryIds = row.categories.split(',').filter(id => id && id.trim()); const categoryIds = row.categories.split(',').filter(id => id && id.trim());
if (categoryIds.length > 0) { if (categoryIds.length > 0) {
const catPlaceholders = categoryIds.map((_, idx) => categoryIds.forEach(catId => {
`($${idx * 2 + 1}, $${idx * 2 + 2})` allCategories.push([row.pid, parseInt(catId.trim(), 10)]);
).join(','); });
const catValues = categoryIds.flatMap(catId => [row.pid, parseInt(catId.trim(), 10)]);
// First delete existing relationships for this product
await localConnection.query(
'DELETE FROM product_categories WHERE pid = $1',
[row.pid]
);
// Then insert the new relationships
await localConnection.query(`
INSERT INTO product_categories (pid, cat_id)
VALUES ${catPlaceholders}
ON CONFLICT (pid, cat_id) DO NOTHING
`, catValues);
} }
} }
} }
// If we have categories to process
if (allCategories.length > 0) {
// First get all products in this batch
const productIds = batch.map(p => p.pid);
// Delete all existing relationships for products in this batch
await localConnection.query(
'DELETE FROM product_categories WHERE pid = ANY($1)',
[productIds]
);
// Insert all new relationships in one batch
const catPlaceholders = allCategories.map((_, idx) =>
`($${idx * 2 + 1}, $${idx * 2 + 2})`
).join(',');
const catValues = allCategories.flat();
await localConnection.query(`
INSERT INTO product_categories (pid, cat_id)
VALUES ${catPlaceholders}
ON CONFLICT (pid, cat_id) DO NOTHING
`, catValues);
}
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Products import", operation: "Products import",

128
inventory-server/scripts/import/purchase-orders.js
View File

@@ -6,6 +6,9 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
let recordsUpdated = 0; let recordsUpdated = 0;
try { try {
// Begin transaction for the entire import process
await localConnection.beginTransaction();
// Get last sync info // Get last sync info
const [syncInfo] = await localConnection.query( const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'" "SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
@@ -39,7 +42,6 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
FROM ( FROM (
SELECT DISTINCT pop.po_id, pop.pid SELECT DISTINCT pop.po_id, pop.pid
FROM po p FROM po p
USE INDEX (idx_date_created)
JOIN po_products pop ON p.po_id = pop.po_id JOIN po_products pop ON p.po_id = pop.po_id
JOIN suppliers s ON p.supplier_id = s.supplierid JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR) WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
@@ -59,6 +61,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
console.log('Fetching purchase orders in batches...'); console.log('Fetching purchase orders in batches...');
const FETCH_BATCH_SIZE = 5000; const FETCH_BATCH_SIZE = 5000;
const INSERT_BATCH_SIZE = 200; // Process 200 records at a time for inserts
let offset = 0; let offset = 0;
let allProcessed = false; let allProcessed = false;
let totalProcessed = 0; let totalProcessed = 0;
@@ -101,64 +104,62 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
console.log(`Processing batch of ${poList.length} purchase order items (${offset}-${offset + poList.length})`); console.log(`Processing batch of ${poList.length} purchase order items (${offset}-${offset + poList.length})`);
let processed = 0; // Process in smaller batches for inserts
for (let i = 0; i < poList.length; i += INSERT_BATCH_SIZE) {
// Process each PO in a separate insert to avoid parameter issues const batch = poList.slice(i, Math.min(i + INSERT_BATCH_SIZE, poList.length));
for (let i = 0; i < poList.length; i++) {
const po = poList[i];
try { // Create parameterized query with placeholders
// Single row insert const placeholders = batch.map((_, idx) => {
await localConnection.query(` const base = idx * 11; // 11 columns
INSERT INTO temp_purchase_orders ( return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11})`;
po_id, pid, sku, name, vendor, date, expected_date, }).join(',');
status, notes, ordered, cost_price
) // Create flattened values array
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) const values = batch.flatMap(po => [
ON CONFLICT (po_id, pid) DO UPDATE SET po.po_id,
sku = EXCLUDED.sku, po.pid,
name = EXCLUDED.name, po.sku,
vendor = EXCLUDED.vendor, po.name,
date = EXCLUDED.date, po.vendor,
expected_date = EXCLUDED.expected_date, po.date,
status = EXCLUDED.status, po.expected_date,
notes = EXCLUDED.notes, po.status,
ordered = EXCLUDED.ordered, po.notes,
cost_price = EXCLUDED.cost_price po.ordered,
`, [ po.cost_price
po.po_id, ]);
po.pid,
po.sku, // Execute batch insert
po.name, await localConnection.query(`
po.vendor, INSERT INTO temp_purchase_orders (
po.date, po_id, pid, sku, name, vendor, date, expected_date,
po.expected_date, status, notes, ordered, cost_price
po.status, )
po.notes, VALUES ${placeholders}
po.ordered, ON CONFLICT (po_id, pid) DO UPDATE SET
po.cost_price sku = EXCLUDED.sku,
]); name = EXCLUDED.name,
vendor = EXCLUDED.vendor,
processed++; date = EXCLUDED.date,
totalProcessed++; expected_date = EXCLUDED.expected_date,
status = EXCLUDED.status,
// Only log occasionally notes = EXCLUDED.notes,
if (processed % 500 === 0 || processed === 1 || processed === poList.length) { ordered = EXCLUDED.ordered,
outputProgress({ cost_price = EXCLUDED.cost_price
status: "running", `, values);
operation: "Purchase orders import",
message: `Batch ${Math.floor(offset/FETCH_BATCH_SIZE) + 1}: ${processed}/${poList.length} (Total: ${totalProcessed}/${total})`, totalProcessed += batch.length;
current: totalProcessed,
total: total, outputProgress({
elapsed: formatElapsedTime((Date.now() - startTime) / 1000), status: "running",
remaining: estimateRemaining(startTime, totalProcessed, total), operation: "Purchase orders import",
rate: calculateRate(startTime, totalProcessed) message: `Processed ${totalProcessed}/${total} purchase order items`,
}); current: totalProcessed,
} total: total,
} catch (error) { elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
console.error(`Error inserting PO #${po.po_id} product #${po.pid}:`, error.message); remaining: estimateRemaining(startTime, totalProcessed, total),
console.log('PO data:', po); rate: calculateRate(startTime, totalProcessed)
} });
} }
// Update offset for next batch // Update offset for next batch
@@ -220,6 +221,9 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
// Clean up temporary tables // Clean up temporary tables
await localConnection.query(`DROP TABLE IF EXISTS temp_purchase_orders;`); await localConnection.query(`DROP TABLE IF EXISTS temp_purchase_orders;`);
// Commit transaction
await localConnection.commit();
return { return {
status: "complete", status: "complete",
@@ -230,11 +234,11 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
} catch (error) { } catch (error) {
console.error("Error during purchase orders import:", error); console.error("Error during purchase orders import:", error);
// Attempt cleanup on error // Rollback transaction
try { try {
await localConnection.query(`DROP TABLE IF EXISTS temp_purchase_orders;`); await localConnection.rollback();
} catch (cleanupError) { } catch (rollbackError) {
console.error('Error during cleanup:', cleanupError.message); console.error('Error during rollback:', rollbackError.message);
} }
return { return {