matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More incremental import fixes

Browse Source
This commit is contained in:
Matt
2025-02-01 11:42:51 -05:00
parent 2d0089dc52
commit 1003ff3cf2
3 changed files with 87 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
inventory-server/scripts/import/orders.js
View File

@@ -19,6 +19,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
const missingProducts = new Set(); const missingProducts = new Set();
let recordsAdded = 0; let recordsAdded = 0;
let recordsUpdated = 0; let recordsUpdated = 0;
let processedCount = 0;
let importedCount = 0;
let totalOrderItems = 0;
let totalUniqueOrders = 0;
try { try {
// Insert temporary table creation queries // Insert temporary table creation queries
@@ -86,7 +90,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
console.log('Orders: Using last sync time:', lastSyncTime); console.log('Orders: Using last sync time:', lastSyncTime);
// First get all relevant order items with basic info // First get count of order items
const [[{ total }]] = await prodConnection.query(` const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total SELECT COUNT(*) as total
FROM order_items oi FROM order_items oi
@@ -115,7 +119,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
` : ''} ` : ''}
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []); `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
console.log('Orders: Found changes:', total); totalOrderItems = total;
console.log('Orders: Found changes:', totalOrderItems);
// Get order items in batches // Get order items in batches
const [orderItems] = await prodConnection.query(` const [orderItems] = await prodConnection.query(`
@@ -155,9 +160,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
console.log('Orders: Processing', orderItems.length, 'order items'); console.log('Orders: Processing', orderItems.length, 'order items');
const totalOrders = orderItems.length;
let processed = 0;
// Insert order items in batches // Insert order items in batches
for (let i = 0; i < orderItems.length; i += 5000) { for (let i = 0; i < orderItems.length; i += 5000) {
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length)); const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
@@ -176,19 +178,23 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
base_discount = VALUES(base_discount) base_discount = VALUES(base_discount)
`, values); `, values);
processed += batch.length; processedCount = i + batch.length;
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Orders import", operation: "Orders import",
message: `Loading order items: ${processed} of ${totalOrders}`, message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
current: processed, current: processedCount,
total: totalOrders total: totalOrderItems
}); });
} }
// Get unique order IDs // Get unique order IDs
const orderIds = [...new Set(orderItems.map(item => item.order_id))]; const orderIds = [...new Set(orderItems.map(item => item.order_id))];
console.log('Total unique order IDs:', orderIds.length); totalUniqueOrders = orderIds.length;
console.log('Total unique order IDs:', totalUniqueOrders);
// Reset processed count for order processing phase
processedCount = 0;
// Get order metadata in batches // Get order metadata in batches
for (let i = 0; i < orderIds.length; i += 5000) { for (let i = 0; i < orderIds.length; i += 5000) {
@@ -232,15 +238,19 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
canceled = VALUES(canceled) canceled = VALUES(canceled)
`, values); `, values);
processedCount = i + orders.length;
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Orders import", operation: "Orders import",
message: `Loading order metadata: ${i + orders.length} of ${orderIds.length}`, message: `Loading order metadata: ${processedCount} of ${totalUniqueOrders}`,
current: i + orders.length, current: processedCount,
total: orderIds.length total: totalUniqueOrders
}); });
} }
// Reset processed count for final phase
processedCount = 0;
// Get promotional discounts in batches // Get promotional discounts in batches
for (let i = 0; i < orderIds.length; i += 5000) { for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000); const batchIds = orderIds.slice(i, i + 5000);
@@ -321,8 +331,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
} }
// Now combine all the data and insert into orders table // Now combine all the data and insert into orders table
let importedCount = 0;
// Pre-check all products at once instead of per batch // Pre-check all products at once instead of per batch
const allOrderPids = [...new Set(orderItems.map(item => item.pid))]; const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query( const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
@@ -403,24 +411,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
return newVal !== oldVal; return newVal !== oldVal;
}); });
if (hasChanges) { if (hasChanges) {
acc.updates.push({ acc.updates.push(order);
order_number: order.order_number,
pid: order.pid,
values: columnNames.map(col => order[col] ?? null)
});
} else {
acc.inserts.push({
order_number: order.order_number,
pid: order.pid,
values: columnNames.map(col => order[col] ?? null)
});
} }
} else { } else {
acc.inserts.push({ acc.inserts.push(order);
order_number: order.order_number,
pid: order.pid,
values: columnNames.map(col => order[col] ?? null)
});
} }
return acc; return acc;
}, { inserts: [], updates: [] }); }, { inserts: [], updates: [] });
@@ -432,9 +426,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
const insertResult = await localConnection.query(` const insertResult = await localConnection.query(`
INSERT INTO orders (${columnNames.join(",")}) INSERT INTO orders (${columnNames.join(",")})
VALUES ${insertPlaceholders} VALUES ${insertPlaceholders}
`, insertsAndUpdates.inserts.map(i => i.values).flat()); `, insertsAndUpdates.inserts.map(i => columnNames.map(col => i[col] ?? null)).flat());
recordsAdded += insertResult[0].affectedRows; recordsAdded += insertResult[0].affectedRows;
importedCount += insertResult[0].affectedRows;
} }
// Handle updates - now we know these actually have changes // Handle updates - now we know these actually have changes
@@ -456,24 +451,26 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
customer = VALUES(customer), customer = VALUES(customer),
customer_name = VALUES(customer_name), customer_name = VALUES(customer_name),
status = VALUES(status), status = VALUES(status),
canceled = VALUES(canceled) canceled = VALUES(canceled),
`, insertsAndUpdates.updates.map(u => u.values).flat()); costeach = VALUES(costeach)
`, insertsAndUpdates.updates.map(u => columnNames.map(col => u[col] ?? null)).flat());
recordsUpdated += updateResult[0].affectedRows / 2; // Each update counts as 2 in affectedRows recordsUpdated += updateResult[0].affectedRows / 2; // Each update counts as 2 in affectedRows
importedCount += updateResult[0].affectedRows / 2;
}
} }
importedCount += validOrders.length; // Update progress based on batch size - this is the number of order items we've processed
} processedCount = i + batchIds.length * (totalOrderItems / totalUniqueOrders);
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Orders import", operation: "Orders import",
message: `Imported ${importedCount} of ${totalOrders} orders`, message: `Imported ${Math.floor(importedCount)} orders (${Math.floor(processedCount)} of ${totalOrderItems} items processed)`,
current: importedCount, current: Math.floor(processedCount),
total: totalOrders, total: totalOrderItems,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000), elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, importedCount, totalOrders), remaining: estimateRemaining(startTime, processedCount, totalOrderItems),
rate: calculateRate(startTime, importedCount) rate: calculateRate(startTime, processedCount)
}); });
} }
@@ -577,22 +574,35 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
INSERT INTO orders (${columnNames.join(", ")}) INSERT INTO orders (${columnNames.join(", ")})
VALUES ${skippedPlaceholders} VALUES ${skippedPlaceholders}
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
${columnNames.map(col => `${col} = VALUES(${col})`).join(", ")} SKU = VALUES(SKU),
date = VALUES(date),
price = VALUES(price),
quantity = VALUES(quantity),
discount = VALUES(discount),
tax = VALUES(tax),
tax_included = VALUES(tax_included),
shipping = VALUES(shipping),
customer = VALUES(customer),
customer_name = VALUES(customer_name),
status = VALUES(status),
canceled = VALUES(canceled),
costeach = VALUES(costeach)
`; `;
// Execute the insert query // Execute the insert query
if (skippedOrderValues.length > 0) { if (skippedOrderValues.length > 0) {
await localConnection.query(skippedInsertQuery, skippedOrderValues.flat()); const result = await localConnection.query(skippedInsertQuery, skippedOrderValues.flat());
} const addedOrUpdated = Math.floor(result[0].affectedRows / 2); // Round down to avoid fractional orders
importedCount += addedOrUpdated;
importedCount += skippedProdOrders.length; recordsUpdated += addedOrUpdated;
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Orders import", operation: "Orders import",
message: `Successfully imported ${skippedProdOrders.length} previously skipped orders`, message: `Successfully imported ${addedOrUpdated} previously skipped orders`,
}); });
} }
}
} catch (error) { } catch (error) {
console.warn('Warning: Failed to import missing products:', error.message); console.warn('Warning: Failed to import missing products:', error.message);
console.warn(`Skipped ${skippedOrders.size} orders due to ${missingProducts.size} missing products`); console.warn(`Skipped ${skippedOrders.size} orders due to ${missingProducts.size} missing products`);
@@ -608,9 +618,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
return { return {
status: "complete", status: "complete",
totalImported: importedCount, totalImported: Math.floor(importedCount), // Round down to avoid fractional orders
recordsAdded: recordsAdded || 0, recordsAdded: recordsAdded || 0,
recordsUpdated: recordsUpdated || 0, recordsUpdated: Math.floor(recordsUpdated), // Round down to avoid fractional orders
totalSkipped: skippedOrders.size, totalSkipped: skippedOrders.size,
missingProducts: missingProducts.size, missingProducts: missingProducts.size,
incrementalUpdate, incrementalUpdate,

16
inventory-server/scripts/import/products.js
View File

@@ -582,7 +582,21 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
ELSE 1 ELSE 1
END AS replenishable, END AS replenishable,
COALESCE(si.available_local, 0) as stock_quantity, COALESCE(si.available_local, 0) as stock_quantity,
COALESCE(pq.qty, 0) as pending_qty, COALESCE(
(SELECT SUM(oi.qty_ordered - oi.qty_placed)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid = p.pid
AND o.date_placed != '0000-00-00 00:00:00'
AND o.date_shipped = '0000-00-00 00:00:00'
AND oi.pick_finished = 0
AND oi.qty_back = 0
AND o.order_status != 15
AND o.order_status < 90
AND oi.qty_ordered >= oi.qty_placed
AND oi.qty_ordered > 0
), 0
) as pending_qty,
COALESCE(ci.onpreorder, 0) as preorder_count, COALESCE(ci.onpreorder, 0) as preorder_count,
COALESCE(pnb.inventory, 0) as notions_inv_count, COALESCE(pnb.inventory, 0) as notions_inv_count,
COALESCE(pcp.price_each, 0) as price, COALESCE(pcp.price_each, 0) as price,

10
inventory-server/scripts/import/purchase-orders.js
View File

@@ -117,7 +117,12 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
WHEN r.receiving_id IS NOT NULL THEN WHEN r.receiving_id IS NOT NULL THEN
DATE(r.date_created) DATE(r.date_created)
END as date, END as date,
NULLIF(p.date_estin, '0000-00-00') as expected_date, CASE
WHEN p.date_estin = '0000-00-00' THEN NULL
WHEN p.date_estin IS NULL THEN NULL
WHEN p.date_estin NOT REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' THEN NULL
ELSE p.date_estin
END as expected_date,
COALESCE(p.status, 50) as status, COALESCE(p.status, 50) as status,
p.short_note as notes, p.short_note as notes,
p.notes as long_note p.notes as long_note
@@ -359,9 +364,12 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
function formatDate(dateStr) { function formatDate(dateStr) {
if (!dateStr) return null; if (!dateStr) return null;
if (dateStr === '0000-00-00' || dateStr === '0000-00-00 00:00:00') return null;
if (typeof dateStr === 'string' && !dateStr.match(/^\d{4}-\d{2}-\d{2}/)) return null;
try { try {
const date = new Date(dateStr); const date = new Date(dateStr);
if (isNaN(date.getTime())) return null; if (isNaN(date.getTime())) return null;
if (date.getFullYear() < 1900 || date.getFullYear() > 2100) return null;
return date.toISOString().split('T')[0]; return date.toISOString().split('T')[0];
} catch (e) { } catch (e) {
return null; return null;