- Modify import scripts to handle edge cases with empty arrays and null conditions
- Improve parameter handling in incremental update queries for purchase orders and products
This commit is contained in:
@@ -222,10 +222,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
|
||||
// Pre-check all products at once instead of per batch
|
||||
const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
|
||||
const [existingProducts] = await localConnection.query(
|
||||
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
|
||||
"SELECT pid FROM products WHERE pid IN (?)",
|
||||
[allOrderPids]
|
||||
);
|
||||
) : [[]];
|
||||
const existingPids = new Set(existingProducts.map(p => p.pid));
|
||||
|
||||
// Process in larger batches
|
||||
@@ -312,8 +312,15 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
// Import missing products if any
|
||||
if (missingProducts.size > 0) {
|
||||
try {
|
||||
// Setup temporary tables again since they were dropped
|
||||
await setupTemporaryTables(localConnection);
|
||||
await materializeCalculations(prodConnection, localConnection);
|
||||
|
||||
await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts));
|
||||
|
||||
// Clean up temporary tables after missing products import
|
||||
await cleanupTemporaryTables(localConnection);
|
||||
|
||||
// Retry skipped orders after importing products
|
||||
if (skippedOrders.size > 0) {
|
||||
outputProgress({
|
||||
@@ -322,7 +329,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
message: `Retrying import of ${skippedOrders.size} orders with previously missing products`
|
||||
});
|
||||
|
||||
const [skippedProdOrders] = await prodConnection.query(`
|
||||
const skippedOrdersArray = Array.from(skippedOrders);
|
||||
const [skippedProdOrders] = skippedOrdersArray.length > 0 ? await prodConnection.query(`
|
||||
SELECT
|
||||
o.order_id,
|
||||
CASE
|
||||
@@ -353,7 +361,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE o.order_id IN (?)
|
||||
`, [Array.from(skippedOrders)]);
|
||||
`, [skippedOrdersArray]) : [[]];
|
||||
|
||||
// Prepare values for insertion
|
||||
const skippedOrderValues = skippedProdOrders.flatMap(order => {
|
||||
@@ -420,7 +428,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync status - do this even if missing products import fails
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('orders', NOW())
|
||||
|
||||
@@ -331,8 +331,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
|
||||
pcp.date_active > ? OR
|
||||
sid.stamp > ? OR
|
||||
pnb.date_updated > ? OR
|
||||
pls.date_sold > ? OR
|
||||
si.stamp > ?
|
||||
pls.date_sold > ?
|
||||
` : 'TRUE'}
|
||||
THEN 1 ELSE 0 END as needs_update
|
||||
FROM products p
|
||||
@@ -349,7 +348,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
|
||||
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
|
||||
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
|
||||
GROUP BY p.pid
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
// Insert production data in batches, but only for products that need updates
|
||||
for (let i = 0; i < prodData.length; i += 1000) {
|
||||
@@ -500,7 +499,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
|
||||
// Drop temporary tables
|
||||
await cleanupTemporaryTables(localConnection);
|
||||
|
||||
// After successful import, update the sync status
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('products', NOW())
|
||||
|
||||
@@ -31,8 +31,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
||||
// Build incremental conditions
|
||||
const incrementalWhereClause = incrementalUpdate
|
||||
? `AND (
|
||||
p.stamp > ?
|
||||
OR p.date_updated > ?
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
OR r.date_updated > ?
|
||||
@@ -43,7 +42,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
||||
)`
|
||||
: "";
|
||||
const incrementalParams = incrementalUpdate
|
||||
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
|
||||
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
|
||||
: [];
|
||||
|
||||
// First get all relevant PO IDs with basic info
|
||||
@@ -56,16 +55,32 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
||||
JOIN po_products pop ON p.po_id = pop.po_id
|
||||
JOIN suppliers s ON p.supplier_id = s.supplierid
|
||||
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalWhereClause}
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
)
|
||||
` : ''}
|
||||
UNION
|
||||
SELECT DISTINCT r.receiving_id as po_id, rp.pid
|
||||
FROM receivings_products rp
|
||||
USE INDEX (received_date)
|
||||
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
|
||||
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalWhereClause}
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
r.date_created > ?
|
||||
OR r.date_checked > ?
|
||||
OR rp.stamp > ?
|
||||
OR rp.received_date > ?
|
||||
)
|
||||
` : ''}
|
||||
) all_items
|
||||
`, [...incrementalParams, ...incrementalParams]);
|
||||
`, incrementalUpdate ? [
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
|
||||
] : []);
|
||||
|
||||
const [poList] = await prodConnection.query(`
|
||||
SELECT DISTINCT
|
||||
@@ -337,7 +352,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync status with proper incrementing of last_sync_id
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('purchase_orders', NOW())
|
||||
|
||||
Reference in New Issue
Block a user