More incremental import fixes
This commit is contained in:
@@ -19,6 +19,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
const missingProducts = new Set();
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
let processedCount = 0;
|
||||
let importedCount = 0;
|
||||
let totalOrderItems = 0;
|
||||
let totalUniqueOrders = 0;
|
||||
|
||||
try {
|
||||
// Insert temporary table creation queries
|
||||
@@ -86,7 +90,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
|
||||
console.log('Orders: Using last sync time:', lastSyncTime);
|
||||
|
||||
// First get all relevant order items with basic info
|
||||
// First get count of order items
|
||||
const [[{ total }]] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM order_items oi
|
||||
@@ -115,7 +119,8 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
console.log('Orders: Found changes:', total);
|
||||
totalOrderItems = total;
|
||||
console.log('Orders: Found changes:', totalOrderItems);
|
||||
|
||||
// Get order items in batches
|
||||
const [orderItems] = await prodConnection.query(`
|
||||
@@ -155,9 +160,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
|
||||
console.log('Orders: Processing', orderItems.length, 'order items');
|
||||
|
||||
const totalOrders = orderItems.length;
|
||||
let processed = 0;
|
||||
|
||||
// Insert order items in batches
|
||||
for (let i = 0; i < orderItems.length; i += 5000) {
|
||||
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
|
||||
@@ -176,19 +178,23 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
base_discount = VALUES(base_discount)
|
||||
`, values);
|
||||
|
||||
processed += batch.length;
|
||||
processedCount = i + batch.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order items: ${processed} of ${totalOrders}`,
|
||||
current: processed,
|
||||
total: totalOrders
|
||||
message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
|
||||
current: processedCount,
|
||||
total: totalOrderItems
|
||||
});
|
||||
}
|
||||
|
||||
// Get unique order IDs
|
||||
const orderIds = [...new Set(orderItems.map(item => item.order_id))];
|
||||
console.log('Total unique order IDs:', orderIds.length);
|
||||
totalUniqueOrders = orderIds.length;
|
||||
console.log('Total unique order IDs:', totalUniqueOrders);
|
||||
|
||||
// Reset processed count for order processing phase
|
||||
processedCount = 0;
|
||||
|
||||
// Get order metadata in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
@@ -232,15 +238,19 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
canceled = VALUES(canceled)
|
||||
`, values);
|
||||
|
||||
processedCount = i + orders.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order metadata: ${i + orders.length} of ${orderIds.length}`,
|
||||
current: i + orders.length,
|
||||
total: orderIds.length
|
||||
message: `Loading order metadata: ${processedCount} of ${totalUniqueOrders}`,
|
||||
current: processedCount,
|
||||
total: totalUniqueOrders
|
||||
});
|
||||
}
|
||||
|
||||
// Reset processed count for final phase
|
||||
processedCount = 0;
|
||||
|
||||
// Get promotional discounts in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
@@ -321,8 +331,6 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
}
|
||||
|
||||
// Now combine all the data and insert into orders table
|
||||
let importedCount = 0;
|
||||
|
||||
// Pre-check all products at once instead of per batch
|
||||
const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
|
||||
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
|
||||
@@ -403,24 +411,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
return newVal !== oldVal;
|
||||
});
|
||||
if (hasChanges) {
|
||||
acc.updates.push({
|
||||
order_number: order.order_number,
|
||||
pid: order.pid,
|
||||
values: columnNames.map(col => order[col] ?? null)
|
||||
});
|
||||
} else {
|
||||
acc.inserts.push({
|
||||
order_number: order.order_number,
|
||||
pid: order.pid,
|
||||
values: columnNames.map(col => order[col] ?? null)
|
||||
});
|
||||
acc.updates.push(order);
|
||||
}
|
||||
} else {
|
||||
acc.inserts.push({
|
||||
order_number: order.order_number,
|
||||
pid: order.pid,
|
||||
values: columnNames.map(col => order[col] ?? null)
|
||||
});
|
||||
acc.inserts.push(order);
|
||||
}
|
||||
return acc;
|
||||
}, { inserts: [], updates: [] });
|
||||
@@ -432,9 +426,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
const insertResult = await localConnection.query(`
|
||||
INSERT INTO orders (${columnNames.join(",")})
|
||||
VALUES ${insertPlaceholders}
|
||||
`, insertsAndUpdates.inserts.map(i => i.values).flat());
|
||||
`, insertsAndUpdates.inserts.map(i => columnNames.map(col => i[col] ?? null)).flat());
|
||||
|
||||
recordsAdded += insertResult[0].affectedRows;
|
||||
importedCount += insertResult[0].affectedRows;
|
||||
}
|
||||
|
||||
// Handle updates - now we know these actually have changes
|
||||
@@ -456,24 +451,26 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
customer = VALUES(customer),
|
||||
customer_name = VALUES(customer_name),
|
||||
status = VALUES(status),
|
||||
canceled = VALUES(canceled)
|
||||
`, insertsAndUpdates.updates.map(u => u.values).flat());
|
||||
canceled = VALUES(canceled),
|
||||
costeach = VALUES(costeach)
|
||||
`, insertsAndUpdates.updates.map(u => columnNames.map(col => u[col] ?? null)).flat());
|
||||
|
||||
recordsUpdated += updateResult[0].affectedRows / 2; // Each update counts as 2 in affectedRows
|
||||
importedCount += updateResult[0].affectedRows / 2;
|
||||
}
|
||||
|
||||
importedCount += validOrders.length;
|
||||
}
|
||||
|
||||
// Update progress based on batch size - this is the number of order items we've processed
|
||||
processedCount = i + batchIds.length * (totalOrderItems / totalUniqueOrders);
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Imported ${importedCount} of ${totalOrders} orders`,
|
||||
current: importedCount,
|
||||
total: totalOrders,
|
||||
message: `Imported ${Math.floor(importedCount)} orders (${Math.floor(processedCount)} of ${totalOrderItems} items processed)`,
|
||||
current: Math.floor(processedCount),
|
||||
total: totalOrderItems,
|
||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
remaining: estimateRemaining(startTime, importedCount, totalOrders),
|
||||
rate: calculateRate(startTime, importedCount)
|
||||
remaining: estimateRemaining(startTime, processedCount, totalOrderItems),
|
||||
rate: calculateRate(startTime, processedCount)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -577,21 +574,34 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
INSERT INTO orders (${columnNames.join(", ")})
|
||||
VALUES ${skippedPlaceholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
${columnNames.map(col => `${col} = VALUES(${col})`).join(", ")}
|
||||
SKU = VALUES(SKU),
|
||||
date = VALUES(date),
|
||||
price = VALUES(price),
|
||||
quantity = VALUES(quantity),
|
||||
discount = VALUES(discount),
|
||||
tax = VALUES(tax),
|
||||
tax_included = VALUES(tax_included),
|
||||
shipping = VALUES(shipping),
|
||||
customer = VALUES(customer),
|
||||
customer_name = VALUES(customer_name),
|
||||
status = VALUES(status),
|
||||
canceled = VALUES(canceled),
|
||||
costeach = VALUES(costeach)
|
||||
`;
|
||||
|
||||
// Execute the insert query
|
||||
if (skippedOrderValues.length > 0) {
|
||||
await localConnection.query(skippedInsertQuery, skippedOrderValues.flat());
|
||||
const result = await localConnection.query(skippedInsertQuery, skippedOrderValues.flat());
|
||||
const addedOrUpdated = Math.floor(result[0].affectedRows / 2); // Round down to avoid fractional orders
|
||||
importedCount += addedOrUpdated;
|
||||
recordsUpdated += addedOrUpdated;
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Successfully imported ${addedOrUpdated} previously skipped orders`,
|
||||
});
|
||||
}
|
||||
|
||||
importedCount += skippedProdOrders.length;
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Successfully imported ${skippedProdOrders.length} previously skipped orders`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Warning: Failed to import missing products:', error.message);
|
||||
@@ -608,9 +618,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: importedCount,
|
||||
totalImported: Math.floor(importedCount), // Round down to avoid fractional orders
|
||||
recordsAdded: recordsAdded || 0,
|
||||
recordsUpdated: recordsUpdated || 0,
|
||||
recordsUpdated: Math.floor(recordsUpdated), // Round down to avoid fractional orders
|
||||
totalSkipped: skippedOrders.size,
|
||||
missingProducts: missingProducts.size,
|
||||
incrementalUpdate,
|
||||
|
||||
Reference in New Issue
Block a user