Break up prod import script into pieces and move csv scripts into folder

This commit is contained in:
2025-01-29 13:23:32 -05:00
parent 814d5d1a84
commit 84baa7e7d3
6 changed files with 187 additions and 266 deletions

View File

@@ -1,4 +1,4 @@
const { updateProgress, outputProgress, formatElapsedTime } = require('./utils');
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
const { importMissingProducts } = require('./products');
async function importOrders(prodConnection, localConnection) {
@@ -25,11 +25,6 @@ async function importOrders(prodConnection, localConnection) {
.filter((name) => name !== "id"); // Skip auto-increment ID
// Get total count first for progress indication
outputProgress({
operation: "Starting orders import - Getting total count",
status: "running",
});
const [countResult] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM order_items oi FORCE INDEX (PRIMARY)
@@ -132,12 +127,15 @@ async function importOrders(prodConnection, localConnection) {
processed += orders.length;
offset += batchSize;
updateProgress(
processed,
outputProgress({
status: "running",
operation: "Orders import",
current: processed,
total,
"Orders import",
startTime
);
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processed, total),
rate: calculateRate(startTime, processed)
});
}
// Now handle missing products and retry skipped orders
@@ -215,13 +213,20 @@ async function importOrders(prodConnection, localConnection) {
}
}
const endTime = Date.now();
outputProgress({
operation: `Orders import complete in ${Math.round(
(endTime - startTime) / 1000
)}s`,
status: "complete",
operation: "Orders import completed",
current: total,
total,
duration: formatElapsedTime(Date.now() - startTime),
});
return {
status: "complete",
totalImported: total,
missingProducts: missingProducts.size,
retriedOrders: skippedOrders.size
};
} catch (error) {
outputProgress({
operation: "Orders import failed",