Streamline incremental imports

This commit is contained in:
2025-01-31 10:01:50 -05:00
parent d0abe9d9a2
commit 996d3d36af
3 changed files with 77 additions and 141 deletions

View File

@@ -1,5 +1,5 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
const { importMissingProducts } = require('./products');
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
/**
* Imports orders from a production MySQL database to a local MySQL database.
@@ -312,15 +312,9 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
// Import missing products if any
if (missingProducts.size > 0) {
try {
// Setup temporary tables again since they were dropped
await setupTemporaryTables(localConnection);
await materializeCalculations(prodConnection, localConnection);
// Import missing products directly without materialization
await importMissingProducts(prodConnection, localConnection, Array.from(missingProducts));
// Clean up temporary tables after missing products import
await cleanupTemporaryTables(localConnection);
// Retry skipped orders after importing products
if (skippedOrders.size > 0) {
outputProgress({