Add back product-category import and product time estimates

This commit is contained in:
2025-01-30 00:00:30 -05:00
parent 6c5f119ee5
commit 31d4011902

View File

@@ -412,13 +412,66 @@ async function importProducts(prodConnection, localConnection) {
await localConnection.query(insertQuery, productValues); await localConnection.query(insertQuery, productValues);
// Insert category relationships
const categoryRelationships = [];
batch.forEach(row => {
if (row.category_ids) {
const catIds = row.category_ids
.split(",")
.map(id => id.trim())
.filter(id => id)
.map(Number);
catIds.forEach(catId => {
if (catId) categoryRelationships.push([row.pid, catId]);
});
}
});
if (categoryRelationships.length > 0) {
// First verify categories exist
const uniqueCatIds = [...new Set(categoryRelationships.map(([_, catId]) => catId))];
const [existingCats] = await localConnection.query(
"SELECT cat_id FROM categories WHERE cat_id IN (?)",
[uniqueCatIds]
);
const existingCatIds = new Set(existingCats.map(c => c.cat_id));
// Filter relationships to only include existing categories
const validRelationships = categoryRelationships.filter(([_, catId]) =>
existingCatIds.has(catId)
);
if (validRelationships.length > 0) {
// Delete existing relationships for these products first
await localConnection.query(
"DELETE FROM product_categories WHERE pid IN (?)",
[batch.map(p => p.pid)]
);
// Insert new relationships using INSERT IGNORE
const catPlaceholders = validRelationships
.map(() => "(?, ?)")
.join(",");
await localConnection.query(
`INSERT IGNORE INTO product_categories (pid, cat_id)
VALUES ${catPlaceholders}`,
validRelationships.flat()
);
}
}
processed += batch.length; processed += batch.length;
outputProgress({ outputProgress({
status: "running", status: "running",
operation: "Products import", operation: "Products import",
message: `Processed ${processed} of ${totalProducts} products`, message: `Processed ${processed} of ${totalProducts} products`,
current: processed, current: processed,
total: totalProducts total: totalProducts,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, processed, totalProducts),
rate: calculateRate(startTime, processed)
}); });
// Force garbage collection between batches // Force garbage collection between batches
@@ -670,9 +723,8 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
.join(","); .join(",");
await localConnection.query( await localConnection.query(
` `
INSERT INTO product_categories (cat_id, pid) INSERT IGNORE INTO product_categories (cat_id, pid)
VALUES ${catPlaceholders} VALUES ${catPlaceholders}
ON DUPLICATE KEY UPDATE cat_id = VALUES(cat_id)
`, `,
validRelationships.flat() validRelationships.flat()
); );