matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More import optimizations + reset optimizations

Browse Source
This commit is contained in:
Matt
2025-01-11 00:18:03 -05:00
parent 7ce5092b69
commit 0bc86a3fee
3 changed files with 183 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
inventory-server/scripts/import-csv.js
View File

@@ -124,7 +124,14 @@ async function importProducts(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -268,8 +275,16 @@ async function importProducts(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
// Process categories within the same transaction
@@ -304,7 +319,14 @@ async function importOrders(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -430,8 +452,16 @@ async function importOrders(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
} catch (error) {
console.error(`\nError processing batch:`, error.message);
@@ -456,7 +486,14 @@ async function importPurchaseOrders(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -583,8 +620,16 @@ async function importPurchaseOrders(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
} catch (error) {
console.error(`\nError processing batch:`, error.message);
@@ -617,8 +662,17 @@ async function main() {
// Import products first since they're referenced by other tables
await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv'));
await importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv'));
await importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'));
// Process orders and purchase orders in parallel
outputProgress({
operation: 'Starting parallel import',
message: 'Processing orders and purchase orders simultaneously...'
});
await Promise.all([
importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')),
importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'))
]);
outputProgress({
status: 'complete',