matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add incremental import support and tracking for database synchronization

Browse Source
This commit is contained in:
Matt
2025-01-29 16:22:00 -05:00
parent d60b2d4fae
commit d2a2dbc812
7 changed files with 760 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
inventory-server/scripts/import/purchase-orders.js
View File

@@ -1,14 +1,37 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
async function importPurchaseOrders(prodConnection, localConnection) {
outputProgress({
operation: "Starting purchase orders import - Initializing",
status: "running",
});
const startTime = Date.now();
let importHistoryId;
try {
// Get last sync info
const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
);
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
// Create import history record
const [historyResult] = await localConnection.query(`
INSERT INTO import_history (
table_name,
start_time,
is_incremental,
status
) VALUES (
'purchase_orders',
NOW(),
?,
'running'
)
`, [!!syncInfo?.[0]]);
importHistoryId = historyResult.insertId;
outputProgress({
operation: "Starting purchase orders import - Initializing",
status: "running",
});
// Get column names for the insert
const [columns] = await localConnection.query(`
SELECT COLUMN_NAME
@@ -20,7 +43,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
.map((col) => col.COLUMN_NAME)
.filter((name) => name !== "id");
// First get all relevant PO IDs with basic info - this is much faster than the full join
// First get all relevant PO IDs with basic info - modified for incremental
const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM (
@@ -29,14 +52,17 @@ async function importPurchaseOrders(prodConnection, localConnection) {
FORCE INDEX (idx_date_created)
JOIN po_products pop ON p.po_id = pop.po_id
JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
WHERE p.date_ordered > ?
OR p.stamp > ?
OR p.date_modified > ?
UNION
SELECT DISTINCT r.receiving_id as po_id, rp.pid
FROM receivings_products rp
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
WHERE rp.received_date > ?
OR rp.stamp > ?
) all_items
`);
`, [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]);
const [poList] = await prodConnection.query(`
SELECT DISTINCT
@@ -53,22 +79,27 @@ async function importPurchaseOrders(prodConnection, localConnection) {
COALESCE(p.notes, '') as long_note
FROM (
SELECT po_id FROM po
WHERE date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
WHERE date_ordered > ?
OR stamp > ?
OR date_modified > ?
UNION
SELECT DISTINCT r.receiving_id as po_id
FROM receivings r
JOIN receivings_products rp ON r.receiving_id = rp.receiving_id
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
WHERE rp.received_date > ?
OR rp.stamp > ?
) ids
LEFT JOIN po p ON ids.po_id = p.po_id
LEFT JOIN suppliers s1 ON p.supplier_id = s1.supplierid
LEFT JOIN receivings r ON ids.po_id = r.receiving_id
LEFT JOIN suppliers s2 ON r.supplier_id = s2.supplierid
ORDER BY po_id
`);
`, [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]);
const totalItems = total;
let processed = 0;
let recordsAdded = 0;
let recordsUpdated = 0;
const BATCH_SIZE = 5000;
const PROGRESS_INTERVAL = 500;
@@ -249,7 +280,9 @@ async function importPurchaseOrders(prodConnection, localConnection) {
.join(",")};
`;
await localConnection.query(query, values.flat());
const result = await localConnection.query(query, values.flat());
recordsAdded += result.affectedRows - result.changedRows;
recordsUpdated += result.changedRows;
}
processed += batchProcessed;
@@ -271,19 +304,56 @@ async function importPurchaseOrders(prodConnection, localConnection) {
}
}
outputProgress({
status: "complete",
operation: "Purchase orders import completed",
current: totalItems,
total: totalItems,
duration: formatElapsedTime((Date.now() - startTime) / 1000),
});
// After successful import, update sync status
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW())
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW()
`);
// Update import history with final stats
const endTime = Date.now();
const durationSeconds = Math.round((endTime - startTime) / 1000);
await localConnection.query(`
UPDATE import_history
SET
end_time = NOW(),
duration_seconds = ?,
records_added = ?,
records_updated = ?,
status = 'completed',
additional_info = JSON_OBJECT(
'total_processed', ?,
'last_sync_time', ?,
'next_sync_time', NOW()
)
WHERE id = ?
`, [durationSeconds, recordsAdded, recordsUpdated, totalItems, lastSyncTime, importHistoryId]);
return {
status: "complete",
totalImported: totalItems
totalImported: totalItems,
recordsAdded,
recordsUpdated,
durationSeconds,
incrementalUpdate: !!syncInfo?.[0]
};
} catch (error) {
// Update import history with error
if (importHistoryId) {
await localConnection.query(`
UPDATE import_history
SET
end_time = NOW(),
duration_seconds = ?,
status = 'failed',
error_message = ?
WHERE id = ?
`, [Math.round((Date.now() - startTime) / 1000), error.message, importHistoryId]);
}
outputProgress({
operation: "Purchase orders import failed",
status: "error",