426 lines
15 KiB
JavaScript
426 lines
15 KiB
JavaScript
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
|
|
|
|
async function importPurchaseOrders(prodConnection, localConnection, incrementalUpdate = true) {
|
|
const startTime = Date.now();
|
|
let recordsAdded = 0;
|
|
let recordsUpdated = 0;
|
|
|
|
try {
|
|
// Get last sync info
|
|
const [syncInfo] = await localConnection.query(
|
|
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
|
|
);
|
|
const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
|
|
|
|
console.log('Purchase Orders: Using last sync time:', lastSyncTime);
|
|
|
|
// Create temporary tables with PostgreSQL syntax
|
|
await localConnection.query(`
|
|
DROP TABLE IF EXISTS temp_purchase_orders;
|
|
DROP TABLE IF EXISTS temp_po_receivings;
|
|
|
|
CREATE TEMP TABLE temp_purchase_orders (
|
|
po_id INTEGER NOT NULL,
|
|
pid INTEGER NOT NULL,
|
|
vendor VARCHAR(255),
|
|
date DATE,
|
|
expected_date DATE,
|
|
status INTEGER,
|
|
notes TEXT,
|
|
PRIMARY KEY (po_id, pid)
|
|
);
|
|
|
|
CREATE TEMP TABLE temp_po_receivings (
|
|
po_id INTEGER,
|
|
pid INTEGER NOT NULL,
|
|
receiving_id INTEGER NOT NULL,
|
|
qty_each INTEGER,
|
|
cost_each DECIMAL(10,3),
|
|
received_date TIMESTAMP,
|
|
received_by INTEGER,
|
|
received_by_name VARCHAR(255),
|
|
is_alt_po INTEGER,
|
|
PRIMARY KEY (receiving_id, pid)
|
|
);
|
|
`);
|
|
|
|
outputProgress({
|
|
operation: `Starting ${incrementalUpdate ? 'incremental' : 'full'} purchase orders import`,
|
|
status: "running",
|
|
});
|
|
|
|
// Get column names - Keep MySQL compatible for production
|
|
const [columns] = await prodConnection.query(`
|
|
SELECT COLUMN_NAME
|
|
FROM INFORMATION_SCHEMA.COLUMNS
|
|
WHERE TABLE_NAME = 'purchase_orders'
|
|
AND COLUMN_NAME != 'updated' -- Exclude the updated column
|
|
ORDER BY ORDINAL_POSITION
|
|
`);
|
|
const columnNames = columns.map(col => col.COLUMN_NAME);
|
|
|
|
// Build incremental conditions
|
|
const incrementalWhereClause = incrementalUpdate
|
|
? `AND (
|
|
p.date_updated > ?
|
|
OR p.date_ordered > ?
|
|
OR p.date_estin > ?
|
|
OR r.date_updated > ?
|
|
OR r.date_created > ?
|
|
OR r.date_checked > ?
|
|
OR rp.stamp > ?
|
|
OR rp.received_date > ?
|
|
)`
|
|
: "";
|
|
const incrementalParams = incrementalUpdate
|
|
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
|
|
: [];
|
|
|
|
// First get all relevant PO IDs with basic info - Keep MySQL compatible for production
|
|
const [[{ total }]] = await prodConnection.query(`
|
|
SELECT COUNT(*) as total
|
|
FROM (
|
|
SELECT DISTINCT pop.po_id, pop.pid
|
|
FROM po p
|
|
USE INDEX (idx_date_created)
|
|
JOIN po_products pop ON p.po_id = pop.po_id
|
|
JOIN suppliers s ON p.supplier_id = s.supplierid
|
|
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
|
${incrementalUpdate ? `
|
|
AND (
|
|
p.date_updated > ?
|
|
OR p.date_ordered > ?
|
|
OR p.date_estin > ?
|
|
)
|
|
` : ''}
|
|
UNION
|
|
SELECT DISTINCT r.receiving_id as po_id, rp.pid
|
|
FROM receivings_products rp
|
|
USE INDEX (received_date)
|
|
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
|
|
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
|
${incrementalUpdate ? `
|
|
AND (
|
|
r.date_created > ?
|
|
OR r.date_checked > ?
|
|
OR rp.stamp > ?
|
|
OR rp.received_date > ?
|
|
)
|
|
` : ''}
|
|
) all_items
|
|
`, incrementalUpdate ? [
|
|
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
|
|
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
|
|
] : []);
|
|
|
|
console.log('Purchase Orders: Found changes:', total);
|
|
|
|
// Get PO list - Keep MySQL compatible for production
|
|
const [poList] = await prodConnection.query(`
|
|
SELECT DISTINCT
|
|
COALESCE(p.po_id, r.receiving_id) as po_id,
|
|
COALESCE(
|
|
NULLIF(s1.companyname, ''),
|
|
NULLIF(s2.companyname, ''),
|
|
'Unknown Vendor'
|
|
) as vendor,
|
|
CASE
|
|
WHEN p.po_id IS NOT NULL THEN
|
|
DATE(COALESCE(
|
|
NULLIF(p.date_ordered, '0000-00-00 00:00:00'),
|
|
p.date_created
|
|
))
|
|
WHEN r.receiving_id IS NOT NULL THEN
|
|
DATE(r.date_created)
|
|
END as date,
|
|
CASE
|
|
WHEN p.date_estin = '0000-00-00' THEN NULL
|
|
WHEN p.date_estin IS NULL THEN NULL
|
|
WHEN p.date_estin NOT REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' THEN NULL
|
|
ELSE p.date_estin
|
|
END as expected_date,
|
|
COALESCE(p.status, 50) as status,
|
|
p.short_note as notes,
|
|
p.notes as long_note
|
|
FROM (
|
|
SELECT po_id FROM po
|
|
USE INDEX (idx_date_created)
|
|
WHERE date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
|
${incrementalUpdate ? `
|
|
AND (
|
|
date_ordered > ?
|
|
OR date_updated > ?
|
|
OR date_estin > ?
|
|
)
|
|
` : ''}
|
|
UNION
|
|
SELECT DISTINCT r.receiving_id as po_id
|
|
FROM receivings r
|
|
JOIN receivings_products rp USE INDEX (received_date) ON r.receiving_id = rp.receiving_id
|
|
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
|
${incrementalUpdate ? `
|
|
AND (
|
|
r.date_created > ?
|
|
OR r.date_checked > ?
|
|
OR rp.stamp > ?
|
|
OR rp.received_date > ?
|
|
)
|
|
` : ''}
|
|
) ids
|
|
LEFT JOIN po p ON ids.po_id = p.po_id
|
|
LEFT JOIN suppliers s1 ON p.supplier_id = s1.supplierid
|
|
LEFT JOIN receivings r ON ids.po_id = r.receiving_id
|
|
LEFT JOIN suppliers s2 ON r.supplier_id = s2.supplierid
|
|
ORDER BY po_id
|
|
`, incrementalUpdate ? [
|
|
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
|
|
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
|
|
] : []);
|
|
|
|
console.log('Sample PO dates:', poList.slice(0, 5).map(po => ({
|
|
po_id: po.po_id,
|
|
raw_date_ordered: po.raw_date_ordered,
|
|
raw_date_created: po.raw_date_created,
|
|
raw_date_estin: po.raw_date_estin,
|
|
computed_date: po.date,
|
|
expected_date: po.expected_date
|
|
})));
|
|
|
|
const totalItems = total;
|
|
let processed = 0;
|
|
|
|
const BATCH_SIZE = 5000;
|
|
const PROGRESS_INTERVAL = 500;
|
|
let lastProgressUpdate = Date.now();
|
|
|
|
outputProgress({
|
|
operation: `Starting purchase orders import - Processing ${totalItems} purchase order items`,
|
|
status: "running",
|
|
});
|
|
|
|
for (let i = 0; i < poList.length; i += BATCH_SIZE) {
|
|
const batch = poList.slice(i, Math.min(i + BATCH_SIZE, poList.length));
|
|
const poIds = batch.map(po => po.po_id);
|
|
|
|
// Get all products for these POs in one query - Keep MySQL compatible for production
|
|
const [poProducts] = await prodConnection.query(`
|
|
SELECT
|
|
pop.po_id,
|
|
pop.pid,
|
|
pr.itemnumber as sku,
|
|
pr.description as name,
|
|
pop.cost_each,
|
|
pop.qty_each as ordered
|
|
FROM po_products pop
|
|
USE INDEX (PRIMARY)
|
|
JOIN products pr ON pop.pid = pr.pid
|
|
WHERE pop.po_id IN (?)
|
|
`, [poIds]);
|
|
|
|
// Process PO products in smaller sub-batches to avoid packet size issues
|
|
const SUB_BATCH_SIZE = 5000;
|
|
for (let j = 0; j < poProducts.length; j += SUB_BATCH_SIZE) {
|
|
const productBatch = poProducts.slice(j, j + SUB_BATCH_SIZE);
|
|
const productPids = [...new Set(productBatch.map(p => p.pid))];
|
|
const batchPoIds = [...new Set(productBatch.map(p => p.po_id))];
|
|
|
|
// Get receivings for this batch with employee names - Keep MySQL compatible for production
|
|
const [receivings] = await prodConnection.query(`
|
|
SELECT
|
|
r.po_id,
|
|
rp.pid,
|
|
rp.receiving_id,
|
|
rp.qty_each,
|
|
rp.cost_each,
|
|
COALESCE(rp.received_date, r.date_created) as received_date,
|
|
rp.received_by,
|
|
CONCAT(e.firstname, ' ', e.lastname) as received_by_name,
|
|
CASE
|
|
WHEN r.po_id IS NULL THEN 2 -- No PO
|
|
WHEN r.po_id IN (?) THEN 0 -- Original PO
|
|
ELSE 1 -- Different PO
|
|
END as is_alt_po
|
|
FROM receivings_products rp
|
|
USE INDEX (received_date)
|
|
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
|
|
LEFT JOIN employees e ON rp.received_by = e.employeeid
|
|
WHERE rp.pid IN (?)
|
|
AND rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
|
|
ORDER BY r.po_id, rp.pid, rp.received_date
|
|
`, [batchPoIds, productPids]);
|
|
|
|
// Insert receivings into temp table
|
|
if (receivings.length > 0) {
|
|
const placeholders = receivings.map((_, idx) => {
|
|
const base = idx * 9;
|
|
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9})`;
|
|
}).join(',');
|
|
|
|
const values = receivings.flatMap(r => [
|
|
r.po_id,
|
|
r.pid,
|
|
r.receiving_id,
|
|
r.qty_each,
|
|
r.cost_each,
|
|
r.received_date,
|
|
r.received_by,
|
|
r.received_by_name,
|
|
r.is_alt_po
|
|
]);
|
|
|
|
await localConnection.query(`
|
|
INSERT INTO temp_po_receivings (
|
|
po_id, pid, receiving_id, qty_each, cost_each, received_date,
|
|
received_by, received_by_name, is_alt_po
|
|
)
|
|
VALUES ${placeholders}
|
|
ON CONFLICT (receiving_id, pid) DO UPDATE SET
|
|
po_id = EXCLUDED.po_id,
|
|
qty_each = EXCLUDED.qty_each,
|
|
cost_each = EXCLUDED.cost_each,
|
|
received_date = EXCLUDED.received_date,
|
|
received_by = EXCLUDED.received_by,
|
|
received_by_name = EXCLUDED.received_by_name,
|
|
is_alt_po = EXCLUDED.is_alt_po
|
|
`, values);
|
|
}
|
|
|
|
// Process each PO product
|
|
for (const product of productBatch) {
|
|
const po = batch.find(p => p.po_id === product.po_id);
|
|
if (!po) continue;
|
|
|
|
// Insert into temp_purchase_orders
|
|
const placeholders = `($1, $2, $3, $4, $5, $6, $7)`;
|
|
const values = [
|
|
product.po_id,
|
|
product.pid,
|
|
po.vendor,
|
|
po.date,
|
|
po.expected_date,
|
|
po.status,
|
|
po.notes || po.long_note
|
|
];
|
|
|
|
await localConnection.query(`
|
|
INSERT INTO temp_purchase_orders (
|
|
po_id, pid, vendor, date, expected_date, status, notes
|
|
)
|
|
VALUES ${placeholders}
|
|
ON CONFLICT (po_id, pid) DO UPDATE SET
|
|
vendor = EXCLUDED.vendor,
|
|
date = EXCLUDED.date,
|
|
expected_date = EXCLUDED.expected_date,
|
|
status = EXCLUDED.status,
|
|
notes = EXCLUDED.notes
|
|
`, values);
|
|
|
|
processed++;
|
|
|
|
// Update progress periodically
|
|
if (Date.now() - lastProgressUpdate > PROGRESS_INTERVAL) {
|
|
outputProgress({
|
|
status: "running",
|
|
operation: "Purchase orders import",
|
|
message: `Processing purchase orders: ${processed} of ${totalItems}`,
|
|
current: processed,
|
|
total: totalItems,
|
|
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
|
remaining: estimateRemaining(startTime, processed, totalItems),
|
|
rate: calculateRate(startTime, processed)
|
|
});
|
|
lastProgressUpdate = Date.now();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Insert final data into purchase_orders table
|
|
const [result] = await localConnection.query(`
|
|
WITH inserted_pos AS (
|
|
INSERT INTO purchase_orders (
|
|
po_id, pid, vendor, date, expected_date, status, notes,
|
|
received_qty, received_cost, last_received_date, last_received_by,
|
|
alt_po_received_qty, alt_po_last_received_date,
|
|
no_po_received_qty, no_po_last_received_date
|
|
)
|
|
SELECT
|
|
po.po_id,
|
|
po.pid,
|
|
po.vendor,
|
|
po.date,
|
|
po.expected_date,
|
|
po.status,
|
|
po.notes,
|
|
COALESCE(SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END), 0) as received_qty,
|
|
COALESCE(AVG(CASE WHEN r.is_alt_po = 0 THEN r.cost_each END), 0) as received_cost,
|
|
MAX(CASE WHEN r.is_alt_po = 0 THEN r.received_date END) as last_received_date,
|
|
MAX(CASE WHEN r.is_alt_po = 0 THEN r.received_by_name END) as last_received_by,
|
|
COALESCE(SUM(CASE WHEN r.is_alt_po = 1 THEN r.qty_each END), 0) as alt_po_received_qty,
|
|
MAX(CASE WHEN r.is_alt_po = 1 THEN r.received_date END) as alt_po_last_received_date,
|
|
COALESCE(SUM(CASE WHEN r.is_alt_po = 2 THEN r.qty_each END), 0) as no_po_received_qty,
|
|
MAX(CASE WHEN r.is_alt_po = 2 THEN r.received_date END) as no_po_last_received_date
|
|
FROM temp_purchase_orders po
|
|
LEFT JOIN temp_po_receivings r ON po.pid = r.pid
|
|
GROUP BY po.po_id, po.pid, po.vendor, po.date, po.expected_date, po.status, po.notes
|
|
ON CONFLICT (po_id, pid) DO UPDATE SET
|
|
vendor = EXCLUDED.vendor,
|
|
date = EXCLUDED.date,
|
|
expected_date = EXCLUDED.expected_date,
|
|
status = EXCLUDED.status,
|
|
notes = EXCLUDED.notes,
|
|
received_qty = EXCLUDED.received_qty,
|
|
received_cost = EXCLUDED.received_cost,
|
|
last_received_date = EXCLUDED.last_received_date,
|
|
last_received_by = EXCLUDED.last_received_by,
|
|
alt_po_received_qty = EXCLUDED.alt_po_received_qty,
|
|
alt_po_last_received_date = EXCLUDED.alt_po_last_received_date,
|
|
no_po_received_qty = EXCLUDED.no_po_received_qty,
|
|
no_po_last_received_date = EXCLUDED.no_po_last_received_date
|
|
RETURNING xmax
|
|
)
|
|
SELECT
|
|
COUNT(*) FILTER (WHERE xmax = 0) as inserted,
|
|
COUNT(*) FILTER (WHERE xmax <> 0) as updated
|
|
FROM inserted_pos
|
|
`);
|
|
|
|
recordsAdded = result.rows[0].inserted;
|
|
recordsUpdated = result.rows[0].updated;
|
|
|
|
// Update sync status
|
|
await localConnection.query(`
|
|
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
|
VALUES ('purchase_orders', NOW())
|
|
ON CONFLICT (table_name) DO UPDATE SET
|
|
last_sync_timestamp = NOW()
|
|
`);
|
|
|
|
// Clean up temporary tables
|
|
await localConnection.query(`
|
|
DROP TABLE IF EXISTS temp_purchase_orders;
|
|
DROP TABLE IF EXISTS temp_po_receivings;
|
|
`);
|
|
|
|
return {
|
|
status: "complete",
|
|
recordsAdded,
|
|
recordsUpdated,
|
|
totalRecords: processed
|
|
};
|
|
} catch (error) {
|
|
console.error("Error during purchase orders import:", error);
|
|
// Attempt cleanup on error
|
|
try {
|
|
await localConnection.query(`
|
|
DROP TABLE IF EXISTS temp_purchase_orders;
|
|
DROP TABLE IF EXISTS temp_po_receivings;
|
|
`);
|
|
} catch (cleanupError) {
|
|
console.error('Error during cleanup:', cleanupError);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
module.exports = importPurchaseOrders;
|