Fix PO import not removing products from edited POs
This commit is contained in:
@@ -46,8 +46,10 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
let poRecordsAdded = 0;
|
let poRecordsAdded = 0;
|
||||||
let poRecordsUpdated = 0;
|
let poRecordsUpdated = 0;
|
||||||
|
let poRecordsDeleted = 0;
|
||||||
let receivingRecordsAdded = 0;
|
let receivingRecordsAdded = 0;
|
||||||
let receivingRecordsUpdated = 0;
|
let receivingRecordsUpdated = 0;
|
||||||
|
let receivingRecordsDeleted = 0;
|
||||||
let totalProcessed = 0;
|
let totalProcessed = 0;
|
||||||
|
|
||||||
// Batch size constants
|
// Batch size constants
|
||||||
@@ -257,148 +259,153 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
const totalPOs = poCount[0].total;
|
const totalPOs = poCount[0].total;
|
||||||
console.log(`Found ${totalPOs} relevant purchase orders`);
|
console.log(`Found ${totalPOs} relevant purchase orders`);
|
||||||
|
|
||||||
// Fetch and process POs in batches
|
// Skip processing if no POs to process
|
||||||
let offset = 0;
|
if (totalPOs === 0) {
|
||||||
let allPOsProcessed = false;
|
console.log('No purchase orders to process, skipping PO import step');
|
||||||
|
} else {
|
||||||
while (!allPOsProcessed) {
|
// Fetch and process POs in batches
|
||||||
const [poList] = await prodConnection.query(`
|
let offset = 0;
|
||||||
SELECT
|
let allPOsProcessed = false;
|
||||||
p.po_id,
|
|
||||||
p.supplier_id,
|
while (!allPOsProcessed) {
|
||||||
s.companyname AS vendor,
|
const [poList] = await prodConnection.query(`
|
||||||
p.status,
|
SELECT
|
||||||
p.notes AS long_note,
|
p.po_id,
|
||||||
p.short_note AS notes,
|
p.supplier_id,
|
||||||
p.date_created,
|
s.companyname AS vendor,
|
||||||
p.date_ordered,
|
p.status,
|
||||||
p.date_estin
|
p.notes AS long_note,
|
||||||
FROM po p
|
p.short_note AS notes,
|
||||||
LEFT JOIN suppliers s ON p.supplier_id = s.supplierid
|
p.date_created,
|
||||||
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
p.date_ordered,
|
||||||
${incrementalUpdate ? `
|
p.date_estin
|
||||||
AND (
|
FROM po p
|
||||||
p.date_updated > ?
|
LEFT JOIN suppliers s ON p.supplier_id = s.supplierid
|
||||||
OR p.date_ordered > ?
|
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||||
OR p.date_estin > ?
|
${incrementalUpdate ? `
|
||||||
|
AND (
|
||||||
|
p.date_updated > ?
|
||||||
|
OR p.date_ordered > ?
|
||||||
|
OR p.date_estin > ?
|
||||||
|
)
|
||||||
|
` : ''}
|
||||||
|
ORDER BY p.po_id
|
||||||
|
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
||||||
|
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||||
|
|
||||||
|
if (poList.length === 0) {
|
||||||
|
allPOsProcessed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get products for these POs
|
||||||
|
const poIds = poList.map(po => po.po_id);
|
||||||
|
|
||||||
|
const [poProducts] = await prodConnection.query(`
|
||||||
|
SELECT
|
||||||
|
pp.po_id,
|
||||||
|
pp.pid,
|
||||||
|
pp.qty_each,
|
||||||
|
pp.cost_each,
|
||||||
|
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
||||||
|
COALESCE(p.description, 'Unknown Product') AS name
|
||||||
|
FROM po_products pp
|
||||||
|
LEFT JOIN products p ON pp.pid = p.pid
|
||||||
|
WHERE pp.po_id IN (?)
|
||||||
|
`, [poIds]);
|
||||||
|
|
||||||
|
// Build complete PO records
|
||||||
|
const completePOs = [];
|
||||||
|
for (const product of poProducts) {
|
||||||
|
const po = poList.find(p => p.po_id == product.po_id);
|
||||||
|
if (!po) continue;
|
||||||
|
|
||||||
|
completePOs.push({
|
||||||
|
po_id: po.po_id.toString(),
|
||||||
|
pid: product.pid,
|
||||||
|
sku: product.sku,
|
||||||
|
name: product.name,
|
||||||
|
vendor: po.vendor || 'Unknown Vendor',
|
||||||
|
date: validateDate(po.date_ordered) || validateDate(po.date_created),
|
||||||
|
expected_date: validateDate(po.date_estin),
|
||||||
|
status: poStatusMap[po.status] || 'created',
|
||||||
|
notes: po.notes || '',
|
||||||
|
long_note: po.long_note || '',
|
||||||
|
ordered: product.qty_each,
|
||||||
|
po_cost_price: product.cost_each,
|
||||||
|
supplier_id: po.supplier_id,
|
||||||
|
date_created: validateDate(po.date_created),
|
||||||
|
date_ordered: validateDate(po.date_ordered)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert PO data in batches
|
||||||
|
for (let i = 0; i < completePOs.length; i += INSERT_BATCH_SIZE) {
|
||||||
|
const batch = completePOs.slice(i, i + INSERT_BATCH_SIZE);
|
||||||
|
|
||||||
|
const placeholders = batch.map((_, idx) => {
|
||||||
|
const base = idx * 15;
|
||||||
|
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
||||||
|
}).join(',');
|
||||||
|
|
||||||
|
const values = batch.flatMap(po => [
|
||||||
|
po.po_id,
|
||||||
|
po.pid,
|
||||||
|
po.sku,
|
||||||
|
po.name,
|
||||||
|
po.vendor,
|
||||||
|
po.date,
|
||||||
|
po.expected_date,
|
||||||
|
po.status,
|
||||||
|
po.notes,
|
||||||
|
po.long_note,
|
||||||
|
po.ordered,
|
||||||
|
po.po_cost_price,
|
||||||
|
po.supplier_id,
|
||||||
|
po.date_created,
|
||||||
|
po.date_ordered
|
||||||
|
]);
|
||||||
|
|
||||||
|
await localConnection.query(`
|
||||||
|
INSERT INTO temp_purchase_orders (
|
||||||
|
po_id, pid, sku, name, vendor, date, expected_date, status, notes, long_note,
|
||||||
|
ordered, po_cost_price, supplier_id, date_created, date_ordered
|
||||||
)
|
)
|
||||||
` : ''}
|
VALUES ${placeholders}
|
||||||
ORDER BY p.po_id
|
ON CONFLICT (po_id, pid) DO UPDATE SET
|
||||||
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
sku = EXCLUDED.sku,
|
||||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
name = EXCLUDED.name,
|
||||||
|
vendor = EXCLUDED.vendor,
|
||||||
if (poList.length === 0) {
|
date = EXCLUDED.date,
|
||||||
allPOsProcessed = true;
|
expected_date = EXCLUDED.expected_date,
|
||||||
break;
|
status = EXCLUDED.status,
|
||||||
}
|
notes = EXCLUDED.notes,
|
||||||
|
long_note = EXCLUDED.long_note,
|
||||||
// Get products for these POs
|
ordered = EXCLUDED.ordered,
|
||||||
const poIds = poList.map(po => po.po_id);
|
po_cost_price = EXCLUDED.po_cost_price,
|
||||||
|
supplier_id = EXCLUDED.supplier_id,
|
||||||
const [poProducts] = await prodConnection.query(`
|
date_created = EXCLUDED.date_created,
|
||||||
SELECT
|
date_ordered = EXCLUDED.date_ordered
|
||||||
pp.po_id,
|
`, values);
|
||||||
pp.pid,
|
}
|
||||||
pp.qty_each,
|
|
||||||
pp.cost_each,
|
|
||||||
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
|
||||||
COALESCE(p.description, 'Unknown Product') AS name
|
|
||||||
FROM po_products pp
|
|
||||||
LEFT JOIN products p ON pp.pid = p.pid
|
|
||||||
WHERE pp.po_id IN (?)
|
|
||||||
`, [poIds]);
|
|
||||||
|
|
||||||
// Build complete PO records
|
|
||||||
const completePOs = [];
|
|
||||||
for (const product of poProducts) {
|
|
||||||
const po = poList.find(p => p.po_id == product.po_id);
|
|
||||||
if (!po) continue;
|
|
||||||
|
|
||||||
completePOs.push({
|
offset += poList.length;
|
||||||
po_id: po.po_id.toString(),
|
totalProcessed += completePOs.length;
|
||||||
pid: product.pid,
|
|
||||||
sku: product.sku,
|
outputProgress({
|
||||||
name: product.name,
|
status: "running",
|
||||||
vendor: po.vendor || 'Unknown Vendor',
|
operation: "Purchase orders import",
|
||||||
date: validateDate(po.date_ordered) || validateDate(po.date_created),
|
message: `Processed ${offset} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
|
||||||
expected_date: validateDate(po.date_estin),
|
current: offset,
|
||||||
status: poStatusMap[po.status] || 'created',
|
total: totalPOs,
|
||||||
notes: po.notes || '',
|
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||||
long_note: po.long_note || '',
|
remaining: estimateRemaining(startTime, offset, totalPOs),
|
||||||
ordered: product.qty_each,
|
rate: calculateRate(startTime, offset)
|
||||||
po_cost_price: product.cost_each,
|
|
||||||
supplier_id: po.supplier_id,
|
|
||||||
date_created: validateDate(po.date_created),
|
|
||||||
date_ordered: validateDate(po.date_ordered)
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
// Insert PO data in batches
|
|
||||||
for (let i = 0; i < completePOs.length; i += INSERT_BATCH_SIZE) {
|
|
||||||
const batch = completePOs.slice(i, i + INSERT_BATCH_SIZE);
|
|
||||||
|
|
||||||
const placeholders = batch.map((_, idx) => {
|
if (poList.length < PO_BATCH_SIZE) {
|
||||||
const base = idx * 15;
|
allPOsProcessed = true;
|
||||||
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
}
|
||||||
}).join(',');
|
|
||||||
|
|
||||||
const values = batch.flatMap(po => [
|
|
||||||
po.po_id,
|
|
||||||
po.pid,
|
|
||||||
po.sku,
|
|
||||||
po.name,
|
|
||||||
po.vendor,
|
|
||||||
po.date,
|
|
||||||
po.expected_date,
|
|
||||||
po.status,
|
|
||||||
po.notes,
|
|
||||||
po.long_note,
|
|
||||||
po.ordered,
|
|
||||||
po.po_cost_price,
|
|
||||||
po.supplier_id,
|
|
||||||
po.date_created,
|
|
||||||
po.date_ordered
|
|
||||||
]);
|
|
||||||
|
|
||||||
await localConnection.query(`
|
|
||||||
INSERT INTO temp_purchase_orders (
|
|
||||||
po_id, pid, sku, name, vendor, date, expected_date, status, notes, long_note,
|
|
||||||
ordered, po_cost_price, supplier_id, date_created, date_ordered
|
|
||||||
)
|
|
||||||
VALUES ${placeholders}
|
|
||||||
ON CONFLICT (po_id, pid) DO UPDATE SET
|
|
||||||
sku = EXCLUDED.sku,
|
|
||||||
name = EXCLUDED.name,
|
|
||||||
vendor = EXCLUDED.vendor,
|
|
||||||
date = EXCLUDED.date,
|
|
||||||
expected_date = EXCLUDED.expected_date,
|
|
||||||
status = EXCLUDED.status,
|
|
||||||
notes = EXCLUDED.notes,
|
|
||||||
long_note = EXCLUDED.long_note,
|
|
||||||
ordered = EXCLUDED.ordered,
|
|
||||||
po_cost_price = EXCLUDED.po_cost_price,
|
|
||||||
supplier_id = EXCLUDED.supplier_id,
|
|
||||||
date_created = EXCLUDED.date_created,
|
|
||||||
date_ordered = EXCLUDED.date_ordered
|
|
||||||
`, values);
|
|
||||||
}
|
|
||||||
|
|
||||||
offset += poList.length;
|
|
||||||
totalProcessed += completePOs.length;
|
|
||||||
|
|
||||||
outputProgress({
|
|
||||||
status: "running",
|
|
||||||
operation: "Purchase orders import",
|
|
||||||
message: `Processed ${offset} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
|
|
||||||
current: offset,
|
|
||||||
total: totalPOs,
|
|
||||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
|
||||||
remaining: estimateRemaining(startTime, offset, totalPOs),
|
|
||||||
rate: calculateRate(startTime, offset)
|
|
||||||
});
|
|
||||||
|
|
||||||
if (poList.length < PO_BATCH_SIZE) {
|
|
||||||
allPOsProcessed = true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -424,183 +431,188 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
const totalReceivings = receivingCount[0].total;
|
const totalReceivings = receivingCount[0].total;
|
||||||
console.log(`Found ${totalReceivings} relevant receivings`);
|
console.log(`Found ${totalReceivings} relevant receivings`);
|
||||||
|
|
||||||
// Fetch and process receivings in batches
|
// Skip processing if no receivings to process
|
||||||
offset = 0; // Reset offset for receivings
|
if (totalReceivings === 0) {
|
||||||
let allReceivingsProcessed = false;
|
console.log('No receivings to process, skipping receivings import step');
|
||||||
|
} else {
|
||||||
while (!allReceivingsProcessed) {
|
// Fetch and process receivings in batches
|
||||||
const [receivingList] = await prodConnection.query(`
|
offset = 0; // Reset offset for receivings
|
||||||
SELECT
|
let allReceivingsProcessed = false;
|
||||||
r.receiving_id,
|
|
||||||
r.supplier_id,
|
while (!allReceivingsProcessed) {
|
||||||
r.status,
|
const [receivingList] = await prodConnection.query(`
|
||||||
r.notes,
|
SELECT
|
||||||
r.shipping,
|
r.receiving_id,
|
||||||
r.total_amount,
|
r.supplier_id,
|
||||||
r.hold,
|
r.status,
|
||||||
r.for_storefront,
|
r.notes,
|
||||||
r.date_created,
|
r.shipping,
|
||||||
r.date_paid,
|
r.total_amount,
|
||||||
r.date_checked
|
r.hold,
|
||||||
FROM receivings r
|
r.for_storefront,
|
||||||
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
r.date_created,
|
||||||
${incrementalUpdate ? `
|
r.date_paid,
|
||||||
AND (
|
r.date_checked
|
||||||
r.date_updated > ?
|
FROM receivings r
|
||||||
OR r.date_created > ?
|
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||||
|
${incrementalUpdate ? `
|
||||||
|
AND (
|
||||||
|
r.date_updated > ?
|
||||||
|
OR r.date_created > ?
|
||||||
|
)
|
||||||
|
` : ''}
|
||||||
|
ORDER BY r.receiving_id
|
||||||
|
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
||||||
|
`, incrementalUpdate ? [lastSyncTime, lastSyncTime] : []);
|
||||||
|
|
||||||
|
if (receivingList.length === 0) {
|
||||||
|
allReceivingsProcessed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get products for these receivings
|
||||||
|
const receivingIds = receivingList.map(r => r.receiving_id);
|
||||||
|
|
||||||
|
const [receivingProducts] = await prodConnection.query(`
|
||||||
|
SELECT
|
||||||
|
rp.receiving_id,
|
||||||
|
rp.pid,
|
||||||
|
rp.qty_each,
|
||||||
|
rp.qty_each_orig,
|
||||||
|
rp.cost_each,
|
||||||
|
rp.cost_each_orig,
|
||||||
|
rp.received_by,
|
||||||
|
rp.received_date,
|
||||||
|
r.date_created as receiving_created_date,
|
||||||
|
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
||||||
|
COALESCE(p.description, 'Unknown Product') AS name
|
||||||
|
FROM receivings_products rp
|
||||||
|
JOIN receivings r ON rp.receiving_id = r.receiving_id
|
||||||
|
LEFT JOIN products p ON rp.pid = p.pid
|
||||||
|
WHERE rp.receiving_id IN (?)
|
||||||
|
`, [receivingIds]);
|
||||||
|
|
||||||
|
// Build complete receiving records
|
||||||
|
const completeReceivings = [];
|
||||||
|
for (const product of receivingProducts) {
|
||||||
|
const receiving = receivingList.find(r => r.receiving_id == product.receiving_id);
|
||||||
|
if (!receiving) continue;
|
||||||
|
|
||||||
|
// Get employee name if available
|
||||||
|
let receivedByName = null;
|
||||||
|
if (product.received_by) {
|
||||||
|
const [employeeResult] = await localConnection.query(`
|
||||||
|
SELECT CONCAT(firstname, ' ', lastname) as full_name
|
||||||
|
FROM employee_names
|
||||||
|
WHERE employeeid = $1
|
||||||
|
`, [product.received_by]);
|
||||||
|
|
||||||
|
if (employeeResult.rows.length > 0) {
|
||||||
|
receivedByName = employeeResult.rows[0].full_name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get vendor name if available
|
||||||
|
let vendorName = 'Unknown Vendor';
|
||||||
|
if (receiving.supplier_id) {
|
||||||
|
const [vendorResult] = await localConnection.query(`
|
||||||
|
SELECT company_name
|
||||||
|
FROM temp_supplier_names
|
||||||
|
WHERE supplier_id = $1
|
||||||
|
`, [receiving.supplier_id]);
|
||||||
|
|
||||||
|
if (vendorResult.rows.length > 0) {
|
||||||
|
vendorName = vendorResult.rows[0].company_name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
completeReceivings.push({
|
||||||
|
receiving_id: receiving.receiving_id.toString(),
|
||||||
|
pid: product.pid,
|
||||||
|
sku: product.sku,
|
||||||
|
name: product.name,
|
||||||
|
vendor: vendorName,
|
||||||
|
qty_each: product.qty_each,
|
||||||
|
qty_each_orig: product.qty_each_orig,
|
||||||
|
cost_each: product.cost_each,
|
||||||
|
cost_each_orig: product.cost_each_orig,
|
||||||
|
received_by: product.received_by,
|
||||||
|
received_by_name: receivedByName,
|
||||||
|
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
|
||||||
|
receiving_created_date: validateDate(product.receiving_created_date),
|
||||||
|
supplier_id: receiving.supplier_id,
|
||||||
|
status: receivingStatusMap[receiving.status] || 'created'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert receiving data in batches
|
||||||
|
for (let i = 0; i < completeReceivings.length; i += INSERT_BATCH_SIZE) {
|
||||||
|
const batch = completeReceivings.slice(i, i + INSERT_BATCH_SIZE);
|
||||||
|
|
||||||
|
const placeholders = batch.map((_, idx) => {
|
||||||
|
const base = idx * 15;
|
||||||
|
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
||||||
|
}).join(',');
|
||||||
|
|
||||||
|
const values = batch.flatMap(r => [
|
||||||
|
r.receiving_id,
|
||||||
|
r.pid,
|
||||||
|
r.sku,
|
||||||
|
r.name,
|
||||||
|
r.vendor,
|
||||||
|
r.qty_each,
|
||||||
|
r.qty_each_orig,
|
||||||
|
r.cost_each,
|
||||||
|
r.cost_each_orig,
|
||||||
|
r.received_by,
|
||||||
|
r.received_by_name,
|
||||||
|
r.received_date,
|
||||||
|
r.receiving_created_date,
|
||||||
|
r.supplier_id,
|
||||||
|
r.status
|
||||||
|
]);
|
||||||
|
|
||||||
|
await localConnection.query(`
|
||||||
|
INSERT INTO temp_receivings (
|
||||||
|
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
||||||
|
cost_each, cost_each_orig, received_by, received_by_name,
|
||||||
|
received_date, receiving_created_date, supplier_id, status
|
||||||
)
|
)
|
||||||
` : ''}
|
VALUES ${placeholders}
|
||||||
ORDER BY r.receiving_id
|
ON CONFLICT (receiving_id, pid) DO UPDATE SET
|
||||||
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
sku = EXCLUDED.sku,
|
||||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime] : []);
|
name = EXCLUDED.name,
|
||||||
|
vendor = EXCLUDED.vendor,
|
||||||
if (receivingList.length === 0) {
|
qty_each = EXCLUDED.qty_each,
|
||||||
allReceivingsProcessed = true;
|
qty_each_orig = EXCLUDED.qty_each_orig,
|
||||||
break;
|
cost_each = EXCLUDED.cost_each,
|
||||||
}
|
cost_each_orig = EXCLUDED.cost_each_orig,
|
||||||
|
received_by = EXCLUDED.received_by,
|
||||||
// Get products for these receivings
|
received_by_name = EXCLUDED.received_by_name,
|
||||||
const receivingIds = receivingList.map(r => r.receiving_id);
|
received_date = EXCLUDED.received_date,
|
||||||
|
receiving_created_date = EXCLUDED.receiving_created_date,
|
||||||
const [receivingProducts] = await prodConnection.query(`
|
supplier_id = EXCLUDED.supplier_id,
|
||||||
SELECT
|
status = EXCLUDED.status
|
||||||
rp.receiving_id,
|
`, values);
|
||||||
rp.pid,
|
|
||||||
rp.qty_each,
|
|
||||||
rp.qty_each_orig,
|
|
||||||
rp.cost_each,
|
|
||||||
rp.cost_each_orig,
|
|
||||||
rp.received_by,
|
|
||||||
rp.received_date,
|
|
||||||
r.date_created as receiving_created_date,
|
|
||||||
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
|
||||||
COALESCE(p.description, 'Unknown Product') AS name
|
|
||||||
FROM receivings_products rp
|
|
||||||
JOIN receivings r ON rp.receiving_id = r.receiving_id
|
|
||||||
LEFT JOIN products p ON rp.pid = p.pid
|
|
||||||
WHERE rp.receiving_id IN (?)
|
|
||||||
`, [receivingIds]);
|
|
||||||
|
|
||||||
// Build complete receiving records
|
|
||||||
const completeReceivings = [];
|
|
||||||
for (const product of receivingProducts) {
|
|
||||||
const receiving = receivingList.find(r => r.receiving_id == product.receiving_id);
|
|
||||||
if (!receiving) continue;
|
|
||||||
|
|
||||||
// Get employee name if available
|
|
||||||
let receivedByName = null;
|
|
||||||
if (product.received_by) {
|
|
||||||
const [employeeResult] = await localConnection.query(`
|
|
||||||
SELECT CONCAT(firstname, ' ', lastname) as full_name
|
|
||||||
FROM employee_names
|
|
||||||
WHERE employeeid = $1
|
|
||||||
`, [product.received_by]);
|
|
||||||
|
|
||||||
if (employeeResult.rows.length > 0) {
|
|
||||||
receivedByName = employeeResult.rows[0].full_name;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get vendor name if available
|
offset += receivingList.length;
|
||||||
let vendorName = 'Unknown Vendor';
|
totalProcessed += completeReceivings.length;
|
||||||
if (receiving.supplier_id) {
|
|
||||||
const [vendorResult] = await localConnection.query(`
|
|
||||||
SELECT company_name
|
|
||||||
FROM temp_supplier_names
|
|
||||||
WHERE supplier_id = $1
|
|
||||||
`, [receiving.supplier_id]);
|
|
||||||
|
|
||||||
if (vendorResult.rows.length > 0) {
|
|
||||||
vendorName = vendorResult.rows[0].company_name;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
completeReceivings.push({
|
outputProgress({
|
||||||
receiving_id: receiving.receiving_id.toString(),
|
status: "running",
|
||||||
pid: product.pid,
|
operation: "Purchase orders import",
|
||||||
sku: product.sku,
|
message: `Processed ${offset} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
|
||||||
name: product.name,
|
current: offset,
|
||||||
vendor: vendorName,
|
total: totalReceivings,
|
||||||
qty_each: product.qty_each,
|
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||||
qty_each_orig: product.qty_each_orig,
|
remaining: estimateRemaining(startTime, offset, totalReceivings),
|
||||||
cost_each: product.cost_each,
|
rate: calculateRate(startTime, offset)
|
||||||
cost_each_orig: product.cost_each_orig,
|
|
||||||
received_by: product.received_by,
|
|
||||||
received_by_name: receivedByName,
|
|
||||||
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
|
|
||||||
receiving_created_date: validateDate(product.receiving_created_date),
|
|
||||||
supplier_id: receiving.supplier_id,
|
|
||||||
status: receivingStatusMap[receiving.status] || 'created'
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
// Insert receiving data in batches
|
|
||||||
for (let i = 0; i < completeReceivings.length; i += INSERT_BATCH_SIZE) {
|
|
||||||
const batch = completeReceivings.slice(i, i + INSERT_BATCH_SIZE);
|
|
||||||
|
|
||||||
const placeholders = batch.map((_, idx) => {
|
if (receivingList.length < PO_BATCH_SIZE) {
|
||||||
const base = idx * 15;
|
allReceivingsProcessed = true;
|
||||||
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
}
|
||||||
}).join(',');
|
|
||||||
|
|
||||||
const values = batch.flatMap(r => [
|
|
||||||
r.receiving_id,
|
|
||||||
r.pid,
|
|
||||||
r.sku,
|
|
||||||
r.name,
|
|
||||||
r.vendor,
|
|
||||||
r.qty_each,
|
|
||||||
r.qty_each_orig,
|
|
||||||
r.cost_each,
|
|
||||||
r.cost_each_orig,
|
|
||||||
r.received_by,
|
|
||||||
r.received_by_name,
|
|
||||||
r.received_date,
|
|
||||||
r.receiving_created_date,
|
|
||||||
r.supplier_id,
|
|
||||||
r.status
|
|
||||||
]);
|
|
||||||
|
|
||||||
await localConnection.query(`
|
|
||||||
INSERT INTO temp_receivings (
|
|
||||||
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
|
||||||
cost_each, cost_each_orig, received_by, received_by_name,
|
|
||||||
received_date, receiving_created_date, supplier_id, status
|
|
||||||
)
|
|
||||||
VALUES ${placeholders}
|
|
||||||
ON CONFLICT (receiving_id, pid) DO UPDATE SET
|
|
||||||
sku = EXCLUDED.sku,
|
|
||||||
name = EXCLUDED.name,
|
|
||||||
vendor = EXCLUDED.vendor,
|
|
||||||
qty_each = EXCLUDED.qty_each,
|
|
||||||
qty_each_orig = EXCLUDED.qty_each_orig,
|
|
||||||
cost_each = EXCLUDED.cost_each,
|
|
||||||
cost_each_orig = EXCLUDED.cost_each_orig,
|
|
||||||
received_by = EXCLUDED.received_by,
|
|
||||||
received_by_name = EXCLUDED.received_by_name,
|
|
||||||
received_date = EXCLUDED.received_date,
|
|
||||||
receiving_created_date = EXCLUDED.receiving_created_date,
|
|
||||||
supplier_id = EXCLUDED.supplier_id,
|
|
||||||
status = EXCLUDED.status
|
|
||||||
`, values);
|
|
||||||
}
|
|
||||||
|
|
||||||
offset += receivingList.length;
|
|
||||||
totalProcessed += completeReceivings.length;
|
|
||||||
|
|
||||||
outputProgress({
|
|
||||||
status: "running",
|
|
||||||
operation: "Purchase orders import",
|
|
||||||
message: `Processed ${offset} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
|
|
||||||
current: offset,
|
|
||||||
total: totalReceivings,
|
|
||||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
|
||||||
remaining: estimateRemaining(startTime, offset, totalReceivings),
|
|
||||||
rate: calculateRate(startTime, offset)
|
|
||||||
});
|
|
||||||
|
|
||||||
if (receivingList.length < PO_BATCH_SIZE) {
|
|
||||||
allReceivingsProcessed = true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -655,6 +667,31 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
message: "Inserting final purchase order records"
|
message: "Inserting final purchase order records"
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Create a temp table to track PO IDs being processed
|
||||||
|
await localConnection.query(`
|
||||||
|
DROP TABLE IF EXISTS processed_po_ids;
|
||||||
|
CREATE TEMP TABLE processed_po_ids AS (
|
||||||
|
SELECT DISTINCT po_id FROM temp_purchase_orders
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
// Delete products that were removed from POs and count them
|
||||||
|
const [poDeletedResult] = await localConnection.query(`
|
||||||
|
WITH deleted AS (
|
||||||
|
DELETE FROM purchase_orders
|
||||||
|
WHERE po_id IN (SELECT po_id FROM processed_po_ids)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM temp_purchase_orders tp
|
||||||
|
WHERE purchase_orders.po_id = tp.po_id AND purchase_orders.pid = tp.pid
|
||||||
|
)
|
||||||
|
RETURNING po_id, pid
|
||||||
|
)
|
||||||
|
SELECT COUNT(*) as count FROM deleted
|
||||||
|
`);
|
||||||
|
|
||||||
|
poRecordsDeleted = poDeletedResult.rows[0]?.count || 0;
|
||||||
|
console.log(`Deleted ${poRecordsDeleted} products that were removed from purchase orders`);
|
||||||
|
|
||||||
const [poResult] = await localConnection.query(`
|
const [poResult] = await localConnection.query(`
|
||||||
INSERT INTO purchase_orders (
|
INSERT INTO purchase_orders (
|
||||||
po_id, vendor, date, expected_date, pid, sku, name,
|
po_id, vendor, date, expected_date, pid, sku, name,
|
||||||
@@ -706,6 +743,31 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
message: "Inserting final receiving records"
|
message: "Inserting final receiving records"
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Create a temp table to track receiving IDs being processed
|
||||||
|
await localConnection.query(`
|
||||||
|
DROP TABLE IF EXISTS processed_receiving_ids;
|
||||||
|
CREATE TEMP TABLE processed_receiving_ids AS (
|
||||||
|
SELECT DISTINCT receiving_id FROM temp_receivings
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
// Delete products that were removed from receivings and count them
|
||||||
|
const [receivingDeletedResult] = await localConnection.query(`
|
||||||
|
WITH deleted AS (
|
||||||
|
DELETE FROM receivings
|
||||||
|
WHERE receiving_id IN (SELECT receiving_id FROM processed_receiving_ids)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM temp_receivings tr
|
||||||
|
WHERE receivings.receiving_id = tr.receiving_id AND receivings.pid = tr.pid
|
||||||
|
)
|
||||||
|
RETURNING receiving_id, pid
|
||||||
|
)
|
||||||
|
SELECT COUNT(*) as count FROM deleted
|
||||||
|
`);
|
||||||
|
|
||||||
|
receivingRecordsDeleted = receivingDeletedResult.rows[0]?.count || 0;
|
||||||
|
console.log(`Deleted ${receivingRecordsDeleted} products that were removed from receivings`);
|
||||||
|
|
||||||
const [receivingsResult] = await localConnection.query(`
|
const [receivingsResult] = await localConnection.query(`
|
||||||
INSERT INTO receivings (
|
INSERT INTO receivings (
|
||||||
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
||||||
@@ -765,6 +827,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
DROP TABLE IF EXISTS employee_names;
|
DROP TABLE IF EXISTS employee_names;
|
||||||
DROP TABLE IF EXISTS temp_supplier_names;
|
DROP TABLE IF EXISTS temp_supplier_names;
|
||||||
DROP TABLE IF EXISTS temp_invalid_pids;
|
DROP TABLE IF EXISTS temp_invalid_pids;
|
||||||
|
DROP TABLE IF EXISTS processed_po_ids;
|
||||||
|
DROP TABLE IF EXISTS processed_receiving_ids;
|
||||||
`);
|
`);
|
||||||
|
|
||||||
// Commit transaction
|
// Commit transaction
|
||||||
@@ -774,10 +838,13 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
status: "complete",
|
status: "complete",
|
||||||
recordsAdded: poRecordsAdded + receivingRecordsAdded,
|
recordsAdded: poRecordsAdded + receivingRecordsAdded,
|
||||||
recordsUpdated: poRecordsUpdated + receivingRecordsUpdated,
|
recordsUpdated: poRecordsUpdated + receivingRecordsUpdated,
|
||||||
|
recordsDeleted: poRecordsDeleted + receivingRecordsDeleted,
|
||||||
poRecordsAdded,
|
poRecordsAdded,
|
||||||
poRecordsUpdated,
|
poRecordsUpdated,
|
||||||
|
poRecordsDeleted,
|
||||||
receivingRecordsAdded,
|
receivingRecordsAdded,
|
||||||
receivingRecordsUpdated,
|
receivingRecordsUpdated,
|
||||||
|
receivingRecordsDeleted,
|
||||||
totalRecords: totalProcessed
|
totalRecords: totalProcessed
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -795,6 +862,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
|
|||||||
error: error.message,
|
error: error.message,
|
||||||
recordsAdded: 0,
|
recordsAdded: 0,
|
||||||
recordsUpdated: 0,
|
recordsUpdated: 0,
|
||||||
|
recordsDeleted: 0,
|
||||||
totalRecords: 0
|
totalRecords: 0
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user