matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import fixes/optimizations

Browse Source
This commit is contained in:
Matt
2025-01-29 21:48:56 -05:00
parent 3c5fb9e435
commit 6c5f119ee5
3 changed files with 227 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
inventory-server/scripts/import/orders.js
View File

@@ -1,158 +1,188 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress'); const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
const { importMissingProducts } = require('./products'); const { importMissingProducts } = require('./products');
async function importOrders(prodConnection, localConnection) { /**
outputProgress({ * Imports orders from a production MySQL database to a local MySQL database.
operation: "Starting orders import - Getting total count", * It can run in two modes:
status: "running", * 1. Incremental update mode (default): Only fetch orders that have changed since the last sync time.
}); * 2. Full update mode: Fetch all eligible orders within the last 5 years regardless of timestamp.
*
* @param {object} prodConnection - A MySQL connection to production DB (MySQL 5.7).
* @param {object} localConnection - A MySQL connection to local DB (MySQL 8.0).
* @param {boolean} incrementalUpdate - Set to false for a full sync; true for incremental.
*
* @returns {object} Information about the sync operation.
*/
async function importOrders(prodConnection, localConnection, incrementalUpdate = true) {
const startTime = Date.now(); const startTime = Date.now();
const skippedOrders = new Set(); // Store orders that need to be retried const skippedOrders = new Set();
const missingProducts = new Set(); // Store products that need to be imported const missingProducts = new Set();
try { try {
// Get last sync info // Get the last sync time
const [syncInfo] = await localConnection.query( const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'" "SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
); );
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01'; const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
// First get the column names from the table structure // Retrieve column names for the 'orders' table, skip 'id' since it's auto-increment
const [columns] = await localConnection.query(` const [columns] = await localConnection.query(`
SELECT COLUMN_NAME SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'orders' WHERE TABLE_NAME = 'orders'
ORDER BY ORDINAL_POSITION ORDER BY ORDINAL_POSITION
`); `);
const columnNames = columns const columnNames = columns
.map((col) => col.COLUMN_NAME) .map(col => col.COLUMN_NAME)
.filter((name) => name !== "id"); // Skip auto-increment ID .filter(name => name !== "id");
// Get total count first for progress indication - modified for incremental // Build query clauses for incremental vs. full update
const [countResult] = await prodConnection.query(` const incrementalWhereClause = incrementalUpdate
SELECT COUNT(*) as total ? `AND (
FROM order_items oi FORCE INDEX (PRIMARY) o.stamp > ?
JOIN _order o FORCE INDEX (PRIMARY) ON oi.order_id = o.order_id OR o.date_modified > ?
WHERE o.order_status >= 15 OR o.date_placed > ?
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR) OR o.date_shipped > ?
AND (o.date_placed_onlydate > ? OR oi.stamp > ?
OR o.stamp > ?) )`
`, [lastSyncTime, lastSyncTime]); : "";
const incrementalParams = incrementalUpdate
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
: [];
const totalOrders = countResult[0].total; // Count how many orders we need to process
const [countResult] = await prodConnection.query(
outputProgress({ `
operation: `Starting orders import - Fetching ${totalOrders} orders from production`, SELECT COUNT(*) AS total
status: "running", FROM order_items oi USE INDEX (PRIMARY)
}); JOIN _order o USE INDEX (PRIMARY)
const total = countResult[0].total;
let processed = 0;
// Process in batches
const batchSize = 10000; // Increased from 1000 since order records are small
let offset = 0;
while (offset < total) {
// First get orders without tax info
const [orders] = await prodConnection.query(`
SELECT
oi.order_id as order_number,
oi.prod_pid as pid,
oi.prod_itemnumber as SKU,
o.date_placed_onlydate as date,
oi.prod_price_reg as price,
oi.qty_ordered as quantity,
(oi.prod_price_reg - oi.prod_price) as discount,
0 as tax,
0 as tax_included,
ROUND(
((o.summary_shipping - COALESCE(o.summary_discount_shipping, 0)) *
(oi.prod_price * oi.qty_ordered) / NULLIF(o.summary_subtotal, 0)), 2
) as shipping,
o.order_cid as customer,
CONCAT(o.bill_firstname, ' ', o.bill_lastname) as customer_name,
'pending' as status,
CASE WHEN o.order_status = 15 THEN 1 ELSE 0 END as canceled
FROM order_items oi
FORCE INDEX (PRIMARY)
JOIN _order o USE INDEX (date_placed_onlydate, idx_status)
ON oi.order_id = o.order_id ON oi.order_id = o.order_id
WHERE o.order_status >= 15 WHERE o.order_status >= 15
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR) AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
AND (o.date_placed_onlydate > ? ${incrementalWhereClause}
OR o.stamp > ?) `,
LIMIT ? OFFSET ? incrementalParams
`, [lastSyncTime, lastSyncTime, batchSize, offset]); );
// Then get tax info for these orders const total = countResult[0].total;
outputProgress({
operation: `Starting ${incrementalUpdate ? 'incremental' : 'full'} orders import - Fetching ${total} orders`,
status: "running",
});
let processed = 0;
// Increase or decrease this if you find a more optimal size
const batchSize = 20000;
let offset = 0;
// Process in batches for memory efficiency
while (offset < total) {
// Fetch orders (initially with tax set to 0, to be updated later)
const [orders] = await prodConnection.query(
`
SELECT
oi.order_id AS order_number,
oi.prod_pid AS pid,
oi.prod_itemnumber AS SKU,
o.date_placed_onlydate AS date,
oi.prod_price_reg AS price,
oi.qty_ordered AS quantity,
(oi.prod_price_reg - oi.prod_price) AS discount,
0 AS tax,
0 AS tax_included,
ROUND(
(
(o.summary_shipping - COALESCE(o.summary_discount_shipping, 0))
* (oi.prod_price * oi.qty_ordered)
) / NULLIF(o.summary_subtotal, 0),
2
) AS shipping,
o.order_cid AS customer,
CONCAT(o.bill_firstname, ' ', o.bill_lastname) AS customer_name,
'pending' AS status,
CASE WHEN o.order_status = 15 THEN 1 ELSE 0 END AS canceled
FROM order_items oi
FORCE INDEX (PRIMARY)
JOIN _order o
ON oi.order_id = o.order_id
WHERE o.order_status >= 15
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
${incrementalWhereClause}
LIMIT ? OFFSET ?
`,
[...incrementalParams, batchSize, offset]
);
// Fetch the latest tax info for these orders
if (orders.length > 0) { if (orders.length > 0) {
const orderIds = [...new Set(orders.map(o => o.order_number))]; const orderIds = [...new Set(orders.map(o => o.order_number))];
const [taxInfo] = await prodConnection.query(` const [taxInfo] = await prodConnection.query(`
SELECT oti.order_id, otp.pid, otp.item_taxes_to_collect SELECT oti.order_id, otp.pid, otp.item_taxes_to_collect
FROM ( FROM (
SELECT order_id, MAX(stamp) as latest_stamp SELECT order_id, MAX(stamp) AS latest_stamp
FROM order_tax_info USE INDEX (order_id, stamp) FROM order_tax_info
WHERE order_id IN (?) WHERE order_id IN (?)
GROUP BY order_id GROUP BY order_id
) latest ) latest
JOIN order_tax_info oti USE INDEX (order_id, stamp) JOIN order_tax_info oti
ON oti.order_id = latest.order_id ON oti.order_id = latest.order_id
AND oti.stamp = latest.latest_stamp AND oti.stamp = latest.latest_stamp
JOIN order_tax_info_products otp FORCE INDEX (PRIMARY) JOIN order_tax_info_products otp
ON oti.taxinfo_id = otp.taxinfo_id ON oti.taxinfo_id = otp.taxinfo_id
`, [orderIds]); `, [orderIds]);
// Create a map for quick tax lookup // Map (order_id-pid) -> tax amount
const taxMap = new Map(); const taxMap = new Map();
taxInfo.forEach(t => { taxInfo.forEach(t => {
taxMap.set(`${t.order_id}-${t.pid}`, t.item_taxes_to_collect); taxMap.set(`${t.order_id}-${t.pid}`, t.item_taxes_to_collect);
}); });
// Add tax info to orders // Merge tax into the orders array
orders.forEach(order => { orders.forEach(order => {
const taxKey = `${order.order_number}-${order.pid}`; const key = `${order.order_number}-${order.pid}`;
order.tax = taxMap.get(taxKey) || 0; if (taxMap.has(key)) {
order.tax = taxMap.get(key) || 0;
}
}); });
} }
// Check if all products exist before inserting orders // Check local DB for existing products to ensure we don't insert orders for missing products
const orderProductPids = [...new Set(orders.map((o) => o.pid))]; const orderProductPids = [...new Set(orders.map(o => o.pid))];
const [existingProducts] = await localConnection.query( const [existingProducts] = await localConnection.query(
"SELECT pid FROM products WHERE pid IN (?)", "SELECT pid FROM products WHERE pid IN (?)",
[orderProductPids] [orderProductPids]
); );
const existingPids = new Set(existingProducts.map((p) => p.pid)); const existingPids = new Set(existingProducts.map(p => p.pid));
// Filter out orders with missing products and track them // Separate valid orders from those referencing missing products
const validOrders = orders.filter((order) => { const validOrders = [];
for (const order of orders) {
if (!existingPids.has(order.pid)) { if (!existingPids.has(order.pid)) {
missingProducts.add(order.pid); missingProducts.add(order.pid);
skippedOrders.add(order.order_number); skippedOrders.add(order.order_number);
return false; } else {
validOrders.push(order);
} }
return true; }
});
// Bulk insert valid orders
if (validOrders.length > 0) { if (validOrders.length > 0) {
const placeholders = validOrders const placeholders = validOrders
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`) .map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
.join(","); .join(",");
const updateClauses = columnNames const updateClauses = columnNames
.filter((col) => col !== "order_number") // Don't update primary key .filter(col => col !== "order_number") // don't overwrite primary key
.map((col) => `${col} = VALUES(${col})`) .map(col => `${col} = VALUES(${col})`)
.join(","); .join(",");
const query = ` const upsertQuery = `
INSERT INTO orders (${columnNames.join(",")}) INSERT INTO orders (${columnNames.join(",")})
VALUES ${placeholders} VALUES ${placeholders}
ON DUPLICATE KEY UPDATE ${updateClauses} ON DUPLICATE KEY UPDATE ${updateClauses}
`; `;
await localConnection.query( await localConnection.query(
query, upsertQuery,
validOrders.flatMap(order => columnNames.map(col => order[col])) validOrders.flatMap(order => columnNames.map(col => order[col]))
); );
} }
@@ -171,18 +201,17 @@ async function importOrders(prodConnection, localConnection) {
}); });
} }
// Now handle missing products and retry skipped orders // If we found missing products, import them and retry the skipped orders
if (missingProducts.size > 0) { if (missingProducts.size > 0) {
outputProgress({ outputProgress({
operation: `Found ${missingProducts.size} missing products, importing them now`, operation: `Found ${missingProducts.size} missing products, importing them now`,
status: "running", status: "running",
}); });
await importMissingProducts(prodConnection, localConnection, [ // Import missing products
...missingProducts, await importMissingProducts(prodConnection, localConnection, [...missingProducts]);
]);
// Retry skipped orders // Retry orders that were skipped due to missing products
if (skippedOrders.size > 0) { if (skippedOrders.size > 0) {
outputProgress({ outputProgress({
operation: `Retrying ${skippedOrders.size} skipped orders`, operation: `Retrying ${skippedOrders.size} skipped orders`,
@@ -191,95 +220,100 @@ async function importOrders(prodConnection, localConnection) {
const [retryOrders] = await prodConnection.query(` const [retryOrders] = await prodConnection.query(`
SELECT SELECT
oi.order_id as order_number, oi.order_id AS order_number,
oi.prod_pid as pid, oi.prod_pid AS pid,
oi.prod_itemnumber as SKU, oi.prod_itemnumber AS SKU,
o.date_placed_onlydate as date, o.date_placed_onlydate AS date,
oi.prod_price_reg as price, oi.prod_price_reg AS price,
oi.qty_ordered as quantity, oi.qty_ordered AS quantity,
(oi.prod_price_reg - oi.prod_price) as discount, (oi.prod_price_reg - oi.prod_price) AS discount,
0 as tax, 0 AS tax,
0 as tax_included, 0 AS tax_included,
ROUND( ROUND(
((o.summary_shipping - COALESCE(o.summary_discount_shipping, 0)) * (
(oi.prod_price * oi.qty_ordered) / NULLIF(o.summary_subtotal, 0)), 2 (o.summary_shipping - COALESCE(o.summary_discount_shipping, 0))
) as shipping, * (oi.prod_price * oi.qty_ordered)
o.order_cid as customer, ) / NULLIF(o.summary_subtotal, 0),
CONCAT(o.bill_firstname, ' ', o.bill_lastname) as customer_name, 2
'pending' as status, ) AS shipping,
CASE WHEN o.order_status = 15 THEN 1 ELSE 0 END as canceled o.order_cid AS customer,
CONCAT(o.bill_firstname, ' ', o.bill_lastname) AS customer_name,
'pending' AS status,
CASE WHEN o.order_status = 15 THEN 1 ELSE 0 END AS canceled
FROM order_items oi FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id JOIN _order o ON oi.order_id = o.order_id
WHERE oi.order_id IN (?) WHERE oi.order_id IN (?)
`, [[...skippedOrders]]); `, [[...skippedOrders]]);
if (retryOrders.length > 0) { if (retryOrders.length > 0) {
// Fetch tax data for these specific retry orders
const retryOrderIds = [...new Set(retryOrders.map(o => o.order_number))]; const retryOrderIds = [...new Set(retryOrders.map(o => o.order_number))];
const [retryTaxInfo] = await prodConnection.query(` const [retryTaxInfo] = await prodConnection.query(`
SELECT oti.order_id, otp.pid, otp.item_taxes_to_collect SELECT oti.order_id, otp.pid, otp.item_taxes_to_collect
FROM ( FROM (
SELECT order_id, MAX(stamp) as latest_stamp SELECT order_id, MAX(stamp) AS latest_stamp
FROM order_tax_info USE INDEX (order_id, stamp) FROM order_tax_info
WHERE order_id IN (?) WHERE order_id IN (?)
GROUP BY order_id GROUP BY order_id
) latest ) latest
JOIN order_tax_info oti USE INDEX (order_id, stamp) JOIN order_tax_info oti
ON oti.order_id = latest.order_id ON oti.order_id = latest.order_id
AND oti.stamp = latest.latest_stamp AND oti.stamp = latest.latest_stamp
JOIN order_tax_info_products otp FORCE INDEX (PRIMARY) JOIN order_tax_info_products otp
ON oti.taxinfo_id = otp.taxinfo_id ON oti.taxinfo_id = otp.taxinfo_id
`, [retryOrderIds]); `, [retryOrderIds]);
// Create a map for quick tax lookup
const taxMap = new Map(); const taxMap = new Map();
retryTaxInfo.forEach(t => { retryTaxInfo.forEach(t => {
taxMap.set(`${t.order_id}-${t.pid}`, t.item_taxes_to_collect); taxMap.set(`${t.order_id}-${t.pid}`, t.item_taxes_to_collect);
}); });
// Add tax info to orders
retryOrders.forEach(order => { retryOrders.forEach(order => {
const taxKey = `${order.order_number}-${order.pid}`; const key = `${order.order_number}-${order.pid}`;
order.tax = taxMap.get(taxKey) || 0; if (taxMap.has(key)) {
order.tax = taxMap.get(key) || 0;
}
}); });
const placeholders = retryOrders
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
.join(",");
const updateClauses = columnNames
.filter(col => col !== "order_number")
.map(col => `${col} = VALUES(${col})`)
.join(",");
const upsertQuery = `
INSERT INTO orders (${columnNames.join(",")})
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE ${updateClauses}
`;
await localConnection.query(
upsertQuery,
retryOrders.flatMap(order => columnNames.map(col => order[col]))
);
} }
const placeholders = retryOrders
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
.join(",");
const updateClauses = columnNames
.filter((col) => col !== "order_number") // Don't update primary key
.map((col) => `${col} = VALUES(${col})`)
.join(",");
const query = `
INSERT INTO orders (${columnNames.join(",")})
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE ${updateClauses}
`;
await localConnection.query(
query,
retryOrders.flatMap(order => columnNames.map(col => order[col]))
);
} }
} }
// After successful import, update the sync status // Update the sync timestamp
await localConnection.query(` await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp) INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('orders', NOW()) VALUES ('orders', NOW())
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW() ON DUPLICATE KEY UPDATE
last_sync_timestamp = NOW(),
last_sync_id = LAST_INSERT_ID(last_sync_id)
`); `);
const endTime = Date.now(); const endTime = Date.now();
const durationSeconds = Math.round((endTime - startTime) / 1000);
outputProgress({ outputProgress({
status: "complete", status: "complete",
operation: "Orders import completed", operation: `${incrementalUpdate ? 'Incremental' : 'Full'} orders import completed`,
current: total, current: total,
total, total,
duration: formatElapsedTime((Date.now() - startTime) / 1000), duration: formatElapsedTime((endTime - startTime) / 1000),
}); });
return { return {
@@ -287,12 +321,12 @@ async function importOrders(prodConnection, localConnection) {
totalImported: total, totalImported: total,
missingProducts: missingProducts.size, missingProducts: missingProducts.size,
retriedOrders: skippedOrders.size, retriedOrders: skippedOrders.size,
incrementalUpdate: true, incrementalUpdate,
lastSyncTime lastSyncTime
}; };
} catch (error) { } catch (error) {
outputProgress({ outputProgress({
operation: "Orders import failed", operation: `${incrementalUpdate ? 'Incremental' : 'Full'} orders import failed`,
status: "error", status: "error",
error: error.message, error: error.message,
}); });

24
inventory-server/scripts/import/products.js
View File

@@ -456,16 +456,32 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
// Materialize calculations for missing products // Materialize calculations for missing products
await localConnection.query(` await localConnection.query(`
INSERT INTO temp_inventory_status INSERT INTO temp_inventory_status
WITH product_stock AS ( SELECT
p.pid,
COALESCE(si.available_local, 0) - COALESCE(ps.pending_qty, 0) as stock_quantity,
COALESCE(ps.pending_qty, 0) as pending_qty,
COALESCE(ci.onpreorder, 0) as preorder_count,
COALESCE(pnb.inventory, 0) as notions_inv_count
FROM products p
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
LEFT JOIN current_inventory ci ON p.pid = ci.pid
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
LEFT JOIN (
SELECT oi.prod_pid, SELECT oi.prod_pid,
SUM(oi.qty_ordered - oi.qty_placed) as pending_qty SUM(oi.qty_ordered - oi.qty_placed) as pending_qty
FROM order_items oi FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid IN (?) WHERE oi.prod_pid IN (?)
AND [rest of conditions] AND o.date_placed != '0000-00-00 00:00:00'
AND o.date_shipped = '0000-00-00 00:00:00'
AND oi.pick_finished = 0
AND oi.qty_back = 0
AND o.order_status != 15
AND o.order_status < 90
AND oi.qty_ordered >= oi.qty_placed
AND oi.qty_ordered > 0
GROUP BY oi.prod_pid GROUP BY oi.prod_pid
) ) ps ON p.pid = ps.prod_pid
SELECT [same as above]
WHERE p.pid IN (?) WHERE p.pid IN (?)
`, [missingPids, missingPids]); `, [missingPids, missingPids]);

42
inventory-server/scripts/import/purchase-orders.js
View File

@@ -1,6 +1,6 @@
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress'); const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
async function importPurchaseOrders(prodConnection, localConnection) { async function importPurchaseOrders(prodConnection, localConnection, incrementalUpdate = true) {
const startTime = Date.now(); const startTime = Date.now();
try { try {
@@ -11,7 +11,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01'; const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
outputProgress({ outputProgress({
operation: "Starting purchase orders import - Initializing", operation: `Starting ${incrementalUpdate ? 'incremental' : 'full'} purchase orders import`,
status: "running", status: "running",
}); });
@@ -26,7 +26,23 @@ async function importPurchaseOrders(prodConnection, localConnection) {
.map((col) => col.COLUMN_NAME) .map((col) => col.COLUMN_NAME)
.filter((name) => name !== "id"); .filter((name) => name !== "id");
// First get all relevant PO IDs with basic info - modified for incremental // Build incremental conditions
const incrementalWhereClause = incrementalUpdate
? `AND (
p.stamp > ?
OR p.date_modified > ?
OR p.date_ordered > ?
OR p.date_estin > ?
OR r.stamp > ?
OR rp.stamp > ?
OR rp.received_date > ?
)`
: "";
const incrementalParams = incrementalUpdate
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
: [];
// First get all relevant PO IDs with basic info
const [[{ total }]] = await prodConnection.query(` const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total SELECT COUNT(*) as total
FROM ( FROM (
@@ -36,19 +52,16 @@ async function importPurchaseOrders(prodConnection, localConnection) {
JOIN po_products pop ON p.po_id = pop.po_id JOIN po_products pop ON p.po_id = pop.po_id
JOIN suppliers s ON p.supplier_id = s.supplierid JOIN suppliers s ON p.supplier_id = s.supplierid
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR) WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
AND (p.date_ordered > ? ${incrementalWhereClause}
OR p.stamp > ?
OR p.date_modified > ?)
UNION UNION
SELECT DISTINCT r.receiving_id as po_id, rp.pid SELECT DISTINCT r.receiving_id as po_id, rp.pid
FROM receivings_products rp FROM receivings_products rp
USE INDEX (received_date) USE INDEX (received_date)
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR) WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
AND (rp.received_date > ? ${incrementalWhereClause}
OR rp.stamp > ?)
) all_items ) all_items
`, [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]); `, [...incrementalParams, ...incrementalParams]);
const [poList] = await prodConnection.query(` const [poList] = await prodConnection.query(`
SELECT DISTINCT SELECT DISTINCT
@@ -294,11 +307,13 @@ async function importPurchaseOrders(prodConnection, localConnection) {
} }
} }
// After successful import, update sync status // Update sync status with proper incrementing of last_sync_id
await localConnection.query(` await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp) INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW()) VALUES ('purchase_orders', NOW())
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW() ON DUPLICATE KEY UPDATE
last_sync_timestamp = NOW(),
last_sync_id = LAST_INSERT_ID(last_sync_id)
`); `);
return { return {
@@ -306,12 +321,13 @@ async function importPurchaseOrders(prodConnection, localConnection) {
totalImported: totalItems, totalImported: totalItems,
recordsAdded, recordsAdded,
recordsUpdated, recordsUpdated,
incrementalUpdate: !!syncInfo?.[0] incrementalUpdate,
lastSyncTime
}; };
} catch (error) { } catch (error) {
outputProgress({ outputProgress({
operation: "Purchase orders import failed", operation: `${incrementalUpdate ? 'Incremental' : 'Full'} purchase orders import failed`,
status: "error", status: "error",
error: error.message, error: error.message,
}); });