matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get orders import started, get all needed products imported

Browse Source
This commit is contained in:
Matt
2025-01-25 21:43:31 -05:00
parent b1a5671531
commit cdce12e9fb
2 changed files with 146 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
inventory-server/db/schema.sql
View File

@@ -122,7 +122,6 @@ CREATE TABLE orders (
billing_address TEXT,
canceled BOOLEAN DEFAULT false,
FOREIGN KEY (pid) REFERENCES products(pid),
FOREIGN KEY (SKU) REFERENCES products(SKU),
INDEX idx_order_number (order_number),
INDEX idx_customer (customer),
INDEX idx_date (date),

180
inventory-server/scripts/import-from-prod.js
View File

@@ -238,7 +238,7 @@ async function importCategories(prodConnection, localConnection) {
async function importProducts(prodConnection, localConnection) {
outputProgress({
operation: 'Starting products import',
operation: 'Starting products import - Getting schema',
status: 'running'
});
@@ -255,6 +255,27 @@ async function importProducts(prodConnection, localConnection) {
const columnNames = columns.map(col => col.COLUMN_NAME);
// Get total count first for progress indication
outputProgress({
operation: 'Starting products import - Getting total count',
status: 'running'
});
const [countResult] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM products p
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
WHERE pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
OR p.itemnumber LIKE 'chbx%'
`);
const totalProducts = countResult[0].total;
outputProgress({
operation: `Starting products import - Fetching ${totalProducts} products from production`,
status: 'running'
});
// Get products from production with optimized query
const [rows] = await prodConnection.query(`
SELECT
@@ -353,14 +374,16 @@ async function importProducts(prodConnection, localConnection) {
WHERE active = 1
GROUP BY pid
) pcp ON p.pid = pcp.pid
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
WHERE (pls.date_sold >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
OR p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
OR p.itemnumber LIKE 'chbx%')
GROUP BY p.pid
`);
let current = 0;
const total = rows.length;
// Process products in larger batches
// Process products in batches
const BATCH_SIZE = 100;
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
@@ -478,20 +501,61 @@ async function importProducts(prodConnection, localConnection) {
}
}
// Helper function to get date ranges for chunked queries
async function getDateRanges(prodConnection, table, dateField, startYearsAgo = 2, chunkMonths = 3) {
const ranges = [];
const [result] = await prodConnection.query(`
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL ? YEAR) as start_date,
CURRENT_DATE as end_date
`, [startYearsAgo]);
let currentDate = new Date(result[0].end_date);
const startDate = new Date(result[0].start_date);
while (currentDate > startDate) {
const rangeEnd = new Date(currentDate);
currentDate.setMonth(currentDate.getMonth() - chunkMonths);
const rangeStart = new Date(Math.max(currentDate, startDate));
ranges.push({
start: rangeStart.toISOString().split('T')[0],
end: rangeEnd.toISOString().split('T')[0]
});
}
return ranges;
}
async function importOrders(prodConnection, localConnection) {
outputProgress({
operation: 'Starting orders import',
operation: 'Starting orders import - Getting total count',
status: 'running'
});
const startTime = Date.now();
try {
// Get total count first for progress indication
const [countResult] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM _order o
JOIN order_items oi ON o.order_id = oi.order_id
WHERE o.order_status >= 15
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
`);
const totalOrders = countResult[0].total;
outputProgress({
operation: `Starting orders import - Fetching ${totalOrders} orders from production`,
status: 'running'
});
// Get orders from production
const [rows] = await prodConnection.query(`
SELECT
oi.order_id AS order_number,
oi.prod_pid AS product_id,
oi.prod_pid AS pid,
oi.prod_itemnumber AS SKU,
o.date_placed_onlydate AS date,
oi.prod_price_reg AS price,
@@ -535,11 +599,31 @@ async function importOrders(prodConnection, localConnection) {
let current = 0;
const total = rows.length;
// Start transaction
await localConnection.query('START TRANSACTION');
// Process in batches
const BATCH_SIZE = 500;
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
// Get unique product IDs from this batch
const productIds = [...new Set(batch.map(row => row.pid))];
// Verify all products exist
const [existingProducts] = await localConnection.query(
'SELECT pid FROM products WHERE pid IN (?)',
[productIds]
);
const existingProductIds = new Set(existingProducts.map(p => p.pid));
const missingProducts = productIds.filter(pid => !existingProductIds.has(pid));
if (missingProducts.length > 0) {
console.error('Missing products:', missingProducts);
throw new Error(`Products missing from database: ${missingProducts.join(', ')}`);
}
// Create placeholders for batch insert
const placeholders = batch.map(() =>
'(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
@@ -548,7 +632,7 @@ async function importOrders(prodConnection, localConnection) {
// Flatten values for batch insert
const values = batch.flatMap(row => [
row.order_number,
row.product_id,
row.pid,
row.SKU,
row.date,
row.price,
@@ -562,27 +646,35 @@ async function importOrders(prodConnection, localConnection) {
row.canceled
]);
await localConnection.query(`
INSERT INTO orders (
order_number, product_id, SKU, date, price, quantity, discount,
tax, tax_included, shipping, customer, customer_name, canceled
)
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
price = VALUES(price),
quantity = VALUES(quantity),
discount = VALUES(discount),
tax = VALUES(tax),
tax_included = VALUES(tax_included),
shipping = VALUES(shipping),
customer_name = VALUES(customer_name),
canceled = VALUES(canceled)
`, values);
try {
await localConnection.query(`
INSERT INTO orders (
order_number, pid, SKU, date, price, quantity, discount,
tax, tax_included, shipping, customer, customer_name, canceled
)
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
price = VALUES(price),
quantity = VALUES(quantity),
discount = VALUES(discount),
tax = VALUES(tax),
tax_included = VALUES(tax_included),
shipping = VALUES(shipping),
customer_name = VALUES(customer_name),
canceled = VALUES(canceled)
`, values);
} catch (insertError) {
console.error('Error inserting batch:', insertError);
throw insertError;
}
current += batch.length;
updateProgress(current, total, 'Orders import', startTime);
}
// If we get here, commit the transaction
await localConnection.query('COMMIT');
outputProgress({
status: 'complete',
operation: 'Orders import completed',
@@ -591,6 +683,8 @@ async function importOrders(prodConnection, localConnection) {
duration: formatDuration((Date.now() - startTime) / 1000)
});
} catch (error) {
// Rollback on error
await localConnection.query('ROLLBACK');
console.error('Error importing orders:', error);
throw error;
}
@@ -598,13 +692,18 @@ async function importOrders(prodConnection, localConnection) {
async function importPurchaseOrders(prodConnection, localConnection) {
outputProgress({
operation: 'Starting purchase orders import',
operation: 'Starting purchase orders import - Initializing',
status: 'running'
});
const startTime = Date.now();
try {
outputProgress({
operation: 'Starting purchase orders import - Fetching POs from production',
status: 'running'
});
// Get purchase orders from production
const [rows] = await prodConnection.query(`
SELECT
@@ -710,6 +809,12 @@ async function importPurchaseOrders(prodConnection, localConnection) {
}
// Modify main function to handle cancellation and avoid process.exit
// Constants to control which imports run
const IMPORT_CATEGORIES = true;
const IMPORT_PRODUCTS = true;
const IMPORT_ORDERS = true;
const IMPORT_PURCHASE_ORDERS = true;
async function main() {
let ssh;
let prodConnection;
@@ -718,7 +823,7 @@ async function main() {
try {
outputProgress({
status: 'running',
operation: 'Starting import process',
operation: 'Starting import process',
message: 'Setting up connections...'
});
@@ -735,19 +840,26 @@ async function main() {
if (isImportCancelled) throw new Error('Import cancelled');
// First import all categories
await importCategories(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
// Run each import based on constants
if (IMPORT_CATEGORIES) {
await importCategories(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
}
// Then import products
await importProducts(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
if (IMPORT_PRODUCTS) {
await importProducts(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
}
await importOrders(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
if (IMPORT_ORDERS) {
await importOrders(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
}
await importPurchaseOrders(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
if (IMPORT_PURCHASE_ORDERS) {
await importPurchaseOrders(prodConnection, localConnection);
if (isImportCancelled) throw new Error('Import cancelled');
}
outputProgress({
status: 'complete',