Put back files
This commit is contained in:
210
inventory-server/scripts/import/categories.js
Normal file
210
inventory-server/scripts/import/categories.js
Normal file
@@ -0,0 +1,210 @@
|
||||
const { outputProgress, formatElapsedTime } = require('../metrics-new/utils/progress');
|
||||
|
||||
async function importCategories(prodConnection, localConnection) {
|
||||
outputProgress({
|
||||
operation: "Starting categories import",
|
||||
status: "running",
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
const typeOrder = [10, 20, 11, 21, 12, 13];
|
||||
let totalInserted = 0;
|
||||
let totalUpdated = 0;
|
||||
let skippedCategories = [];
|
||||
|
||||
try {
|
||||
// Start a single transaction for the entire import
|
||||
await localConnection.query('BEGIN');
|
||||
|
||||
// Temporarily disable the trigger that's causing problems
|
||||
await localConnection.query('ALTER TABLE categories DISABLE TRIGGER update_categories_updated_at');
|
||||
|
||||
// Process each type in order with its own savepoint
|
||||
for (const type of typeOrder) {
|
||||
try {
|
||||
// Create a savepoint for this type
|
||||
await localConnection.query(`SAVEPOINT category_type_${type}`);
|
||||
|
||||
// Production query remains MySQL compatible
|
||||
const [categories] = await prodConnection.query(
|
||||
`
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
pc.name,
|
||||
pc.type,
|
||||
CASE
|
||||
WHEN pc.type IN (10, 20) THEN NULL -- Top level categories should have no parent
|
||||
WHEN pc.master_cat_id IS NULL THEN NULL
|
||||
ELSE pc.master_cat_id
|
||||
END as parent_id,
|
||||
pc.combined_name as description
|
||||
FROM product_categories pc
|
||||
WHERE pc.type = ?
|
||||
ORDER BY pc.cat_id
|
||||
`,
|
||||
[type]
|
||||
);
|
||||
|
||||
if (categories.length === 0) {
|
||||
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`Processing ${categories.length} type ${type} categories`);
|
||||
|
||||
// For types that can have parents (11, 21, 12, 13), we'll proceed directly
|
||||
// No need to check for parent existence since we process in hierarchical order
|
||||
let categoriesToInsert = categories;
|
||||
|
||||
if (categoriesToInsert.length === 0) {
|
||||
console.log(`No valid categories of type ${type} to insert`);
|
||||
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// PostgreSQL upsert query with parameterized values
|
||||
const values = categoriesToInsert.flatMap((cat) => [
|
||||
cat.cat_id,
|
||||
cat.name,
|
||||
cat.type,
|
||||
cat.parent_id,
|
||||
cat.description,
|
||||
'active',
|
||||
new Date(),
|
||||
new Date()
|
||||
]);
|
||||
|
||||
const placeholders = categoriesToInsert
|
||||
.map((_, i) => `($${i * 8 + 1}, $${i * 8 + 2}, $${i * 8 + 3}, $${i * 8 + 4}, $${i * 8 + 5}, $${i * 8 + 6}, $${i * 8 + 7}, $${i * 8 + 8})`)
|
||||
.join(',');
|
||||
|
||||
// Insert categories with ON CONFLICT clause for PostgreSQL
|
||||
const query = `
|
||||
WITH inserted_categories AS (
|
||||
INSERT INTO categories (
|
||||
cat_id, name, type, parent_id, description, status, created_at, updated_at
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (cat_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
type = EXCLUDED.type,
|
||||
parent_id = EXCLUDED.parent_id,
|
||||
description = EXCLUDED.description,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
WHERE -- Only update if at least one field has changed
|
||||
categories.name IS DISTINCT FROM EXCLUDED.name OR
|
||||
categories.type IS DISTINCT FROM EXCLUDED.type OR
|
||||
categories.parent_id IS DISTINCT FROM EXCLUDED.parent_id OR
|
||||
categories.description IS DISTINCT FROM EXCLUDED.description OR
|
||||
categories.status IS DISTINCT FROM EXCLUDED.status
|
||||
RETURNING
|
||||
cat_id,
|
||||
CASE
|
||||
WHEN xmax = 0 THEN true
|
||||
ELSE false
|
||||
END as is_insert
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE is_insert) as inserted,
|
||||
COUNT(*) FILTER (WHERE NOT is_insert) as updated
|
||||
FROM inserted_categories`;
|
||||
|
||||
const result = await localConnection.query(query, values);
|
||||
|
||||
// Get the first result since query returns an array
|
||||
const queryResult = Array.isArray(result) ? result[0] : result;
|
||||
|
||||
if (!queryResult || !queryResult.rows || !queryResult.rows[0]) {
|
||||
console.error('Query failed to return results');
|
||||
throw new Error('Query did not return expected results');
|
||||
}
|
||||
|
||||
const total = parseInt(queryResult.rows[0].total) || 0;
|
||||
const inserted = parseInt(queryResult.rows[0].inserted) || 0;
|
||||
const updated = parseInt(queryResult.rows[0].updated) || 0;
|
||||
|
||||
console.log(`Total: ${total}, Inserted: ${inserted}, Updated: ${updated}`);
|
||||
|
||||
totalInserted += inserted;
|
||||
totalUpdated += updated;
|
||||
|
||||
// Release the savepoint for this type
|
||||
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Categories import",
|
||||
message: `Imported ${inserted} (updated ${updated}) categories of type ${type}`,
|
||||
current: totalInserted + totalUpdated,
|
||||
total: categories.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
});
|
||||
} catch (error) {
|
||||
// Rollback to the savepoint for this type
|
||||
await localConnection.query(`ROLLBACK TO SAVEPOINT category_type_${type}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Commit the entire transaction - we'll do this even if we have skipped categories
|
||||
await localConnection.query('COMMIT');
|
||||
|
||||
// Update sync status
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('categories', NOW())
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
// Re-enable the trigger
|
||||
await localConnection.query('ALTER TABLE categories ENABLE TRIGGER update_categories_updated_at');
|
||||
|
||||
outputProgress({
|
||||
status: "complete",
|
||||
operation: "Categories import completed",
|
||||
current: totalInserted + totalUpdated,
|
||||
total: totalInserted + totalUpdated,
|
||||
duration: formatElapsedTime(startTime),
|
||||
warnings: skippedCategories.length > 0 ? {
|
||||
message: "Some categories were skipped due to missing parents",
|
||||
skippedCategories
|
||||
} : undefined
|
||||
});
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
recordsAdded: totalInserted,
|
||||
recordsUpdated: totalUpdated,
|
||||
totalRecords: totalInserted + totalUpdated,
|
||||
warnings: skippedCategories.length > 0 ? {
|
||||
message: "Some categories were skipped due to missing parents",
|
||||
skippedCategories
|
||||
} : undefined
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error importing categories:", error);
|
||||
|
||||
// Only rollback if we haven't committed yet
|
||||
try {
|
||||
await localConnection.query('ROLLBACK');
|
||||
|
||||
// Make sure we re-enable the trigger even if there was an error
|
||||
await localConnection.query('ALTER TABLE categories ENABLE TRIGGER update_categories_updated_at');
|
||||
} catch (rollbackError) {
|
||||
console.error("Error during rollback:", rollbackError);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "error",
|
||||
operation: "Categories import failed",
|
||||
error: error.message
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importCategories;
|
||||
779
inventory-server/scripts/import/orders.js
Normal file
779
inventory-server/scripts/import/orders.js
Normal file
@@ -0,0 +1,779 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
|
||||
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
|
||||
|
||||
/**
|
||||
* Imports orders from a production MySQL database to a local PostgreSQL database.
|
||||
* It can run in two modes:
|
||||
* 1. Incremental update mode (default): Only fetch orders that have changed since the last sync time.
|
||||
* 2. Full update mode: Fetch all eligible orders within the last 5 years regardless of timestamp.
|
||||
*
|
||||
* @param {object} prodConnection - A MySQL connection to production DB (MySQL 5.7).
|
||||
* @param {object} localConnection - A MySQL connection to local DB (MySQL 8.0).
|
||||
* @param {boolean} incrementalUpdate - Set to false for a full sync; true for incremental.
|
||||
*
|
||||
* @returns {object} Information about the sync operation.
|
||||
*/
|
||||
async function importOrders(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
const skippedOrders = new Set();
|
||||
const missingProducts = new Set();
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
let processedCount = 0;
|
||||
let importedCount = 0;
|
||||
let totalOrderItems = 0;
|
||||
let totalUniqueOrders = 0;
|
||||
let cumulativeProcessedOrders = 0;
|
||||
|
||||
try {
|
||||
// Get last sync info - NOT in a transaction anymore
|
||||
const [syncInfo] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
|
||||
);
|
||||
const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
|
||||
|
||||
console.log('Orders: Using last sync time:', lastSyncTime);
|
||||
|
||||
// First get count of order items - Keep MySQL compatible for production
|
||||
const [[{ total }]] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE o.order_status >= 15
|
||||
AND o.date_placed >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
AND o.date_placed IS NOT NULL
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
o.stamp > ?
|
||||
OR oi.stamp > ?
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_discount_items odi
|
||||
WHERE odi.order_id = o.order_id
|
||||
AND odi.pid = oi.prod_pid
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_tax_info oti
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
WHERE oti.order_id = o.order_id
|
||||
AND otip.pid = oi.prod_pid
|
||||
AND oti.stamp > ?
|
||||
)
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
totalOrderItems = total;
|
||||
console.log('Orders: Found changes:', totalOrderItems);
|
||||
|
||||
// Get order items - Keep MySQL compatible for production
|
||||
console.log('Orders: Starting MySQL query...');
|
||||
const [orderItems] = await prodConnection.query(`
|
||||
SELECT
|
||||
oi.order_id,
|
||||
oi.prod_pid,
|
||||
COALESCE(NULLIF(TRIM(oi.prod_itemnumber), ''), 'NO-SKU') as SKU,
|
||||
oi.prod_price as price,
|
||||
oi.qty_ordered as quantity,
|
||||
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
|
||||
oi.stamp as last_modified
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE o.order_status >= 15
|
||||
AND o.date_placed >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
AND o.date_placed IS NOT NULL
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
o.stamp > ?
|
||||
OR oi.stamp > ?
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_discount_items odi
|
||||
WHERE odi.order_id = o.order_id
|
||||
AND odi.pid = oi.prod_pid
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_tax_info oti
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
WHERE oti.order_id = o.order_id
|
||||
AND otip.pid = oi.prod_pid
|
||||
AND oti.stamp > ?
|
||||
)
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
console.log('Orders: Found', orderItems.length, 'order items to process');
|
||||
|
||||
// Create tables in PostgreSQL for data processing
|
||||
// Start a transaction just for creating the temp tables
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS temp_order_items;
|
||||
DROP TABLE IF EXISTS temp_order_meta;
|
||||
DROP TABLE IF EXISTS temp_order_discounts;
|
||||
DROP TABLE IF EXISTS temp_order_taxes;
|
||||
DROP TABLE IF EXISTS temp_order_costs;
|
||||
DROP TABLE IF EXISTS temp_main_discounts;
|
||||
DROP TABLE IF EXISTS temp_item_discounts;
|
||||
|
||||
CREATE TEMP TABLE temp_order_items (
|
||||
order_id INTEGER NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
sku TEXT NOT NULL,
|
||||
price NUMERIC(14, 4) NOT NULL,
|
||||
quantity INTEGER NOT NULL,
|
||||
base_discount NUMERIC(14, 4) DEFAULT 0,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_order_meta (
|
||||
order_id INTEGER NOT NULL,
|
||||
date TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
customer TEXT NOT NULL,
|
||||
customer_name TEXT NOT NULL,
|
||||
status TEXT,
|
||||
canceled BOOLEAN,
|
||||
summary_discount NUMERIC(14, 4) DEFAULT 0.0000,
|
||||
summary_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
|
||||
summary_discount_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
|
||||
PRIMARY KEY (order_id)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_order_discounts (
|
||||
order_id INTEGER NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
discount NUMERIC(14, 4) NOT NULL,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_main_discounts (
|
||||
order_id INTEGER NOT NULL,
|
||||
discount_id INTEGER NOT NULL,
|
||||
discount_amount_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
|
||||
PRIMARY KEY (order_id, discount_id)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_item_discounts (
|
||||
order_id INTEGER NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
discount_id INTEGER NOT NULL,
|
||||
amount NUMERIC(14, 4) NOT NULL,
|
||||
PRIMARY KEY (order_id, pid, discount_id)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_order_taxes (
|
||||
order_id INTEGER NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
tax NUMERIC(14, 4) NOT NULL,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE temp_order_costs (
|
||||
order_id INTEGER NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
costeach NUMERIC(14, 4) DEFAULT 0.0000,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_temp_order_items_pid ON temp_order_items(pid);
|
||||
CREATE INDEX idx_temp_order_meta_order_id ON temp_order_meta(order_id);
|
||||
CREATE INDEX idx_temp_order_discounts_order_pid ON temp_order_discounts(order_id, pid);
|
||||
CREATE INDEX idx_temp_order_taxes_order_pid ON temp_order_taxes(order_id, pid);
|
||||
CREATE INDEX idx_temp_order_costs_order_pid ON temp_order_costs(order_id, pid);
|
||||
CREATE INDEX idx_temp_main_discounts_discount_id ON temp_main_discounts(discount_id);
|
||||
CREATE INDEX idx_temp_item_discounts_order_pid ON temp_item_discounts(order_id, pid);
|
||||
CREATE INDEX idx_temp_item_discounts_discount_id ON temp_item_discounts(discount_id);
|
||||
`);
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Insert order items in batches - each batch gets its own transaction
|
||||
for (let i = 0; i < orderItems.length; i += 5000) {
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
|
||||
const placeholders = batch.map((_, idx) =>
|
||||
`($${idx * 6 + 1}, $${idx * 6 + 2}, $${idx * 6 + 3}, $${idx * 6 + 4}, $${idx * 6 + 5}, $${idx * 6 + 6})`
|
||||
).join(",");
|
||||
const values = batch.flatMap(item => [
|
||||
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity, item.base_discount
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity, base_discount)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id, pid) DO UPDATE SET
|
||||
sku = EXCLUDED.sku,
|
||||
price = EXCLUDED.price,
|
||||
quantity = EXCLUDED.quantity,
|
||||
base_discount = EXCLUDED.base_discount
|
||||
`, values);
|
||||
|
||||
await localConnection.commit();
|
||||
|
||||
processedCount = i + batch.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
|
||||
current: processedCount,
|
||||
total: totalOrderItems,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalOrderItems),
|
||||
rate: calculateRate(startTime, processedCount)
|
||||
});
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Get unique order IDs
|
||||
const orderIds = [...new Set(orderItems.map(item => item.order_id))];
|
||||
totalUniqueOrders = orderIds.length;
|
||||
console.log('Orders: Processing', totalUniqueOrders, 'unique orders');
|
||||
|
||||
// Reset processed count for order processing phase
|
||||
processedCount = 0;
|
||||
|
||||
// Process metadata, discounts, taxes, and costs in parallel
|
||||
const METADATA_BATCH_SIZE = 2000;
|
||||
const PG_BATCH_SIZE = 200;
|
||||
|
||||
// Add a helper function for title case conversion
|
||||
function toTitleCase(str) {
|
||||
if (!str) return '';
|
||||
return str.toLowerCase().split(' ').map(word => {
|
||||
return word.charAt(0).toUpperCase() + word.slice(1);
|
||||
}).join(' ');
|
||||
}
|
||||
|
||||
const processMetadataBatch = async (batchIds) => {
|
||||
const [orders] = await prodConnection.query(`
|
||||
SELECT
|
||||
o.order_id,
|
||||
o.date_placed as date,
|
||||
o.order_cid as customer,
|
||||
CONCAT(COALESCE(u.firstname, ''), ' ', COALESCE(u.lastname, '')) as customer_name,
|
||||
o.order_status as status,
|
||||
CASE WHEN o.date_cancelled != '0000-00-00 00:00:00' THEN 1 ELSE 0 END as canceled,
|
||||
o.summary_discount,
|
||||
o.summary_subtotal,
|
||||
o.summary_discount_subtotal
|
||||
FROM _order o
|
||||
LEFT JOIN users u ON o.order_cid = u.cid
|
||||
WHERE o.order_id IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
// Process in sub-batches for PostgreSQL
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
for (let j = 0; j < orders.length; j += PG_BATCH_SIZE) {
|
||||
const subBatch = orders.slice(j, j + PG_BATCH_SIZE);
|
||||
if (subBatch.length === 0) continue;
|
||||
|
||||
const placeholders = subBatch.map((_, idx) =>
|
||||
`($${idx * 9 + 1}, $${idx * 9 + 2}, $${idx * 9 + 3}, $${idx * 9 + 4}, $${idx * 9 + 5}, $${idx * 9 + 6}, $${idx * 9 + 7}, $${idx * 9 + 8}, $${idx * 9 + 9})`
|
||||
).join(",");
|
||||
|
||||
const values = subBatch.flatMap(order => [
|
||||
order.order_id,
|
||||
new Date(order.date), // Convert to TIMESTAMP WITH TIME ZONE
|
||||
order.customer,
|
||||
toTitleCase(order.customer_name) || '',
|
||||
order.status.toString(), // Convert status to TEXT
|
||||
order.canceled,
|
||||
order.summary_discount || 0,
|
||||
order.summary_subtotal || 0,
|
||||
order.summary_discount_subtotal || 0
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_meta (
|
||||
order_id, date, customer, customer_name, status, canceled,
|
||||
summary_discount, summary_subtotal, summary_discount_subtotal
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id) DO UPDATE SET
|
||||
date = EXCLUDED.date,
|
||||
customer = EXCLUDED.customer,
|
||||
customer_name = EXCLUDED.customer_name,
|
||||
status = EXCLUDED.status,
|
||||
canceled = EXCLUDED.canceled,
|
||||
summary_discount = EXCLUDED.summary_discount,
|
||||
summary_subtotal = EXCLUDED.summary_subtotal,
|
||||
summary_discount_subtotal = EXCLUDED.summary_discount_subtotal
|
||||
`, values);
|
||||
}
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const processDiscountsBatch = async (batchIds) => {
|
||||
// First, load main discount records
|
||||
const [mainDiscounts] = await prodConnection.query(`
|
||||
SELECT order_id, discount_id, discount_amount_subtotal
|
||||
FROM order_discounts
|
||||
WHERE order_id IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
if (mainDiscounts.length > 0) {
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
for (let j = 0; j < mainDiscounts.length; j += PG_BATCH_SIZE) {
|
||||
const subBatch = mainDiscounts.slice(j, j + PG_BATCH_SIZE);
|
||||
if (subBatch.length === 0) continue;
|
||||
|
||||
const placeholders = subBatch.map((_, idx) =>
|
||||
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
|
||||
).join(",");
|
||||
|
||||
const values = subBatch.flatMap(d => [
|
||||
d.order_id,
|
||||
d.discount_id,
|
||||
d.discount_amount_subtotal || 0
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_main_discounts (order_id, discount_id, discount_amount_subtotal)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id, discount_id) DO UPDATE SET
|
||||
discount_amount_subtotal = EXCLUDED.discount_amount_subtotal
|
||||
`, values);
|
||||
}
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Then, load item discount records
|
||||
const [discounts] = await prodConnection.query(`
|
||||
SELECT order_id, pid, discount_id, amount
|
||||
FROM order_discount_items
|
||||
WHERE order_id IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
if (discounts.length === 0) return;
|
||||
|
||||
// Process in memory to handle potential duplicates
|
||||
const discountMap = new Map();
|
||||
for (const d of discounts) {
|
||||
const key = `${d.order_id}-${d.pid}-${d.discount_id}`;
|
||||
discountMap.set(key, d);
|
||||
}
|
||||
|
||||
const uniqueDiscounts = Array.from(discountMap.values());
|
||||
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
for (let j = 0; j < uniqueDiscounts.length; j += PG_BATCH_SIZE) {
|
||||
const subBatch = uniqueDiscounts.slice(j, j + PG_BATCH_SIZE);
|
||||
if (subBatch.length === 0) continue;
|
||||
|
||||
const placeholders = subBatch.map((_, idx) =>
|
||||
`($${idx * 4 + 1}, $${idx * 4 + 2}, $${idx * 4 + 3}, $${idx * 4 + 4})`
|
||||
).join(",");
|
||||
|
||||
const values = subBatch.flatMap(d => [
|
||||
d.order_id,
|
||||
d.pid,
|
||||
d.discount_id,
|
||||
d.amount || 0
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_item_discounts (order_id, pid, discount_id, amount)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id, pid, discount_id) DO UPDATE SET
|
||||
amount = EXCLUDED.amount
|
||||
`, values);
|
||||
}
|
||||
|
||||
// Create aggregated view with a simpler, safer query that avoids duplicates
|
||||
await localConnection.query(`
|
||||
TRUNCATE temp_order_discounts;
|
||||
|
||||
INSERT INTO temp_order_discounts (order_id, pid, discount)
|
||||
SELECT order_id, pid, SUM(amount) as discount
|
||||
FROM temp_item_discounts
|
||||
GROUP BY order_id, pid
|
||||
`);
|
||||
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const processTaxesBatch = async (batchIds) => {
|
||||
// Optimized tax query to avoid subquery
|
||||
const [taxes] = await prodConnection.query(`
|
||||
SELECT oti.order_id, otip.pid, otip.item_taxes_to_collect as tax
|
||||
FROM (
|
||||
SELECT order_id, MAX(taxinfo_id) as latest_taxinfo_id
|
||||
FROM order_tax_info
|
||||
WHERE order_id IN (?)
|
||||
GROUP BY order_id
|
||||
) latest_info
|
||||
JOIN order_tax_info oti ON oti.order_id = latest_info.order_id
|
||||
AND oti.taxinfo_id = latest_info.latest_taxinfo_id
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
`, [batchIds]);
|
||||
|
||||
if (taxes.length === 0) return;
|
||||
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
for (let j = 0; j < taxes.length; j += PG_BATCH_SIZE) {
|
||||
const subBatch = taxes.slice(j, j + PG_BATCH_SIZE);
|
||||
if (subBatch.length === 0) continue;
|
||||
|
||||
const placeholders = subBatch.map((_, idx) =>
|
||||
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
|
||||
).join(",");
|
||||
|
||||
const values = subBatch.flatMap(t => [
|
||||
t.order_id,
|
||||
t.pid,
|
||||
t.tax || 0
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_taxes (order_id, pid, tax)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id, pid) DO UPDATE SET
|
||||
tax = EXCLUDED.tax
|
||||
`, values);
|
||||
}
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const processCostsBatch = async (batchIds) => {
|
||||
// Modified query to ensure one row per order_id/pid by using a subquery
|
||||
const [costs] = await prodConnection.query(`
|
||||
SELECT
|
||||
oc.orderid as order_id,
|
||||
oc.pid,
|
||||
oc.costeach
|
||||
FROM order_costs oc
|
||||
INNER JOIN (
|
||||
SELECT
|
||||
orderid,
|
||||
pid,
|
||||
MAX(id) as max_id
|
||||
FROM order_costs
|
||||
WHERE orderid IN (?)
|
||||
AND pending = 0
|
||||
GROUP BY orderid, pid
|
||||
) latest ON oc.orderid = latest.orderid AND oc.pid = latest.pid AND oc.id = latest.max_id
|
||||
`, [batchIds]);
|
||||
|
||||
if (costs.length === 0) return;
|
||||
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
for (let j = 0; j < costs.length; j += PG_BATCH_SIZE) {
|
||||
const subBatch = costs.slice(j, j + PG_BATCH_SIZE);
|
||||
if (subBatch.length === 0) continue;
|
||||
|
||||
const placeholders = subBatch.map((_, idx) =>
|
||||
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
|
||||
).join(",");
|
||||
|
||||
const values = subBatch.flatMap(c => [
|
||||
c.order_id,
|
||||
c.pid,
|
||||
c.costeach || 0
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_costs (order_id, pid, costeach)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_id, pid) DO UPDATE SET
|
||||
costeach = EXCLUDED.costeach
|
||||
`, values);
|
||||
}
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// Process all data types SEQUENTIALLY for each batch - not in parallel
|
||||
for (let i = 0; i < orderIds.length; i += METADATA_BATCH_SIZE) {
|
||||
const batchIds = orderIds.slice(i, i + METADATA_BATCH_SIZE);
|
||||
|
||||
// Run these sequentially instead of in parallel to avoid transaction conflicts
|
||||
await processMetadataBatch(batchIds);
|
||||
await processDiscountsBatch(batchIds);
|
||||
await processTaxesBatch(batchIds);
|
||||
await processCostsBatch(batchIds);
|
||||
|
||||
processedCount = i + batchIds.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order data: ${processedCount} of ${totalUniqueOrders}`,
|
||||
current: processedCount,
|
||||
total: totalUniqueOrders,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalUniqueOrders),
|
||||
rate: calculateRate(startTime, processedCount)
|
||||
});
|
||||
}
|
||||
|
||||
// Pre-check all products at once
|
||||
const allOrderPids = [...new Set(orderItems.map(item => item.prod_pid))];
|
||||
console.log('Orders: Checking', allOrderPids.length, 'unique products');
|
||||
|
||||
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
|
||||
"SELECT pid FROM products WHERE pid = ANY($1::bigint[])",
|
||||
[allOrderPids]
|
||||
) : [[]];
|
||||
|
||||
const existingPids = new Set(existingProducts.rows.map(p => p.pid));
|
||||
|
||||
// Process in smaller batches
|
||||
for (let i = 0; i < orderIds.length; i += 2000) { // Increased from 1000 to 2000
|
||||
const batchIds = orderIds.slice(i, i + 2000);
|
||||
|
||||
// Get combined data for this batch in sub-batches
|
||||
const PG_BATCH_SIZE = 200; // Increased from 100 to 200
|
||||
for (let j = 0; j < batchIds.length; j += PG_BATCH_SIZE) {
|
||||
const subBatchIds = batchIds.slice(j, j + PG_BATCH_SIZE);
|
||||
|
||||
// Start a transaction for this sub-batch
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
const [orders] = await localConnection.query(`
|
||||
WITH order_totals AS (
|
||||
SELECT
|
||||
oi.order_id,
|
||||
oi.pid,
|
||||
-- Instead of using ARRAY_AGG which can cause duplicate issues, use SUM with a CASE
|
||||
SUM(CASE
|
||||
WHEN COALESCE(md.discount_amount_subtotal, 0) > 0 THEN id.amount
|
||||
ELSE 0
|
||||
END) as promo_discount_sum,
|
||||
COALESCE(ot.tax, 0) as total_tax,
|
||||
COALESCE(oc.costeach, oi.price * 0.5) as costeach
|
||||
FROM temp_order_items oi
|
||||
LEFT JOIN temp_item_discounts id ON oi.order_id = id.order_id AND oi.pid = id.pid
|
||||
LEFT JOIN temp_main_discounts md ON id.order_id = md.order_id AND id.discount_id = md.discount_id
|
||||
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
|
||||
LEFT JOIN temp_order_costs oc ON oi.order_id = oc.order_id AND oi.pid = oc.pid
|
||||
WHERE oi.order_id = ANY($1)
|
||||
GROUP BY oi.order_id, oi.pid, ot.tax, oc.costeach
|
||||
)
|
||||
SELECT
|
||||
oi.order_id as order_number,
|
||||
oi.pid::bigint as pid,
|
||||
oi.sku,
|
||||
om.date,
|
||||
oi.price,
|
||||
oi.quantity,
|
||||
(
|
||||
-- Part 1: Sale Savings for the Line
|
||||
(oi.base_discount * oi.quantity)
|
||||
+
|
||||
-- Part 2: Prorated Points Discount (if applicable)
|
||||
CASE
|
||||
WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
|
||||
COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
|
||||
ELSE 0
|
||||
END
|
||||
+
|
||||
-- Part 3: Specific Item-Level Discount (only if parent discount affected subtotal)
|
||||
COALESCE(ot.promo_discount_sum, 0)
|
||||
)::NUMERIC(14, 4) as discount,
|
||||
COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
|
||||
false as tax_included,
|
||||
0 as shipping,
|
||||
om.customer,
|
||||
om.customer_name,
|
||||
om.status,
|
||||
om.canceled,
|
||||
COALESCE(ot.costeach, oi.price * 0.5)::NUMERIC(14, 4) as costeach
|
||||
FROM temp_order_items oi
|
||||
JOIN temp_order_meta om ON oi.order_id = om.order_id
|
||||
LEFT JOIN order_totals ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
|
||||
WHERE oi.order_id = ANY($1)
|
||||
ORDER BY oi.order_id, oi.pid
|
||||
`, [subBatchIds]);
|
||||
|
||||
// Filter orders and track missing products
|
||||
const validOrders = [];
|
||||
const processedOrderItems = new Set();
|
||||
const processedOrders = new Set();
|
||||
|
||||
for (const order of orders.rows) {
|
||||
if (!existingPids.has(order.pid)) {
|
||||
missingProducts.add(order.pid);
|
||||
skippedOrders.add(order.order_number);
|
||||
continue;
|
||||
}
|
||||
validOrders.push(order);
|
||||
processedOrderItems.add(`${order.order_number}-${order.pid}`);
|
||||
processedOrders.add(order.order_number);
|
||||
}
|
||||
|
||||
// Process valid orders in smaller sub-batches
|
||||
const FINAL_BATCH_SIZE = 100; // Increased from 50 to 100
|
||||
for (let k = 0; k < validOrders.length; k += FINAL_BATCH_SIZE) {
|
||||
const subBatch = validOrders.slice(k, k + FINAL_BATCH_SIZE);
|
||||
|
||||
const placeholders = subBatch.map((_, idx) => {
|
||||
const base = idx * 15; // 15 columns including costeach
|
||||
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
||||
}).join(',');
|
||||
|
||||
const batchValues = subBatch.flatMap(o => [
|
||||
o.order_number,
|
||||
o.pid,
|
||||
o.sku || 'NO-SKU',
|
||||
o.date, // This is now a TIMESTAMP WITH TIME ZONE
|
||||
o.price,
|
||||
o.quantity,
|
||||
o.discount,
|
||||
o.tax,
|
||||
o.tax_included,
|
||||
o.shipping,
|
||||
o.customer,
|
||||
o.customer_name,
|
||||
o.status.toString(), // Convert status to TEXT
|
||||
o.canceled,
|
||||
o.costeach
|
||||
]);
|
||||
|
||||
const [result] = await localConnection.query(`
|
||||
WITH inserted_orders AS (
|
||||
INSERT INTO orders (
|
||||
order_number, pid, sku, date, price, quantity, discount,
|
||||
tax, tax_included, shipping, customer, customer_name,
|
||||
status, canceled, costeach
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (order_number, pid) DO UPDATE SET
|
||||
sku = EXCLUDED.sku,
|
||||
date = EXCLUDED.date,
|
||||
price = EXCLUDED.price,
|
||||
quantity = EXCLUDED.quantity,
|
||||
discount = EXCLUDED.discount,
|
||||
tax = EXCLUDED.tax,
|
||||
tax_included = EXCLUDED.tax_included,
|
||||
shipping = EXCLUDED.shipping,
|
||||
customer = EXCLUDED.customer,
|
||||
customer_name = EXCLUDED.customer_name,
|
||||
status = EXCLUDED.status,
|
||||
canceled = EXCLUDED.canceled,
|
||||
costeach = EXCLUDED.costeach
|
||||
WHERE -- Only update if at least one key field has changed
|
||||
orders.price IS DISTINCT FROM EXCLUDED.price OR
|
||||
orders.quantity IS DISTINCT FROM EXCLUDED.quantity OR
|
||||
orders.discount IS DISTINCT FROM EXCLUDED.discount OR
|
||||
orders.tax IS DISTINCT FROM EXCLUDED.tax OR
|
||||
orders.status IS DISTINCT FROM EXCLUDED.status OR
|
||||
orders.canceled IS DISTINCT FROM EXCLUDED.canceled OR
|
||||
orders.costeach IS DISTINCT FROM EXCLUDED.costeach OR
|
||||
orders.date IS DISTINCT FROM EXCLUDED.date
|
||||
RETURNING xmax = 0 as inserted
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE inserted) as inserted,
|
||||
COUNT(*) FILTER (WHERE NOT inserted) as updated
|
||||
FROM inserted_orders
|
||||
`, batchValues);
|
||||
|
||||
const { inserted, updated } = result.rows[0];
|
||||
recordsAdded += parseInt(inserted) || 0;
|
||||
recordsUpdated += parseInt(updated) || 0;
|
||||
importedCount += subBatch.length;
|
||||
}
|
||||
|
||||
await localConnection.commit();
|
||||
|
||||
cumulativeProcessedOrders += processedOrders.size;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Importing orders: ${cumulativeProcessedOrders} of ${totalUniqueOrders}`,
|
||||
current: cumulativeProcessedOrders,
|
||||
total: totalUniqueOrders,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, cumulativeProcessedOrders, totalUniqueOrders),
|
||||
rate: calculateRate(startTime, cumulativeProcessedOrders)
|
||||
});
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start a transaction for updating sync status and dropping temp tables
|
||||
await localConnection.beginTransaction();
|
||||
try {
|
||||
// Update sync status
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('orders', NOW())
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
// Cleanup temporary tables
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS temp_order_items;
|
||||
DROP TABLE IF EXISTS temp_order_meta;
|
||||
DROP TABLE IF EXISTS temp_order_discounts;
|
||||
DROP TABLE IF EXISTS temp_order_taxes;
|
||||
DROP TABLE IF EXISTS temp_order_costs;
|
||||
DROP TABLE IF EXISTS temp_main_discounts;
|
||||
DROP TABLE IF EXISTS temp_item_discounts;
|
||||
`);
|
||||
|
||||
// Commit final transaction
|
||||
await localConnection.commit();
|
||||
} catch (error) {
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: Math.floor(importedCount) || 0,
|
||||
recordsAdded: parseInt(recordsAdded) || 0,
|
||||
recordsUpdated: parseInt(recordsUpdated) || 0,
|
||||
totalSkipped: skippedOrders.size || 0,
|
||||
missingProducts: missingProducts.size || 0,
|
||||
totalProcessed: orderItems.length, // Total order items in source
|
||||
incrementalUpdate,
|
||||
lastSyncTime,
|
||||
details: {
|
||||
uniqueOrdersProcessed: cumulativeProcessedOrders,
|
||||
totalOrderItems: orderItems.length,
|
||||
skippedDueToMissingProducts: skippedOrders.size,
|
||||
missingProductIds: Array.from(missingProducts).slice(0, 100) // First 100 for debugging
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error during orders import:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importOrders;
|
||||
950
inventory-server/scripts/import/products.js
Normal file
950
inventory-server/scripts/import/products.js
Normal file
@@ -0,0 +1,950 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
|
||||
const BATCH_SIZE = 1000; // Smaller batch size for better progress tracking
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY = 5000; // 5 seconds
|
||||
const dotenv = require("dotenv");
|
||||
const path = require("path");
|
||||
dotenv.config({ path: path.join(__dirname, "../../.env") });
|
||||
|
||||
// Utility functions
|
||||
const imageUrlBase = process.env.PRODUCT_IMAGE_URL_BASE || 'https://sbing.com/i/products/0000/';
|
||||
const getImageUrls = (pid, iid = 1) => {
|
||||
const paddedPid = pid.toString().padStart(6, '0');
|
||||
// Use padded PID only for the first 3 digits
|
||||
const prefix = paddedPid.slice(0, 3);
|
||||
// Use the actual pid for the rest of the URL
|
||||
const basePath = `${imageUrlBase}${prefix}/${pid}`;
|
||||
return {
|
||||
image: `${basePath}-t-${iid}.jpg`,
|
||||
image_175: `${basePath}-175x175-${iid}.jpg`,
|
||||
image_full: `${basePath}-o-${iid}.jpg`
|
||||
};
|
||||
};
|
||||
|
||||
// Add helper function for retrying operations with exponential backoff
|
||||
async function withRetry(operation, errorMessage) {
|
||||
let lastError;
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const backoffTime = RETRY_DELAY * Math.pow(2, attempt - 1);
|
||||
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
// Add helper function at the top of the file
|
||||
function validateDate(mysqlDate) {
|
||||
if (!mysqlDate || mysqlDate === '0000-00-00' || mysqlDate === '0000-00-00 00:00:00') {
|
||||
return null;
|
||||
}
|
||||
// Check if the date is valid
|
||||
const date = new Date(mysqlDate);
|
||||
return isNaN(date.getTime()) ? null : mysqlDate;
|
||||
}
|
||||
|
||||
async function setupTemporaryTables(connection) {
|
||||
// Drop the table if it exists
|
||||
await connection.query('DROP TABLE IF EXISTS temp_products');
|
||||
|
||||
// Create the temporary table
|
||||
await connection.query(`
|
||||
CREATE TEMP TABLE temp_products (
|
||||
pid BIGINT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
sku TEXT,
|
||||
stock_quantity INTEGER DEFAULT 0,
|
||||
preorder_count INTEGER DEFAULT 0,
|
||||
notions_inv_count INTEGER DEFAULT 0,
|
||||
price NUMERIC(14, 4) NOT NULL DEFAULT 0,
|
||||
regular_price NUMERIC(14, 4) NOT NULL DEFAULT 0,
|
||||
cost_price NUMERIC(14, 4),
|
||||
vendor TEXT,
|
||||
vendor_reference TEXT,
|
||||
notions_reference TEXT,
|
||||
brand TEXT,
|
||||
line TEXT,
|
||||
subline TEXT,
|
||||
artist TEXT,
|
||||
categories TEXT,
|
||||
created_at TIMESTAMP WITH TIME ZONE,
|
||||
first_received TIMESTAMP WITH TIME ZONE,
|
||||
landing_cost_price NUMERIC(14, 4),
|
||||
barcode TEXT,
|
||||
harmonized_tariff_code TEXT,
|
||||
updated_at TIMESTAMP WITH TIME ZONE,
|
||||
visible BOOLEAN,
|
||||
managing_stock BOOLEAN DEFAULT true,
|
||||
replenishable BOOLEAN,
|
||||
permalink TEXT,
|
||||
moq INTEGER DEFAULT 1,
|
||||
uom INTEGER DEFAULT 1,
|
||||
rating NUMERIC(14, 4),
|
||||
reviews INTEGER,
|
||||
weight NUMERIC(14, 4),
|
||||
length NUMERIC(14, 4),
|
||||
width NUMERIC(14, 4),
|
||||
height NUMERIC(14, 4),
|
||||
country_of_origin TEXT,
|
||||
location TEXT,
|
||||
total_sold INTEGER,
|
||||
baskets INTEGER,
|
||||
notifies INTEGER,
|
||||
date_last_sold TIMESTAMP WITH TIME ZONE,
|
||||
primary_iid INTEGER,
|
||||
image TEXT,
|
||||
image_175 TEXT,
|
||||
image_full TEXT,
|
||||
options TEXT,
|
||||
tags TEXT,
|
||||
needs_update BOOLEAN DEFAULT TRUE,
|
||||
PRIMARY KEY (pid)
|
||||
)`);
|
||||
|
||||
// Create the index
|
||||
await connection.query('CREATE INDEX idx_temp_products_needs_update ON temp_products (needs_update)');
|
||||
}
|
||||
|
||||
async function cleanupTemporaryTables(connection) {
|
||||
await connection.query('DROP TABLE IF EXISTS temp_products');
|
||||
}
|
||||
|
||||
async function importMissingProducts(prodConnection, localConnection, missingPids) {
|
||||
if (!missingPids || missingPids.length === 0) {
|
||||
return {
|
||||
status: "complete",
|
||||
recordsAdded: 0,
|
||||
message: "No missing products to import"
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Setup temporary tables
|
||||
await setupTemporaryTables(localConnection);
|
||||
|
||||
// Get product data from production - Keep MySQL compatible
|
||||
const [prodData] = await prodConnection.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.description AS title,
|
||||
p.notes AS description,
|
||||
p.itemnumber AS sku,
|
||||
p.date_created,
|
||||
p.datein AS first_received,
|
||||
p.location,
|
||||
p.upc AS barcode,
|
||||
p.harmonized_tariff_code,
|
||||
p.stamp AS updated_at,
|
||||
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
|
||||
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
|
||||
WHEN (
|
||||
(COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
|
||||
AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END AS replenishable,
|
||||
COALESCE(si.available_local, 0) as stock_quantity,
|
||||
0 as pending_qty,
|
||||
COALESCE(ci.onpreorder, 0) as preorder_count,
|
||||
COALESCE(pnb.inventory, 0) as notions_inv_count,
|
||||
COALESCE(pcp.price_each, 0) as price,
|
||||
COALESCE(p.sellingprice, 0) AS regular_price,
|
||||
CASE
|
||||
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
THEN (
|
||||
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
|
||||
FROM product_inventory
|
||||
WHERE pid = p.pid AND count > 0
|
||||
)
|
||||
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
|
||||
END AS cost_price,
|
||||
NULL as landing_cost_price,
|
||||
s.companyname AS vendor,
|
||||
CASE
|
||||
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
|
||||
ELSE sid.supplier_itemnumber
|
||||
END AS vendor_reference,
|
||||
sid.notions_itemnumber AS notions_reference,
|
||||
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
|
||||
pc1.name AS brand,
|
||||
pc2.name AS line,
|
||||
pc3.name AS subline,
|
||||
pc4.name AS artist,
|
||||
COALESCE(CASE
|
||||
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
|
||||
ELSE sid.supplier_qty_per_unit
|
||||
END, sid.notions_qty_per_unit) AS moq,
|
||||
p.rating,
|
||||
p.rating_votes AS reviews,
|
||||
p.weight,
|
||||
p.length,
|
||||
p.width,
|
||||
p.height,
|
||||
p.country_of_origin,
|
||||
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
|
||||
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
|
||||
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
|
||||
pls.date_sold as date_last_sold,
|
||||
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
|
||||
GROUP_CONCAT(DISTINCT CASE
|
||||
WHEN pc.cat_id IS NOT NULL
|
||||
AND pc.type IN (10, 20, 11, 21, 12, 13)
|
||||
AND pci.cat_id NOT IN (16, 17)
|
||||
THEN pci.cat_id
|
||||
END) as category_ids
|
||||
FROM products p
|
||||
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
|
||||
LEFT JOIN current_inventory ci ON p.pid = ci.pid
|
||||
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
|
||||
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
|
||||
LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
|
||||
LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
|
||||
LEFT JOIN product_category_index pci ON p.pid = pci.pid
|
||||
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
|
||||
LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
|
||||
LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
|
||||
LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
|
||||
LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
|
||||
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
|
||||
WHERE p.pid IN (?)
|
||||
GROUP BY p.pid
|
||||
`, [missingPids]);
|
||||
|
||||
if (!prodData || prodData.length === 0) {
|
||||
return {
|
||||
status: "complete",
|
||||
recordsAdded: 0,
|
||||
message: "No products found in production database"
|
||||
};
|
||||
}
|
||||
|
||||
// Process in batches
|
||||
let recordsAdded = 0;
|
||||
for (let i = 0; i < prodData.length; i += BATCH_SIZE) {
|
||||
const batch = prodData.slice(i, i + BATCH_SIZE);
|
||||
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 48; // 48 columns
|
||||
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(row => {
|
||||
const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
|
||||
return [
|
||||
row.pid,
|
||||
row.title,
|
||||
row.description,
|
||||
row.sku || '',
|
||||
row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
|
||||
row.preorder_count,
|
||||
row.notions_inv_count,
|
||||
row.price,
|
||||
row.regular_price,
|
||||
row.cost_price,
|
||||
row.vendor,
|
||||
row.vendor_reference,
|
||||
row.notions_reference,
|
||||
row.brand,
|
||||
row.line,
|
||||
row.subline,
|
||||
row.artist,
|
||||
row.category_ids,
|
||||
validateDate(row.date_created),
|
||||
validateDate(row.first_received),
|
||||
row.landing_cost_price,
|
||||
row.barcode,
|
||||
row.harmonized_tariff_code,
|
||||
validateDate(row.updated_at),
|
||||
row.visible,
|
||||
true,
|
||||
row.replenishable,
|
||||
row.permalink,
|
||||
Math.max(1, Math.round(row.moq || 1)),
|
||||
1,
|
||||
row.rating,
|
||||
row.reviews,
|
||||
row.weight,
|
||||
row.length,
|
||||
row.width,
|
||||
row.height,
|
||||
row.country_of_origin,
|
||||
row.location,
|
||||
row.total_sold,
|
||||
row.baskets,
|
||||
row.notifies,
|
||||
validateDate(row.date_last_sold),
|
||||
row.primary_iid,
|
||||
imageUrls.image,
|
||||
imageUrls.image_175,
|
||||
imageUrls.image_full,
|
||||
null,
|
||||
null
|
||||
];
|
||||
});
|
||||
|
||||
const [result] = await localConnection.query(`
|
||||
WITH inserted_products AS (
|
||||
INSERT INTO products (
|
||||
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
|
||||
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
|
||||
brand, line, subline, artist, categories, created_at, first_received,
|
||||
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
|
||||
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
|
||||
weight, length, width, height, country_of_origin, location, total_sold,
|
||||
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (pid) DO NOTHING
|
||||
RETURNING pid
|
||||
)
|
||||
SELECT COUNT(*) as inserted FROM inserted_products
|
||||
`, values);
|
||||
|
||||
recordsAdded += parseInt(result.rows[0].inserted, 10) || 0;
|
||||
}
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
recordsAdded,
|
||||
message: `Successfully imported ${recordsAdded} missing products`
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error importing missing products:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function materializeCalculations(prodConnection, localConnection, incrementalUpdate = true, lastSyncTime = '1970-01-01', startTime = Date.now()) {
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: "Fetching product data from production"
|
||||
});
|
||||
|
||||
// Get all product data in a single optimized query - Keep MySQL compatible
|
||||
const [prodData] = await prodConnection.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.description AS title,
|
||||
p.notes AS description,
|
||||
p.itemnumber AS sku,
|
||||
p.date_created,
|
||||
p.datein AS first_received,
|
||||
p.location,
|
||||
p.upc AS barcode,
|
||||
p.harmonized_tariff_code,
|
||||
p.stamp AS updated_at,
|
||||
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
|
||||
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
|
||||
WHEN (
|
||||
(COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
|
||||
AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END AS replenishable,
|
||||
COALESCE(si.available_local, 0) as stock_quantity,
|
||||
0 as pending_qty,
|
||||
COALESCE(ci.onpreorder, 0) as preorder_count,
|
||||
COALESCE(pnb.inventory, 0) as notions_inv_count,
|
||||
COALESCE(pcp.price_each, 0) as price,
|
||||
COALESCE(p.sellingprice, 0) AS regular_price,
|
||||
CASE
|
||||
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
THEN (
|
||||
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
|
||||
FROM product_inventory
|
||||
WHERE pid = p.pid AND count > 0
|
||||
)
|
||||
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
|
||||
END AS cost_price,
|
||||
NULL as landing_cost_price,
|
||||
s.companyname AS vendor,
|
||||
CASE
|
||||
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
|
||||
ELSE sid.supplier_itemnumber
|
||||
END AS vendor_reference,
|
||||
sid.notions_itemnumber AS notions_reference,
|
||||
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
|
||||
pc1.name AS brand,
|
||||
pc2.name AS line,
|
||||
pc3.name AS subline,
|
||||
pc4.name AS artist,
|
||||
COALESCE(CASE
|
||||
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
|
||||
ELSE sid.supplier_qty_per_unit
|
||||
END, sid.notions_qty_per_unit) AS moq,
|
||||
p.rating,
|
||||
p.rating_votes AS reviews,
|
||||
p.weight,
|
||||
p.length,
|
||||
p.width,
|
||||
p.height,
|
||||
p.country_of_origin,
|
||||
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
|
||||
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
|
||||
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
|
||||
pls.date_sold as date_last_sold,
|
||||
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
|
||||
GROUP_CONCAT(DISTINCT CASE
|
||||
WHEN pc.cat_id IS NOT NULL
|
||||
AND pc.type IN (10, 20, 11, 21, 12, 13)
|
||||
AND pci.cat_id NOT IN (16, 17)
|
||||
THEN pci.cat_id
|
||||
END) as category_ids
|
||||
FROM products p
|
||||
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
|
||||
LEFT JOIN current_inventory ci ON p.pid = ci.pid
|
||||
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
|
||||
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
|
||||
LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
|
||||
LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
|
||||
LEFT JOIN product_category_index pci ON p.pid = pci.pid
|
||||
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
|
||||
LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
|
||||
LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
|
||||
LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
|
||||
LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
|
||||
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
|
||||
WHERE ${incrementalUpdate ? `
|
||||
p.stamp > ? OR
|
||||
ci.stamp > ? OR
|
||||
pcp.date_deactive > ? OR
|
||||
pcp.date_active > ? OR
|
||||
pnb.date_updated > ?
|
||||
-- Add condition for product_images changes if needed for incremental updates
|
||||
-- OR EXISTS (SELECT 1 FROM product_images pi WHERE pi.pid = p.pid AND pi.stamp > ?)
|
||||
` : 'TRUE'}
|
||||
GROUP BY p.pid
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime /*, lastSyncTime */] : []);
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Processing ${prodData.length} product records`
|
||||
});
|
||||
|
||||
// Insert all product data into temp table in batches
|
||||
for (let i = 0; i < prodData.length; i += BATCH_SIZE) {
|
||||
const batch = prodData.slice(i, Math.min(i + BATCH_SIZE, prodData.length));
|
||||
|
||||
await withRetry(async () => {
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 48; // 48 columns
|
||||
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(row => {
|
||||
const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
|
||||
return [
|
||||
row.pid,
|
||||
row.title,
|
||||
row.description,
|
||||
row.sku || '',
|
||||
row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
|
||||
row.preorder_count,
|
||||
row.notions_inv_count,
|
||||
row.price,
|
||||
row.regular_price,
|
||||
row.cost_price,
|
||||
row.vendor,
|
||||
row.vendor_reference,
|
||||
row.notions_reference,
|
||||
row.brand,
|
||||
row.line,
|
||||
row.subline,
|
||||
row.artist,
|
||||
row.category_ids,
|
||||
validateDate(row.date_created),
|
||||
validateDate(row.first_received),
|
||||
row.landing_cost_price,
|
||||
row.barcode,
|
||||
row.harmonized_tariff_code,
|
||||
validateDate(row.updated_at),
|
||||
row.visible,
|
||||
true,
|
||||
row.replenishable,
|
||||
row.permalink,
|
||||
Math.max(1, Math.round(row.moq || 1)),
|
||||
1,
|
||||
row.rating,
|
||||
row.reviews,
|
||||
row.weight,
|
||||
row.length,
|
||||
row.width,
|
||||
row.height,
|
||||
row.country_of_origin,
|
||||
row.location,
|
||||
row.total_sold,
|
||||
row.baskets,
|
||||
row.notifies,
|
||||
validateDate(row.date_last_sold),
|
||||
row.primary_iid,
|
||||
imageUrls.image,
|
||||
imageUrls.image_175,
|
||||
imageUrls.image_full,
|
||||
null,
|
||||
null
|
||||
];
|
||||
});
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_products (
|
||||
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
|
||||
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
|
||||
brand, line, subline, artist, categories, created_at, first_received,
|
||||
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
|
||||
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
|
||||
weight, length, width, height, country_of_origin, location, total_sold,
|
||||
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
|
||||
) VALUES ${placeholders}
|
||||
ON CONFLICT (pid) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
description = EXCLUDED.description,
|
||||
sku = EXCLUDED.sku,
|
||||
stock_quantity = EXCLUDED.stock_quantity,
|
||||
preorder_count = EXCLUDED.preorder_count,
|
||||
notions_inv_count = EXCLUDED.notions_inv_count,
|
||||
price = EXCLUDED.price,
|
||||
regular_price = EXCLUDED.regular_price,
|
||||
cost_price = EXCLUDED.cost_price,
|
||||
vendor = EXCLUDED.vendor,
|
||||
vendor_reference = EXCLUDED.vendor_reference,
|
||||
notions_reference = EXCLUDED.notions_reference,
|
||||
brand = EXCLUDED.brand,
|
||||
line = EXCLUDED.line,
|
||||
subline = EXCLUDED.subline,
|
||||
artist = EXCLUDED.artist,
|
||||
created_at = EXCLUDED.created_at,
|
||||
first_received = EXCLUDED.first_received,
|
||||
landing_cost_price = EXCLUDED.landing_cost_price,
|
||||
barcode = EXCLUDED.barcode,
|
||||
harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
visible = EXCLUDED.visible,
|
||||
managing_stock = EXCLUDED.managing_stock,
|
||||
replenishable = EXCLUDED.replenishable,
|
||||
permalink = EXCLUDED.permalink,
|
||||
moq = EXCLUDED.moq,
|
||||
uom = EXCLUDED.uom,
|
||||
rating = EXCLUDED.rating,
|
||||
reviews = EXCLUDED.reviews,
|
||||
weight = EXCLUDED.weight,
|
||||
length = EXCLUDED.length,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
country_of_origin = EXCLUDED.country_of_origin,
|
||||
location = EXCLUDED.location,
|
||||
total_sold = EXCLUDED.total_sold,
|
||||
baskets = EXCLUDED.baskets,
|
||||
notifies = EXCLUDED.notifies,
|
||||
date_last_sold = EXCLUDED.date_last_sold,
|
||||
primary_iid = EXCLUDED.primary_iid,
|
||||
image = EXCLUDED.image,
|
||||
image_175 = EXCLUDED.image_175,
|
||||
image_full = EXCLUDED.image_full,
|
||||
options = EXCLUDED.options,
|
||||
tags = EXCLUDED.tags
|
||||
RETURNING
|
||||
xmax = 0 as inserted
|
||||
`, values);
|
||||
}, `Error inserting batch ${i} to ${i + batch.length}`);
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Imported ${i + batch.length} of ${prodData.length} products`,
|
||||
current: i + batch.length,
|
||||
total: prodData.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, i + batch.length, prodData.length),
|
||||
rate: calculateRate(startTime, i + batch.length)
|
||||
});
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: "Finished materializing calculations"
|
||||
});
|
||||
|
||||
// Add step to identify which products actually need updating
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: "Identifying changed products"
|
||||
});
|
||||
|
||||
// Mark products that haven't changed as needs_update = false
|
||||
await localConnection.query(`
|
||||
UPDATE temp_products t
|
||||
SET needs_update = FALSE
|
||||
FROM products p
|
||||
WHERE t.pid = p.pid
|
||||
AND t.title IS NOT DISTINCT FROM p.title
|
||||
AND t.description IS NOT DISTINCT FROM p.description
|
||||
AND t.sku IS NOT DISTINCT FROM p.sku
|
||||
AND t.stock_quantity = p.stock_quantity
|
||||
AND t.price = p.price
|
||||
AND t.regular_price = p.regular_price
|
||||
AND t.cost_price IS NOT DISTINCT FROM p.cost_price
|
||||
AND t.vendor IS NOT DISTINCT FROM p.vendor
|
||||
AND t.brand IS NOT DISTINCT FROM p.brand
|
||||
AND t.visible = p.visible
|
||||
AND t.replenishable = p.replenishable
|
||||
AND t.barcode IS NOT DISTINCT FROM p.barcode
|
||||
AND t.updated_at IS NOT DISTINCT FROM p.updated_at
|
||||
AND t.total_sold IS NOT DISTINCT FROM p.total_sold
|
||||
-- Check key fields that are likely to change
|
||||
-- We don't need to check every single field, just the important ones
|
||||
`);
|
||||
|
||||
// Get count of products that need updating
|
||||
const [countResult] = await localConnection.query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE needs_update = true) as update_count,
|
||||
COUNT(*) FILTER (WHERE needs_update = false) as skip_count,
|
||||
COUNT(*) as total_count
|
||||
FROM temp_products
|
||||
`);
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Found ${countResult.rows[0].update_count} products that need updating, ${countResult.rows[0].skip_count} unchanged`
|
||||
});
|
||||
|
||||
// Return the total products processed
|
||||
return {
|
||||
totalProcessed: prodData.length,
|
||||
needsUpdate: parseInt(countResult.rows[0].update_count),
|
||||
skipped: parseInt(countResult.rows[0].skip_count)
|
||||
};
|
||||
}
|
||||
|
||||
async function importProducts(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
let lastSyncTime = '1970-01-01';
|
||||
|
||||
try {
|
||||
// Get last sync time if doing incremental update
|
||||
if (incrementalUpdate) {
|
||||
const [syncResult] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'products'"
|
||||
);
|
||||
if (syncResult.rows.length > 0) {
|
||||
lastSyncTime = syncResult.rows[0].last_sync_timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
// Start a transaction to ensure temporary tables persist
|
||||
await localConnection.beginTransaction();
|
||||
|
||||
try {
|
||||
// Setup temporary tables
|
||||
await setupTemporaryTables(localConnection);
|
||||
|
||||
// Materialize calculations into temp table
|
||||
const materializeResult = await materializeCalculations(prodConnection, localConnection, incrementalUpdate, lastSyncTime, startTime);
|
||||
|
||||
// Get the list of products that need updating
|
||||
const [products] = await localConnection.query(`
|
||||
SELECT
|
||||
t.pid,
|
||||
t.title,
|
||||
t.description,
|
||||
t.sku,
|
||||
t.stock_quantity,
|
||||
t.preorder_count,
|
||||
t.notions_inv_count,
|
||||
t.price,
|
||||
t.regular_price,
|
||||
t.cost_price,
|
||||
t.vendor,
|
||||
t.vendor_reference,
|
||||
t.notions_reference,
|
||||
t.brand,
|
||||
t.line,
|
||||
t.subline,
|
||||
t.artist,
|
||||
t.categories,
|
||||
t.created_at,
|
||||
t.first_received,
|
||||
t.landing_cost_price,
|
||||
t.barcode,
|
||||
t.harmonized_tariff_code,
|
||||
t.updated_at,
|
||||
t.visible,
|
||||
t.managing_stock,
|
||||
t.replenishable,
|
||||
t.permalink,
|
||||
t.moq,
|
||||
t.rating,
|
||||
t.reviews,
|
||||
t.weight,
|
||||
t.length,
|
||||
t.width,
|
||||
t.height,
|
||||
t.country_of_origin,
|
||||
t.location,
|
||||
t.total_sold,
|
||||
t.baskets,
|
||||
t.notifies,
|
||||
t.date_last_sold,
|
||||
t.primary_iid,
|
||||
t.image,
|
||||
t.image_175,
|
||||
t.image_full,
|
||||
t.options,
|
||||
t.tags
|
||||
FROM temp_products t
|
||||
WHERE t.needs_update = true
|
||||
`);
|
||||
|
||||
// Process products in batches
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
|
||||
for (let i = 0; i < products.rows.length; i += BATCH_SIZE) {
|
||||
const batch = products.rows.slice(i, i + BATCH_SIZE);
|
||||
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 47; // 47 columns
|
||||
return `(${Array.from({ length: 47 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(row => {
|
||||
const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
|
||||
return [
|
||||
row.pid,
|
||||
row.title,
|
||||
row.description,
|
||||
row.sku || '',
|
||||
row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
|
||||
row.preorder_count,
|
||||
row.notions_inv_count,
|
||||
row.price,
|
||||
row.regular_price,
|
||||
row.cost_price,
|
||||
row.vendor,
|
||||
row.vendor_reference,
|
||||
row.notions_reference,
|
||||
row.brand,
|
||||
row.line,
|
||||
row.subline,
|
||||
row.artist,
|
||||
row.categories,
|
||||
validateDate(row.created_at),
|
||||
validateDate(row.first_received),
|
||||
row.landing_cost_price,
|
||||
row.barcode,
|
||||
row.harmonized_tariff_code,
|
||||
validateDate(row.updated_at),
|
||||
row.visible,
|
||||
row.managing_stock,
|
||||
row.replenishable,
|
||||
row.permalink,
|
||||
row.moq,
|
||||
1,
|
||||
row.rating,
|
||||
row.reviews,
|
||||
row.weight,
|
||||
row.length,
|
||||
row.width,
|
||||
row.height,
|
||||
row.country_of_origin,
|
||||
row.location,
|
||||
row.total_sold,
|
||||
row.baskets,
|
||||
row.notifies,
|
||||
validateDate(row.date_last_sold),
|
||||
imageUrls.image,
|
||||
imageUrls.image_175,
|
||||
imageUrls.image_full,
|
||||
row.options,
|
||||
row.tags
|
||||
];
|
||||
});
|
||||
|
||||
const [result] = await localConnection.query(`
|
||||
WITH upserted AS (
|
||||
INSERT INTO products (
|
||||
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
|
||||
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
|
||||
brand, line, subline, artist, categories, created_at, first_received,
|
||||
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
|
||||
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
|
||||
weight, length, width, height, country_of_origin, location, total_sold,
|
||||
baskets, notifies, date_last_sold, image, image_175, image_full, options, tags
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (pid) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
description = EXCLUDED.description,
|
||||
sku = EXCLUDED.sku,
|
||||
stock_quantity = EXCLUDED.stock_quantity,
|
||||
preorder_count = EXCLUDED.preorder_count,
|
||||
notions_inv_count = EXCLUDED.notions_inv_count,
|
||||
price = EXCLUDED.price,
|
||||
regular_price = EXCLUDED.regular_price,
|
||||
cost_price = EXCLUDED.cost_price,
|
||||
vendor = EXCLUDED.vendor,
|
||||
vendor_reference = EXCLUDED.vendor_reference,
|
||||
notions_reference = EXCLUDED.notions_reference,
|
||||
brand = EXCLUDED.brand,
|
||||
line = EXCLUDED.line,
|
||||
subline = EXCLUDED.subline,
|
||||
artist = EXCLUDED.artist,
|
||||
created_at = EXCLUDED.created_at,
|
||||
first_received = EXCLUDED.first_received,
|
||||
landing_cost_price = EXCLUDED.landing_cost_price,
|
||||
barcode = EXCLUDED.barcode,
|
||||
harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
visible = EXCLUDED.visible,
|
||||
managing_stock = EXCLUDED.managing_stock,
|
||||
replenishable = EXCLUDED.replenishable,
|
||||
permalink = EXCLUDED.permalink,
|
||||
moq = EXCLUDED.moq,
|
||||
uom = EXCLUDED.uom,
|
||||
rating = EXCLUDED.rating,
|
||||
reviews = EXCLUDED.reviews,
|
||||
weight = EXCLUDED.weight,
|
||||
length = EXCLUDED.length,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
country_of_origin = EXCLUDED.country_of_origin,
|
||||
location = EXCLUDED.location,
|
||||
total_sold = EXCLUDED.total_sold,
|
||||
baskets = EXCLUDED.baskets,
|
||||
notifies = EXCLUDED.notifies,
|
||||
date_last_sold = EXCLUDED.date_last_sold,
|
||||
image = EXCLUDED.image,
|
||||
image_175 = EXCLUDED.image_175,
|
||||
image_full = EXCLUDED.image_full,
|
||||
options = EXCLUDED.options,
|
||||
tags = EXCLUDED.tags
|
||||
RETURNING
|
||||
xmax = 0 as inserted
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE inserted) as inserted,
|
||||
COUNT(*) FILTER (WHERE NOT inserted) as updated
|
||||
FROM upserted
|
||||
`, values);
|
||||
|
||||
recordsAdded += parseInt(result.rows[0].inserted, 10) || 0;
|
||||
recordsUpdated += parseInt(result.rows[0].updated, 10) || 0;
|
||||
|
||||
// Process category relationships in batches
|
||||
const allCategories = [];
|
||||
for (const row of batch) {
|
||||
if (row.categories) {
|
||||
const categoryIds = row.categories.split(',').filter(id => id && id.trim());
|
||||
if (categoryIds.length > 0) {
|
||||
categoryIds.forEach(catId => {
|
||||
allCategories.push([row.pid, parseInt(catId.trim(), 10)]);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have categories to process
|
||||
if (allCategories.length > 0) {
|
||||
// First get all products in this batch
|
||||
const productIds = batch.map(p => p.pid);
|
||||
|
||||
// Delete all existing relationships for products in this batch
|
||||
await localConnection.query(
|
||||
'DELETE FROM product_categories WHERE pid = ANY($1)',
|
||||
[productIds]
|
||||
);
|
||||
|
||||
// Insert all new relationships in one batch
|
||||
const catPlaceholders = allCategories.map((_, idx) =>
|
||||
`($${idx * 2 + 1}, $${idx * 2 + 2})`
|
||||
).join(',');
|
||||
|
||||
const catValues = allCategories.flat();
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO product_categories (pid, cat_id)
|
||||
VALUES ${catPlaceholders}
|
||||
ON CONFLICT (pid, cat_id) DO NOTHING
|
||||
`, catValues);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Processing products: ${i + batch.length} of ${products.rows.length}`,
|
||||
current: i + batch.length,
|
||||
total: products.rows.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, i + batch.length, products.rows.length),
|
||||
rate: calculateRate(startTime, i + batch.length)
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup temporary tables
|
||||
await cleanupTemporaryTables(localConnection);
|
||||
|
||||
// Commit the transaction
|
||||
await localConnection.commit();
|
||||
|
||||
// Update sync status
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('products', NOW())
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
return {
|
||||
status: 'complete',
|
||||
recordsAdded,
|
||||
recordsUpdated,
|
||||
totalRecords: products.rows.length,
|
||||
totalProcessed: materializeResult.totalProcessed,
|
||||
duration: formatElapsedTime(startTime),
|
||||
needsUpdate: materializeResult.needsUpdate,
|
||||
skippedUnchanged: materializeResult.skipped
|
||||
};
|
||||
} catch (error) {
|
||||
// Rollback on error
|
||||
await localConnection.rollback();
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error in importProducts:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importProducts,
|
||||
importMissingProducts,
|
||||
setupTemporaryTables,
|
||||
cleanupTemporaryTables,
|
||||
materializeCalculations
|
||||
};
|
||||
884
inventory-server/scripts/import/purchase-orders.js
Normal file
884
inventory-server/scripts/import/purchase-orders.js
Normal file
@@ -0,0 +1,884 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
|
||||
|
||||
/**
|
||||
* Validates a date from MySQL before inserting it into PostgreSQL
|
||||
* @param {string|Date|null} mysqlDate - Date string or object from MySQL
|
||||
* @returns {string|null} Valid date string or null if invalid
|
||||
*/
|
||||
function validateDate(mysqlDate) {
|
||||
// Handle null, undefined, or empty values
|
||||
if (!mysqlDate) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Convert to string if it's not already
|
||||
const dateStr = String(mysqlDate);
|
||||
|
||||
// Handle MySQL zero dates and empty values
|
||||
if (dateStr === '0000-00-00' ||
|
||||
dateStr === '0000-00-00 00:00:00' ||
|
||||
dateStr.indexOf('0000-00-00') !== -1 ||
|
||||
dateStr === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if the date is valid
|
||||
const date = new Date(mysqlDate);
|
||||
|
||||
// If the date is invalid or suspiciously old (pre-1970), return null
|
||||
if (isNaN(date.getTime()) || date.getFullYear() < 1970) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return mysqlDate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports purchase orders and receivings from a production MySQL database to a local PostgreSQL database.
|
||||
* Handles these as separate data streams without complex FIFO allocation.
|
||||
*
|
||||
* @param {object} prodConnection - A MySQL connection to production DB
|
||||
* @param {object} localConnection - A PostgreSQL connection to local DB
|
||||
* @param {boolean} incrementalUpdate - Set to false for a full sync; true for incremental
|
||||
* @returns {object} Information about the sync operation
|
||||
*/
|
||||
async function importPurchaseOrders(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
let poRecordsAdded = 0;
|
||||
let poRecordsUpdated = 0;
|
||||
let poRecordsDeleted = 0;
|
||||
let receivingRecordsAdded = 0;
|
||||
let receivingRecordsUpdated = 0;
|
||||
let receivingRecordsDeleted = 0;
|
||||
let totalProcessed = 0;
|
||||
|
||||
// Batch size constants
|
||||
const PO_BATCH_SIZE = 500;
|
||||
const INSERT_BATCH_SIZE = 100;
|
||||
|
||||
try {
|
||||
// Begin transaction for the entire import process
|
||||
await localConnection.beginTransaction();
|
||||
|
||||
// Get last sync info
|
||||
const [syncInfo] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
|
||||
);
|
||||
const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
|
||||
|
||||
console.log('Purchase Orders: Using last sync time:', lastSyncTime);
|
||||
|
||||
// Create temp tables for processing
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS temp_purchase_orders;
|
||||
DROP TABLE IF EXISTS temp_receivings;
|
||||
DROP TABLE IF EXISTS employee_names;
|
||||
DROP TABLE IF EXISTS temp_supplier_names;
|
||||
|
||||
-- Temporary table for purchase orders
|
||||
CREATE TEMP TABLE temp_purchase_orders (
|
||||
po_id TEXT NOT NULL,
|
||||
pid BIGINT NOT NULL,
|
||||
sku TEXT,
|
||||
name TEXT,
|
||||
vendor TEXT,
|
||||
date TIMESTAMP WITH TIME ZONE,
|
||||
expected_date DATE,
|
||||
status TEXT,
|
||||
notes TEXT,
|
||||
long_note TEXT,
|
||||
ordered INTEGER,
|
||||
po_cost_price NUMERIC(14, 4),
|
||||
supplier_id INTEGER,
|
||||
date_created TIMESTAMP WITH TIME ZONE,
|
||||
date_ordered TIMESTAMP WITH TIME ZONE,
|
||||
PRIMARY KEY (po_id, pid)
|
||||
);
|
||||
|
||||
-- Temporary table for receivings
|
||||
CREATE TEMP TABLE temp_receivings (
|
||||
receiving_id TEXT NOT NULL,
|
||||
pid BIGINT NOT NULL,
|
||||
sku TEXT,
|
||||
name TEXT,
|
||||
vendor TEXT,
|
||||
qty_each INTEGER,
|
||||
qty_each_orig INTEGER,
|
||||
cost_each NUMERIC(14, 5),
|
||||
cost_each_orig NUMERIC(14, 5),
|
||||
received_by INTEGER,
|
||||
received_by_name TEXT,
|
||||
received_date TIMESTAMP WITH TIME ZONE,
|
||||
receiving_created_date TIMESTAMP WITH TIME ZONE,
|
||||
supplier_id INTEGER,
|
||||
status TEXT,
|
||||
PRIMARY KEY (receiving_id, pid)
|
||||
);
|
||||
|
||||
-- Temporary table for employee names
|
||||
CREATE TEMP TABLE employee_names (
|
||||
employeeid INTEGER PRIMARY KEY,
|
||||
firstname TEXT,
|
||||
lastname TEXT
|
||||
);
|
||||
|
||||
-- Create indexes for efficient joins
|
||||
CREATE INDEX idx_temp_po_pid ON temp_purchase_orders(pid);
|
||||
CREATE INDEX idx_temp_receiving_pid ON temp_receivings(pid);
|
||||
`);
|
||||
|
||||
// Map status codes to text values
|
||||
const poStatusMap = {
|
||||
0: 'canceled',
|
||||
1: 'created',
|
||||
10: 'electronically_ready_send',
|
||||
11: 'ordered',
|
||||
12: 'preordered',
|
||||
13: 'electronically_sent',
|
||||
15: 'receiving_started',
|
||||
50: 'done'
|
||||
};
|
||||
|
||||
const receivingStatusMap = {
|
||||
0: 'canceled',
|
||||
1: 'created',
|
||||
30: 'partial_received',
|
||||
40: 'full_received',
|
||||
50: 'paid'
|
||||
};
|
||||
|
||||
// Get time window for data retrieval
|
||||
const yearInterval = incrementalUpdate ? 1 : 5;
|
||||
|
||||
// Fetch employee data from production
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Fetching employee data"
|
||||
});
|
||||
|
||||
const [employees] = await prodConnection.query(`
|
||||
SELECT
|
||||
employeeid,
|
||||
firstname,
|
||||
lastname
|
||||
FROM employees
|
||||
`);
|
||||
|
||||
// Insert employee data into temp table
|
||||
if (employees.length > 0) {
|
||||
const employeeValues = employees.map(emp => [
|
||||
emp.employeeid,
|
||||
emp.firstname || '',
|
||||
emp.lastname || ''
|
||||
]).flat();
|
||||
|
||||
const placeholders = employees.map((_, idx) => {
|
||||
const base = idx * 3;
|
||||
return `($${base + 1}, $${base + 2}, $${base + 3})`;
|
||||
}).join(',');
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO employee_names (employeeid, firstname, lastname)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (employeeid) DO UPDATE SET
|
||||
firstname = EXCLUDED.firstname,
|
||||
lastname = EXCLUDED.lastname
|
||||
`, employeeValues);
|
||||
}
|
||||
|
||||
// Add this section before the PO import to create a supplier names mapping
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Fetching supplier data for vendor mapping"
|
||||
});
|
||||
|
||||
// Fetch supplier data from production and store in a temp table
|
||||
const [suppliers] = await prodConnection.query(`
|
||||
SELECT
|
||||
supplierid,
|
||||
companyname
|
||||
FROM suppliers
|
||||
WHERE companyname IS NOT NULL AND companyname != ''
|
||||
`);
|
||||
|
||||
if (suppliers.length > 0) {
|
||||
// Create temp table for supplier names
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS temp_supplier_names;
|
||||
CREATE TEMP TABLE temp_supplier_names (
|
||||
supplier_id INTEGER PRIMARY KEY,
|
||||
company_name TEXT NOT NULL
|
||||
);
|
||||
`);
|
||||
|
||||
// Insert supplier data in batches
|
||||
for (let i = 0; i < suppliers.length; i += INSERT_BATCH_SIZE) {
|
||||
const batch = suppliers.slice(i, i + INSERT_BATCH_SIZE);
|
||||
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 2;
|
||||
return `($${base + 1}, $${base + 2})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(s => [
|
||||
s.supplierid,
|
||||
s.companyname || 'Unnamed Supplier'
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_supplier_names (supplier_id, company_name)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (supplier_id) DO UPDATE SET
|
||||
company_name = EXCLUDED.company_name
|
||||
`, values);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Fetch and process purchase orders
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Fetching purchase orders"
|
||||
});
|
||||
|
||||
const [poCount] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM po p
|
||||
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
const totalPOs = poCount[0].total;
|
||||
console.log(`Found ${totalPOs} relevant purchase orders`);
|
||||
|
||||
// Skip processing if no POs to process
|
||||
if (totalPOs === 0) {
|
||||
console.log('No purchase orders to process, skipping PO import step');
|
||||
} else {
|
||||
// Fetch and process POs in batches
|
||||
let offset = 0;
|
||||
let allPOsProcessed = false;
|
||||
|
||||
while (!allPOsProcessed) {
|
||||
const [poList] = await prodConnection.query(`
|
||||
SELECT
|
||||
p.po_id,
|
||||
p.supplier_id,
|
||||
s.companyname AS vendor,
|
||||
p.status,
|
||||
p.notes AS long_note,
|
||||
p.short_note AS notes,
|
||||
p.date_created,
|
||||
p.date_ordered,
|
||||
p.date_estin
|
||||
FROM po p
|
||||
LEFT JOIN suppliers s ON p.supplier_id = s.supplierid
|
||||
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
)
|
||||
` : ''}
|
||||
ORDER BY p.po_id
|
||||
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
if (poList.length === 0) {
|
||||
allPOsProcessed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Get products for these POs
|
||||
const poIds = poList.map(po => po.po_id);
|
||||
|
||||
const [poProducts] = await prodConnection.query(`
|
||||
SELECT
|
||||
pp.po_id,
|
||||
pp.pid,
|
||||
pp.qty_each,
|
||||
pp.cost_each,
|
||||
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
||||
COALESCE(p.description, 'Unknown Product') AS name
|
||||
FROM po_products pp
|
||||
LEFT JOIN products p ON pp.pid = p.pid
|
||||
WHERE pp.po_id IN (?)
|
||||
`, [poIds]);
|
||||
|
||||
// Build complete PO records
|
||||
const completePOs = [];
|
||||
for (const product of poProducts) {
|
||||
const po = poList.find(p => p.po_id == product.po_id);
|
||||
if (!po) continue;
|
||||
|
||||
completePOs.push({
|
||||
po_id: po.po_id.toString(),
|
||||
pid: product.pid,
|
||||
sku: product.sku,
|
||||
name: product.name,
|
||||
vendor: po.vendor || 'Unknown Vendor',
|
||||
date: validateDate(po.date_ordered) || validateDate(po.date_created),
|
||||
expected_date: validateDate(po.date_estin),
|
||||
status: poStatusMap[po.status] || 'created',
|
||||
notes: po.notes || '',
|
||||
long_note: po.long_note || '',
|
||||
ordered: product.qty_each,
|
||||
po_cost_price: product.cost_each,
|
||||
supplier_id: po.supplier_id,
|
||||
date_created: validateDate(po.date_created),
|
||||
date_ordered: validateDate(po.date_ordered)
|
||||
});
|
||||
}
|
||||
|
||||
// Insert PO data in batches
|
||||
for (let i = 0; i < completePOs.length; i += INSERT_BATCH_SIZE) {
|
||||
const batch = completePOs.slice(i, i + INSERT_BATCH_SIZE);
|
||||
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 15;
|
||||
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(po => [
|
||||
po.po_id,
|
||||
po.pid,
|
||||
po.sku,
|
||||
po.name,
|
||||
po.vendor,
|
||||
po.date,
|
||||
po.expected_date,
|
||||
po.status,
|
||||
po.notes,
|
||||
po.long_note,
|
||||
po.ordered,
|
||||
po.po_cost_price,
|
||||
po.supplier_id,
|
||||
po.date_created,
|
||||
po.date_ordered
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_purchase_orders (
|
||||
po_id, pid, sku, name, vendor, date, expected_date, status, notes, long_note,
|
||||
ordered, po_cost_price, supplier_id, date_created, date_ordered
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (po_id, pid) DO UPDATE SET
|
||||
sku = EXCLUDED.sku,
|
||||
name = EXCLUDED.name,
|
||||
vendor = EXCLUDED.vendor,
|
||||
date = EXCLUDED.date,
|
||||
expected_date = EXCLUDED.expected_date,
|
||||
status = EXCLUDED.status,
|
||||
notes = EXCLUDED.notes,
|
||||
long_note = EXCLUDED.long_note,
|
||||
ordered = EXCLUDED.ordered,
|
||||
po_cost_price = EXCLUDED.po_cost_price,
|
||||
supplier_id = EXCLUDED.supplier_id,
|
||||
date_created = EXCLUDED.date_created,
|
||||
date_ordered = EXCLUDED.date_ordered
|
||||
`, values);
|
||||
}
|
||||
|
||||
offset += poList.length;
|
||||
totalProcessed += completePOs.length;
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: `Processed ${offset} of ${totalPOs} purchase orders (${totalProcessed} line items)`,
|
||||
current: offset,
|
||||
total: totalPOs,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, offset, totalPOs),
|
||||
rate: calculateRate(startTime, offset)
|
||||
});
|
||||
|
||||
if (poList.length < PO_BATCH_SIZE) {
|
||||
allPOsProcessed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Next, fetch all relevant receivings
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Fetching receivings data"
|
||||
});
|
||||
|
||||
const [receivingCount] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM receivings r
|
||||
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
r.date_updated > ?
|
||||
OR r.date_created > ?
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
const totalReceivings = receivingCount[0].total;
|
||||
console.log(`Found ${totalReceivings} relevant receivings`);
|
||||
|
||||
// Skip processing if no receivings to process
|
||||
if (totalReceivings === 0) {
|
||||
console.log('No receivings to process, skipping receivings import step');
|
||||
} else {
|
||||
// Fetch and process receivings in batches
|
||||
offset = 0; // Reset offset for receivings
|
||||
let allReceivingsProcessed = false;
|
||||
|
||||
while (!allReceivingsProcessed) {
|
||||
const [receivingList] = await prodConnection.query(`
|
||||
SELECT
|
||||
r.receiving_id,
|
||||
r.supplier_id,
|
||||
r.status,
|
||||
r.notes,
|
||||
r.shipping,
|
||||
r.total_amount,
|
||||
r.hold,
|
||||
r.for_storefront,
|
||||
r.date_created,
|
||||
r.date_paid,
|
||||
r.date_checked
|
||||
FROM receivings r
|
||||
WHERE r.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL ${yearInterval} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
r.date_updated > ?
|
||||
OR r.date_created > ?
|
||||
)
|
||||
` : ''}
|
||||
ORDER BY r.receiving_id
|
||||
LIMIT ${PO_BATCH_SIZE} OFFSET ${offset}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
if (receivingList.length === 0) {
|
||||
allReceivingsProcessed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Get products for these receivings
|
||||
const receivingIds = receivingList.map(r => r.receiving_id);
|
||||
|
||||
const [receivingProducts] = await prodConnection.query(`
|
||||
SELECT
|
||||
rp.receiving_id,
|
||||
rp.pid,
|
||||
rp.qty_each,
|
||||
rp.qty_each_orig,
|
||||
rp.cost_each,
|
||||
rp.cost_each_orig,
|
||||
rp.received_by,
|
||||
rp.received_date,
|
||||
r.date_created as receiving_created_date,
|
||||
COALESCE(p.itemnumber, 'NO-SKU') AS sku,
|
||||
COALESCE(p.description, 'Unknown Product') AS name
|
||||
FROM receivings_products rp
|
||||
JOIN receivings r ON rp.receiving_id = r.receiving_id
|
||||
LEFT JOIN products p ON rp.pid = p.pid
|
||||
WHERE rp.receiving_id IN (?)
|
||||
`, [receivingIds]);
|
||||
|
||||
// Build complete receiving records
|
||||
const completeReceivings = [];
|
||||
for (const product of receivingProducts) {
|
||||
const receiving = receivingList.find(r => r.receiving_id == product.receiving_id);
|
||||
if (!receiving) continue;
|
||||
|
||||
// Get employee name if available
|
||||
let receivedByName = null;
|
||||
if (product.received_by) {
|
||||
const [employeeResult] = await localConnection.query(`
|
||||
SELECT CONCAT(firstname, ' ', lastname) as full_name
|
||||
FROM employee_names
|
||||
WHERE employeeid = $1
|
||||
`, [product.received_by]);
|
||||
|
||||
if (employeeResult.rows.length > 0) {
|
||||
receivedByName = employeeResult.rows[0].full_name;
|
||||
}
|
||||
}
|
||||
|
||||
// Get vendor name if available
|
||||
let vendorName = 'Unknown Vendor';
|
||||
if (receiving.supplier_id) {
|
||||
const [vendorResult] = await localConnection.query(`
|
||||
SELECT company_name
|
||||
FROM temp_supplier_names
|
||||
WHERE supplier_id = $1
|
||||
`, [receiving.supplier_id]);
|
||||
|
||||
if (vendorResult.rows.length > 0) {
|
||||
vendorName = vendorResult.rows[0].company_name;
|
||||
}
|
||||
}
|
||||
|
||||
completeReceivings.push({
|
||||
receiving_id: receiving.receiving_id.toString(),
|
||||
pid: product.pid,
|
||||
sku: product.sku,
|
||||
name: product.name,
|
||||
vendor: vendorName,
|
||||
qty_each: product.qty_each,
|
||||
qty_each_orig: product.qty_each_orig,
|
||||
cost_each: product.cost_each,
|
||||
cost_each_orig: product.cost_each_orig,
|
||||
received_by: product.received_by,
|
||||
received_by_name: receivedByName,
|
||||
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
|
||||
receiving_created_date: validateDate(product.receiving_created_date),
|
||||
supplier_id: receiving.supplier_id,
|
||||
status: receivingStatusMap[receiving.status] || 'created'
|
||||
});
|
||||
}
|
||||
|
||||
// Insert receiving data in batches
|
||||
for (let i = 0; i < completeReceivings.length; i += INSERT_BATCH_SIZE) {
|
||||
const batch = completeReceivings.slice(i, i + INSERT_BATCH_SIZE);
|
||||
|
||||
const placeholders = batch.map((_, idx) => {
|
||||
const base = idx * 15;
|
||||
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
|
||||
}).join(',');
|
||||
|
||||
const values = batch.flatMap(r => [
|
||||
r.receiving_id,
|
||||
r.pid,
|
||||
r.sku,
|
||||
r.name,
|
||||
r.vendor,
|
||||
r.qty_each,
|
||||
r.qty_each_orig,
|
||||
r.cost_each,
|
||||
r.cost_each_orig,
|
||||
r.received_by,
|
||||
r.received_by_name,
|
||||
r.received_date,
|
||||
r.receiving_created_date,
|
||||
r.supplier_id,
|
||||
r.status
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_receivings (
|
||||
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
||||
cost_each, cost_each_orig, received_by, received_by_name,
|
||||
received_date, receiving_created_date, supplier_id, status
|
||||
)
|
||||
VALUES ${placeholders}
|
||||
ON CONFLICT (receiving_id, pid) DO UPDATE SET
|
||||
sku = EXCLUDED.sku,
|
||||
name = EXCLUDED.name,
|
||||
vendor = EXCLUDED.vendor,
|
||||
qty_each = EXCLUDED.qty_each,
|
||||
qty_each_orig = EXCLUDED.qty_each_orig,
|
||||
cost_each = EXCLUDED.cost_each,
|
||||
cost_each_orig = EXCLUDED.cost_each_orig,
|
||||
received_by = EXCLUDED.received_by,
|
||||
received_by_name = EXCLUDED.received_by_name,
|
||||
received_date = EXCLUDED.received_date,
|
||||
receiving_created_date = EXCLUDED.receiving_created_date,
|
||||
supplier_id = EXCLUDED.supplier_id,
|
||||
status = EXCLUDED.status
|
||||
`, values);
|
||||
}
|
||||
|
||||
offset += receivingList.length;
|
||||
totalProcessed += completeReceivings.length;
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: `Processed ${offset} of ${totalReceivings} receivings (${totalProcessed} line items total)`,
|
||||
current: offset,
|
||||
total: totalReceivings,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, offset, totalReceivings),
|
||||
rate: calculateRate(startTime, offset)
|
||||
});
|
||||
|
||||
if (receivingList.length < PO_BATCH_SIZE) {
|
||||
allReceivingsProcessed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add this section to filter out invalid PIDs before final import
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Validating product IDs before final import"
|
||||
});
|
||||
|
||||
await localConnection.query(`
|
||||
-- Create temp table to store invalid PIDs
|
||||
DROP TABLE IF EXISTS temp_invalid_pids;
|
||||
CREATE TEMP TABLE temp_invalid_pids AS (
|
||||
-- Get all unique PIDs from our temp tables
|
||||
WITH all_pids AS (
|
||||
SELECT DISTINCT pid FROM temp_purchase_orders
|
||||
UNION
|
||||
SELECT DISTINCT pid FROM temp_receivings
|
||||
)
|
||||
-- Filter to only those that don't exist in products table
|
||||
SELECT p.pid
|
||||
FROM all_pids p
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM products WHERE pid = p.pid
|
||||
)
|
||||
);
|
||||
|
||||
-- Remove purchase orders with invalid PIDs
|
||||
DELETE FROM temp_purchase_orders
|
||||
WHERE pid IN (SELECT pid FROM temp_invalid_pids);
|
||||
|
||||
-- Remove receivings with invalid PIDs
|
||||
DELETE FROM temp_receivings
|
||||
WHERE pid IN (SELECT pid FROM temp_invalid_pids);
|
||||
`);
|
||||
|
||||
// Get count of filtered items for reporting
|
||||
const [filteredResult] = await localConnection.query(`
|
||||
SELECT COUNT(*) as count FROM temp_invalid_pids
|
||||
`);
|
||||
const filteredCount = filteredResult.rows[0].count;
|
||||
|
||||
if (filteredCount > 0) {
|
||||
console.log(`Filtered out ${filteredCount} items with invalid product IDs`);
|
||||
}
|
||||
|
||||
// 3. Insert final purchase order records to the actual table
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Inserting final purchase order records"
|
||||
});
|
||||
|
||||
// Create a temp table to track PO IDs being processed
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS processed_po_ids;
|
||||
CREATE TEMP TABLE processed_po_ids AS (
|
||||
SELECT DISTINCT po_id FROM temp_purchase_orders
|
||||
);
|
||||
`);
|
||||
|
||||
// Delete products that were removed from POs and count them
|
||||
const [poDeletedResult] = await localConnection.query(`
|
||||
WITH deleted AS (
|
||||
DELETE FROM purchase_orders
|
||||
WHERE po_id IN (SELECT po_id FROM processed_po_ids)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM temp_purchase_orders tp
|
||||
WHERE purchase_orders.po_id = tp.po_id AND purchase_orders.pid = tp.pid
|
||||
)
|
||||
RETURNING po_id, pid
|
||||
)
|
||||
SELECT COUNT(*) as count FROM deleted
|
||||
`);
|
||||
|
||||
poRecordsDeleted = poDeletedResult.rows[0]?.count || 0;
|
||||
console.log(`Deleted ${poRecordsDeleted} products that were removed from purchase orders`);
|
||||
|
||||
const [poResult] = await localConnection.query(`
|
||||
INSERT INTO purchase_orders (
|
||||
po_id, vendor, date, expected_date, pid, sku, name,
|
||||
po_cost_price, status, notes, long_note,
|
||||
ordered, supplier_id, date_created, date_ordered
|
||||
)
|
||||
SELECT
|
||||
po_id,
|
||||
vendor,
|
||||
COALESCE(date, date_created, now()) as date,
|
||||
expected_date,
|
||||
pid,
|
||||
sku,
|
||||
name,
|
||||
po_cost_price,
|
||||
status,
|
||||
notes,
|
||||
long_note,
|
||||
ordered,
|
||||
supplier_id,
|
||||
date_created,
|
||||
date_ordered
|
||||
FROM temp_purchase_orders
|
||||
ON CONFLICT (po_id, pid) DO UPDATE SET
|
||||
vendor = EXCLUDED.vendor,
|
||||
date = EXCLUDED.date,
|
||||
expected_date = EXCLUDED.expected_date,
|
||||
sku = EXCLUDED.sku,
|
||||
name = EXCLUDED.name,
|
||||
po_cost_price = EXCLUDED.po_cost_price,
|
||||
status = EXCLUDED.status,
|
||||
notes = EXCLUDED.notes,
|
||||
long_note = EXCLUDED.long_note,
|
||||
ordered = EXCLUDED.ordered,
|
||||
supplier_id = EXCLUDED.supplier_id,
|
||||
date_created = EXCLUDED.date_created,
|
||||
date_ordered = EXCLUDED.date_ordered,
|
||||
updated = CURRENT_TIMESTAMP
|
||||
WHERE -- Only update if at least one key field has changed
|
||||
purchase_orders.ordered IS DISTINCT FROM EXCLUDED.ordered OR
|
||||
purchase_orders.po_cost_price IS DISTINCT FROM EXCLUDED.po_cost_price OR
|
||||
purchase_orders.status IS DISTINCT FROM EXCLUDED.status OR
|
||||
purchase_orders.expected_date IS DISTINCT FROM EXCLUDED.expected_date OR
|
||||
purchase_orders.date IS DISTINCT FROM EXCLUDED.date OR
|
||||
purchase_orders.vendor IS DISTINCT FROM EXCLUDED.vendor
|
||||
RETURNING (xmax = 0) as inserted
|
||||
`);
|
||||
|
||||
poRecordsAdded = poResult.rows.filter(r => r.inserted).length;
|
||||
poRecordsUpdated = poResult.rows.filter(r => !r.inserted).length;
|
||||
|
||||
// 4. Insert final receiving records to the actual table
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
message: "Inserting final receiving records"
|
||||
});
|
||||
|
||||
// Create a temp table to track receiving IDs being processed
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS processed_receiving_ids;
|
||||
CREATE TEMP TABLE processed_receiving_ids AS (
|
||||
SELECT DISTINCT receiving_id FROM temp_receivings
|
||||
);
|
||||
`);
|
||||
|
||||
// Delete products that were removed from receivings and count them
|
||||
const [receivingDeletedResult] = await localConnection.query(`
|
||||
WITH deleted AS (
|
||||
DELETE FROM receivings
|
||||
WHERE receiving_id IN (SELECT receiving_id FROM processed_receiving_ids)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM temp_receivings tr
|
||||
WHERE receivings.receiving_id = tr.receiving_id AND receivings.pid = tr.pid
|
||||
)
|
||||
RETURNING receiving_id, pid
|
||||
)
|
||||
SELECT COUNT(*) as count FROM deleted
|
||||
`);
|
||||
|
||||
receivingRecordsDeleted = receivingDeletedResult.rows[0]?.count || 0;
|
||||
console.log(`Deleted ${receivingRecordsDeleted} products that were removed from receivings`);
|
||||
|
||||
const [receivingsResult] = await localConnection.query(`
|
||||
INSERT INTO receivings (
|
||||
receiving_id, pid, sku, name, vendor, qty_each, qty_each_orig,
|
||||
cost_each, cost_each_orig, received_by, received_by_name,
|
||||
received_date, receiving_created_date, supplier_id, status
|
||||
)
|
||||
SELECT
|
||||
receiving_id,
|
||||
pid,
|
||||
sku,
|
||||
name,
|
||||
vendor,
|
||||
qty_each,
|
||||
qty_each_orig,
|
||||
cost_each,
|
||||
cost_each_orig,
|
||||
received_by,
|
||||
received_by_name,
|
||||
COALESCE(received_date, receiving_created_date, now()) as received_date,
|
||||
receiving_created_date,
|
||||
supplier_id,
|
||||
status
|
||||
FROM temp_receivings
|
||||
ON CONFLICT (receiving_id, pid) DO UPDATE SET
|
||||
sku = EXCLUDED.sku,
|
||||
name = EXCLUDED.name,
|
||||
vendor = EXCLUDED.vendor,
|
||||
qty_each = EXCLUDED.qty_each,
|
||||
qty_each_orig = EXCLUDED.qty_each_orig,
|
||||
cost_each = EXCLUDED.cost_each,
|
||||
cost_each_orig = EXCLUDED.cost_each_orig,
|
||||
received_by = EXCLUDED.received_by,
|
||||
received_by_name = EXCLUDED.received_by_name,
|
||||
received_date = EXCLUDED.received_date,
|
||||
receiving_created_date = EXCLUDED.receiving_created_date,
|
||||
supplier_id = EXCLUDED.supplier_id,
|
||||
status = EXCLUDED.status,
|
||||
updated = CURRENT_TIMESTAMP
|
||||
WHERE -- Only update if at least one key field has changed
|
||||
receivings.qty_each IS DISTINCT FROM EXCLUDED.qty_each OR
|
||||
receivings.cost_each IS DISTINCT FROM EXCLUDED.cost_each OR
|
||||
receivings.status IS DISTINCT FROM EXCLUDED.status OR
|
||||
receivings.received_date IS DISTINCT FROM EXCLUDED.received_date OR
|
||||
receivings.received_by IS DISTINCT FROM EXCLUDED.received_by
|
||||
RETURNING (xmax = 0) as inserted
|
||||
`);
|
||||
|
||||
receivingRecordsAdded = receivingsResult.rows.filter(r => r.inserted).length;
|
||||
receivingRecordsUpdated = receivingsResult.rows.filter(r => !r.inserted).length;
|
||||
|
||||
// Update sync status
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('purchase_orders', NOW())
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
// Clean up temporary tables
|
||||
await localConnection.query(`
|
||||
DROP TABLE IF EXISTS temp_purchase_orders;
|
||||
DROP TABLE IF EXISTS temp_receivings;
|
||||
DROP TABLE IF EXISTS employee_names;
|
||||
DROP TABLE IF EXISTS temp_supplier_names;
|
||||
DROP TABLE IF EXISTS temp_invalid_pids;
|
||||
DROP TABLE IF EXISTS processed_po_ids;
|
||||
DROP TABLE IF EXISTS processed_receiving_ids;
|
||||
`);
|
||||
|
||||
// Commit transaction
|
||||
await localConnection.commit();
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
recordsAdded: poRecordsAdded + receivingRecordsAdded,
|
||||
recordsUpdated: poRecordsUpdated + receivingRecordsUpdated,
|
||||
recordsDeleted: poRecordsDeleted + receivingRecordsDeleted,
|
||||
poRecordsAdded,
|
||||
poRecordsUpdated,
|
||||
poRecordsDeleted,
|
||||
receivingRecordsAdded,
|
||||
receivingRecordsUpdated,
|
||||
receivingRecordsDeleted,
|
||||
totalRecords: totalProcessed
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error during purchase orders import:", error);
|
||||
|
||||
// Rollback transaction
|
||||
try {
|
||||
await localConnection.rollback();
|
||||
} catch (rollbackError) {
|
||||
console.error('Error during rollback:', rollbackError.message);
|
||||
}
|
||||
|
||||
return {
|
||||
status: "error",
|
||||
error: error.message,
|
||||
recordsAdded: 0,
|
||||
recordsUpdated: 0,
|
||||
recordsDeleted: 0,
|
||||
totalRecords: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importPurchaseOrders;
|
||||
156
inventory-server/scripts/import/utils.js
Normal file
156
inventory-server/scripts/import/utils.js
Normal file
@@ -0,0 +1,156 @@
|
||||
const mysql = require("mysql2/promise");
|
||||
const { Client } = require("ssh2");
|
||||
const { Pool } = require('pg');
|
||||
const dotenv = require("dotenv");
|
||||
const path = require("path");
|
||||
|
||||
// Helper function to setup SSH tunnel
|
||||
async function setupSshTunnel(sshConfig) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ssh = new Client();
|
||||
|
||||
ssh.on('error', (err) => {
|
||||
console.error('SSH connection error:', err);
|
||||
});
|
||||
|
||||
ssh.on('end', () => {
|
||||
console.log('SSH connection ended normally');
|
||||
});
|
||||
|
||||
ssh.on('close', () => {
|
||||
console.log('SSH connection closed');
|
||||
});
|
||||
|
||||
ssh
|
||||
.on("ready", () => {
|
||||
ssh.forwardOut(
|
||||
"127.0.0.1",
|
||||
0,
|
||||
sshConfig.prodDbConfig.host,
|
||||
sshConfig.prodDbConfig.port,
|
||||
async (err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve({ ssh, stream });
|
||||
}
|
||||
);
|
||||
})
|
||||
.connect(sshConfig.ssh);
|
||||
});
|
||||
}
|
||||
|
||||
// Helper function to setup database connections
|
||||
async function setupConnections(sshConfig) {
|
||||
const tunnel = await setupSshTunnel(sshConfig);
|
||||
|
||||
// Setup MySQL connection for production
|
||||
const prodConnection = await mysql.createConnection({
|
||||
...sshConfig.prodDbConfig,
|
||||
stream: tunnel.stream,
|
||||
});
|
||||
|
||||
// Setup PostgreSQL connection pool for local
|
||||
const localPool = new Pool(sshConfig.localDbConfig);
|
||||
|
||||
// Test the PostgreSQL connection
|
||||
try {
|
||||
const client = await localPool.connect();
|
||||
await client.query('SELECT NOW()');
|
||||
client.release();
|
||||
console.log('PostgreSQL connection successful');
|
||||
} catch (err) {
|
||||
console.error('PostgreSQL connection error:', err);
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Create a wrapper for the PostgreSQL pool to match MySQL interface
|
||||
const localConnection = {
|
||||
_client: null,
|
||||
_transactionActive: false,
|
||||
|
||||
query: async (text, params) => {
|
||||
// If we're not in a transaction, use the pool directly
|
||||
if (!localConnection._transactionActive) {
|
||||
const client = await localPool.connect();
|
||||
try {
|
||||
const result = await client.query(text, params);
|
||||
return [result];
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
// If we're in a transaction, use the dedicated client
|
||||
if (!localConnection._client) {
|
||||
throw new Error('No active transaction client');
|
||||
}
|
||||
const result = await localConnection._client.query(text, params);
|
||||
return [result];
|
||||
},
|
||||
|
||||
beginTransaction: async () => {
|
||||
if (localConnection._transactionActive) {
|
||||
throw new Error('Transaction already active');
|
||||
}
|
||||
localConnection._client = await localPool.connect();
|
||||
await localConnection._client.query('BEGIN');
|
||||
localConnection._transactionActive = true;
|
||||
},
|
||||
|
||||
commit: async () => {
|
||||
if (!localConnection._transactionActive) {
|
||||
throw new Error('No active transaction to commit');
|
||||
}
|
||||
await localConnection._client.query('COMMIT');
|
||||
localConnection._client.release();
|
||||
localConnection._client = null;
|
||||
localConnection._transactionActive = false;
|
||||
},
|
||||
|
||||
rollback: async () => {
|
||||
if (!localConnection._transactionActive) {
|
||||
throw new Error('No active transaction to rollback');
|
||||
}
|
||||
await localConnection._client.query('ROLLBACK');
|
||||
localConnection._client.release();
|
||||
localConnection._client = null;
|
||||
localConnection._transactionActive = false;
|
||||
},
|
||||
|
||||
end: async () => {
|
||||
if (localConnection._client) {
|
||||
localConnection._client.release();
|
||||
localConnection._client = null;
|
||||
}
|
||||
await localPool.end();
|
||||
}
|
||||
};
|
||||
|
||||
return { prodConnection, localConnection, tunnel };
|
||||
}
|
||||
|
||||
// Helper function to close connections
|
||||
async function closeConnections(connections) {
|
||||
const { ssh, prodConnection, localConnection } = connections;
|
||||
|
||||
try {
|
||||
if (prodConnection) await prodConnection.end();
|
||||
if (localConnection) await localConnection.end();
|
||||
|
||||
// Wait a bit for any pending data to be written before closing SSH
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
if (ssh) {
|
||||
ssh.on('close', () => {
|
||||
console.log('SSH connection closed cleanly');
|
||||
});
|
||||
ssh.end();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error during cleanup:', err);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
setupConnections,
|
||||
closeConnections
|
||||
};
|
||||
Reference in New Issue
Block a user