Optimize metrics import and split off metrics import functions (untested)
This commit is contained in:
@@ -401,38 +401,75 @@ async function calculateVendorMetrics(connection) {
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to update product metrics
|
||||
async function updateProductMetrics(connection, productId, startTime, current, total) {
|
||||
// Helper function to calculate metrics in batches
|
||||
async function calculateMetricsInBatch(connection) {
|
||||
try {
|
||||
// Calculate sales velocity metrics
|
||||
const velocityMetrics = await calculateSalesVelocity(connection, productId);
|
||||
|
||||
// Calculate stock metrics
|
||||
const stockMetrics = await calculateStockMetrics(connection, productId, velocityMetrics.daily_sales_avg);
|
||||
|
||||
// Calculate financial metrics
|
||||
const financialMetrics = await calculateFinancialMetrics(connection, productId);
|
||||
|
||||
// Calculate purchase metrics
|
||||
const purchaseMetrics = await calculatePurchaseMetrics(connection, productId);
|
||||
|
||||
// Update metrics in database
|
||||
// Clear temporary tables
|
||||
await connection.query('TRUNCATE TABLE temp_sales_metrics');
|
||||
await connection.query('TRUNCATE TABLE temp_purchase_metrics');
|
||||
|
||||
// Calculate sales metrics for all products in one go
|
||||
await connection.query(`
|
||||
INSERT INTO temp_sales_metrics
|
||||
SELECT
|
||||
o.product_id,
|
||||
COUNT(*) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) as daily_sales_avg,
|
||||
SUM(o.quantity) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) * 7 as weekly_sales_avg,
|
||||
SUM(o.quantity) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) * 30 as monthly_sales_avg,
|
||||
SUM(o.price * o.quantity) as total_revenue,
|
||||
AVG((o.price - p.cost_price) / o.price * 100) as avg_margin_percent,
|
||||
MIN(o.date) as first_sale_date,
|
||||
MAX(o.date) as last_sale_date
|
||||
FROM orders o
|
||||
JOIN products p ON o.product_id = p.product_id
|
||||
WHERE o.canceled = false
|
||||
GROUP BY o.product_id
|
||||
`);
|
||||
|
||||
// Calculate purchase metrics for all products in one go
|
||||
await connection.query(`
|
||||
INSERT INTO temp_purchase_metrics
|
||||
SELECT
|
||||
product_id,
|
||||
AVG(DATEDIFF(received_date, date)) as avg_lead_time_days,
|
||||
MAX(date) as last_purchase_date,
|
||||
MAX(received_date) as last_received_date
|
||||
FROM purchase_orders
|
||||
WHERE status = 'closed'
|
||||
GROUP BY product_id
|
||||
`);
|
||||
|
||||
// Update product_metrics table with all metrics at once
|
||||
await connection.query(`
|
||||
INSERT INTO product_metrics (
|
||||
product_id,
|
||||
daily_sales_avg,
|
||||
weekly_sales_avg,
|
||||
monthly_sales_avg,
|
||||
days_of_inventory,
|
||||
weeks_of_inventory,
|
||||
safety_stock,
|
||||
reorder_point,
|
||||
total_revenue,
|
||||
avg_margin_percent,
|
||||
avg_lead_time_days,
|
||||
last_purchase_date,
|
||||
last_received_date
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
product_id, daily_sales_avg, weekly_sales_avg, monthly_sales_avg,
|
||||
days_of_inventory, weeks_of_inventory, safety_stock, reorder_point,
|
||||
avg_margin_percent, total_revenue, avg_lead_time_days,
|
||||
last_purchase_date, last_received_date
|
||||
)
|
||||
SELECT
|
||||
p.product_id,
|
||||
COALESCE(s.daily_sales_avg, 0),
|
||||
COALESCE(s.weekly_sales_avg, 0),
|
||||
COALESCE(s.monthly_sales_avg, 0),
|
||||
CASE
|
||||
WHEN s.daily_sales_avg > 0 THEN FLOOR(p.stock_quantity / s.daily_sales_avg)
|
||||
ELSE 999
|
||||
END as days_of_inventory,
|
||||
CASE
|
||||
WHEN s.daily_sales_avg > 0 THEN FLOOR(p.stock_quantity / s.daily_sales_avg / 7)
|
||||
ELSE 999
|
||||
END as weeks_of_inventory,
|
||||
CEIL(COALESCE(s.daily_sales_avg, 0) * 14) as safety_stock,
|
||||
CEIL(COALESCE(s.daily_sales_avg, 0) * 21) as reorder_point,
|
||||
COALESCE(s.avg_margin_percent, 0),
|
||||
COALESCE(s.total_revenue, 0),
|
||||
COALESCE(pm.avg_lead_time_days, 0),
|
||||
pm.last_purchase_date,
|
||||
pm.last_received_date
|
||||
FROM products p
|
||||
LEFT JOIN temp_sales_metrics s ON p.product_id = s.product_id
|
||||
LEFT JOIN temp_purchase_metrics pm ON p.product_id = pm.product_id
|
||||
ON DUPLICATE KEY UPDATE
|
||||
daily_sales_avg = VALUES(daily_sales_avg),
|
||||
weekly_sales_avg = VALUES(weekly_sales_avg),
|
||||
@@ -441,34 +478,37 @@ async function updateProductMetrics(connection, productId, startTime, current, t
|
||||
weeks_of_inventory = VALUES(weeks_of_inventory),
|
||||
safety_stock = VALUES(safety_stock),
|
||||
reorder_point = VALUES(reorder_point),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
avg_margin_percent = VALUES(avg_margin_percent),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
avg_lead_time_days = VALUES(avg_lead_time_days),
|
||||
last_purchase_date = VALUES(last_purchase_date),
|
||||
last_received_date = VALUES(last_received_date)
|
||||
`, [
|
||||
productId,
|
||||
velocityMetrics.daily_sales_avg,
|
||||
velocityMetrics.weekly_sales_avg,
|
||||
velocityMetrics.monthly_sales_avg,
|
||||
stockMetrics?.days_of_inventory || 0,
|
||||
stockMetrics?.weeks_of_inventory || 0,
|
||||
stockMetrics?.safety_stock || 0,
|
||||
stockMetrics?.reorder_point || 0,
|
||||
financialMetrics.total_revenue,
|
||||
financialMetrics.avg_margin_percent,
|
||||
purchaseMetrics.avg_lead_time_days,
|
||||
purchaseMetrics.last_purchase_date,
|
||||
purchaseMetrics.last_received_date
|
||||
]);
|
||||
last_received_date = VALUES(last_received_date),
|
||||
last_calculated_at = CURRENT_TIMESTAMP
|
||||
`);
|
||||
|
||||
// Calculate ABC classification in one go
|
||||
await connection.query(`
|
||||
WITH revenue_ranks AS (
|
||||
SELECT
|
||||
product_id,
|
||||
total_revenue,
|
||||
total_revenue / SUM(total_revenue) OVER () * 100 as revenue_percent,
|
||||
ROW_NUMBER() OVER (ORDER BY total_revenue DESC) as rank
|
||||
FROM product_metrics
|
||||
WHERE total_revenue > 0
|
||||
)
|
||||
UPDATE product_metrics pm
|
||||
JOIN revenue_ranks r ON pm.product_id = r.product_id
|
||||
SET abc_class =
|
||||
CASE
|
||||
WHEN r.revenue_percent >= 20 THEN 'A'
|
||||
WHEN r.revenue_percent >= 5 THEN 'B'
|
||||
ELSE 'C'
|
||||
END
|
||||
`);
|
||||
|
||||
// Output progress every 5 products or every second
|
||||
if (current % 5 === 0 || Date.now() - startTime > 1000) {
|
||||
updateProgress(current, total, 'Calculating product metrics', startTime);
|
||||
startTime = Date.now();
|
||||
}
|
||||
} catch (error) {
|
||||
logError(error, `Error updating metrics for product ${productId}`);
|
||||
logError(error, 'Error in batch metrics calculation');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -1051,43 +1091,8 @@ async function main() {
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
try {
|
||||
// Calculate product metrics
|
||||
const [products] = await connection.query('SELECT DISTINCT product_id FROM products');
|
||||
const totalProducts = products.length;
|
||||
let processedProducts = 0;
|
||||
const metricsStartTime = Date.now();
|
||||
|
||||
outputProgress({
|
||||
operation: 'Starting product metrics calculation',
|
||||
message: `Calculating metrics for ${totalProducts} products...`,
|
||||
current: 0,
|
||||
total: totalProducts,
|
||||
percentage: '0'
|
||||
});
|
||||
|
||||
for (const product of products) {
|
||||
try {
|
||||
// Update progress every 5 products or 1 second
|
||||
if (processedProducts % 5 === 0 || (Date.now() - lastUpdate) > 1000) {
|
||||
updateProgress(processedProducts, totalProducts, 'Calculating product metrics', metricsStartTime);
|
||||
lastUpdate = Date.now();
|
||||
}
|
||||
|
||||
await updateProductMetrics(connection, product.product_id, metricsStartTime, processedProducts, totalProducts);
|
||||
processedProducts++;
|
||||
} catch (error) {
|
||||
logError(error, `Error calculating metrics for product ${product.product_id}`);
|
||||
// Continue with next product instead of failing completely
|
||||
}
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Product metrics calculation completed',
|
||||
current: processedProducts,
|
||||
total: totalProducts,
|
||||
duration: formatDuration((Date.now() - metricsStartTime) / 1000),
|
||||
percentage: '100'
|
||||
});
|
||||
// Calculate metrics in batches
|
||||
await calculateMetricsInBatch(connection);
|
||||
|
||||
// Calculate vendor metrics
|
||||
await calculateVendorMetrics(connection);
|
||||
|
||||
Reference in New Issue
Block a user