matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make calculations incremental

Browse Source
This commit is contained in:
Matt
2025-02-09 13:35:44 -05:00
parent 2a6a0d0a87
commit 843ce71506
9 changed files with 935 additions and 1654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
inventory-server/db/metrics-schema.sql
View File

@@ -126,13 +126,13 @@ CREATE TABLE IF NOT EXISTS vendor_metrics (
order_fill_rate DECIMAL(5,2), order_fill_rate DECIMAL(5,2),
total_orders INT DEFAULT 0, total_orders INT DEFAULT 0,
total_late_orders INT DEFAULT 0, total_late_orders INT DEFAULT 0,
total_purchase_value DECIMAL(10,3) DEFAULT 0, total_purchase_value DECIMAL(15,3) DEFAULT 0,
avg_order_value DECIMAL(10,3), avg_order_value DECIMAL(15,3),
-- Product metrics -- Product metrics
active_products INT DEFAULT 0, active_products INT DEFAULT 0,
total_products INT DEFAULT 0, total_products INT DEFAULT 0,
-- Financial metrics -- Financial metrics
total_revenue DECIMAL(10,3) DEFAULT 0, total_revenue DECIMAL(15,3) DEFAULT 0,
avg_margin_percent DECIMAL(5,2), avg_margin_percent DECIMAL(5,2),
-- Status -- Status
status VARCHAR(20) DEFAULT 'active', status VARCHAR(20) DEFAULT 'active',

37
inventory-server/scripts/calculate-metrics.js
View File

@@ -104,17 +104,44 @@ async function calculateMetrics() {
WHERE status = 'running' WHERE status = 'running'
`); `);
// Get counts from all relevant tables // Get counts of records that need updating based on last calculation time
const [[productCount], [orderCount], [poCount]] = await Promise.all([ const [[productCount], [orderCount], [poCount]] = await Promise.all([
connection.query('SELECT COUNT(*) as total FROM products'), connection.query(`
connection.query('SELECT COUNT(*) as total FROM orders'), SELECT COUNT(*) as total
connection.query('SELECT COUNT(*) as total FROM purchase_orders') FROM products p
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
WHERE p.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
`),
connection.query(`
SELECT COUNT(*) as total
FROM orders o
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
WHERE o.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
AND o.canceled = false
`),
connection.query(`
SELECT COUNT(*) as total
FROM purchase_orders po
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
WHERE po.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
`)
]); ]);
totalProducts = productCount.total; totalProducts = productCount.total;
totalOrders = orderCount.total; totalOrders = orderCount.total;
totalPurchaseOrders = poCount.total; totalPurchaseOrders = poCount.total;
// If nothing needs updating, we can exit early
if (totalProducts === 0 && totalOrders === 0 && totalPurchaseOrders === 0) {
console.log('No records need updating');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
// Create history record for this calculation // Create history record for this calculation
const [historyResult] = await connection.query(` const [historyResult] = await connection.query(`
INSERT INTO calculate_history ( INSERT INTO calculate_history (
@@ -239,7 +266,7 @@ async function calculateMetrics() {
}); });
if (!SKIP_PRODUCT_METRICS) { if (!SKIP_PRODUCT_METRICS) {
const result = await calculateProductMetrics(startTime, totalProducts); const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders); await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
if (!result.success) { if (!result.success) {
throw new Error('Product metrics calculation failed'); throw new Error('Product metrics calculation failed');

410
inventory-server/scripts/metrics/brand-metrics.js
View File

@@ -4,19 +4,50 @@ const { getConnection } = require('./utils/db');
async function calculateBrandMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateBrandMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'brand_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of brands needing updates
const [brandCount] = await connection.query(`
SELECT COUNT(DISTINCT p.brand) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.brand IS NOT NULL
AND (
p.updated > ?
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalBrands = brandCount[0].count;
if (totalBrands === 0) {
console.log('No brands need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
operation: 'Brand metrics calculation cancelled', operation: 'Brand metrics calculation cancelled',
current: processedCount, current: processedCount,
total: totalProducts, total: totalBrands,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: null, remaining: null,
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalBrands) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -31,23 +62,15 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
}; };
} }
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting brand metrics calculation', operation: 'Starting brand metrics calculation',
current: processedCount, current: processedCount,
total: totalProducts, total: totalBrands,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts), remaining: estimateRemaining(startTime, processedCount, totalBrands),
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalBrands) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -55,237 +78,144 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
} }
}); });
// Calculate brand metrics with optimized queries // Process in batches
await connection.query(` let lastBrand = '';
INSERT INTO brand_metrics ( while (true) {
brand, if (isCancelled) break;
product_count,
active_products, const [batch] = await connection.query(`
total_stock_units, SELECT DISTINCT p.brand
total_stock_cost,
total_stock_retail,
total_revenue,
avg_margin,
growth_rate
)
WITH filtered_products AS (
SELECT
p.*,
CASE
WHEN p.stock_quantity <= 5000 AND p.stock_quantity >= 0
THEN p.pid
END as valid_pid,
CASE
WHEN p.visible = true
AND p.stock_quantity <= 5000
AND p.stock_quantity >= 0
THEN p.pid
END as active_pid,
CASE
WHEN p.stock_quantity IS NULL
OR p.stock_quantity < 0
OR p.stock_quantity > 5000
THEN 0
ELSE p.stock_quantity
END as valid_stock
FROM products p FROM products p
WHERE p.brand IS NOT NULL WHERE p.brand IS NOT NULL
), AND p.brand > ?
sales_periods AS ( AND (
SELECT p.updated > ?
p.brand, OR EXISTS (
SUM(o.quantity * (o.price - COALESCE(o.discount, 0))) as period_revenue, SELECT 1 FROM orders o
SUM(o.quantity * (o.price - COALESCE(o.discount, 0) - p.cost_price)) as period_margin, WHERE o.pid = p.pid
COUNT(DISTINCT DATE(o.date)) as period_days, AND o.updated > ?
CASE )
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH) THEN 'current' )
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH) ORDER BY p.brand
AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) THEN 'previous' LIMIT ?
END as period_type `, [lastBrand, lastCalculationTime, lastCalculationTime, BATCH_SIZE]);
FROM filtered_products p
JOIN orders o ON p.pid = o.pid if (batch.length === 0) break;
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH) // Update brand metrics for this batch
GROUP BY p.brand, period_type await connection.query(`
), INSERT INTO brand_metrics (
brand_data AS ( brand,
SELECT product_count,
p.brand, active_products,
COUNT(DISTINCT p.valid_pid) as product_count, total_stock_units,
COUNT(DISTINCT p.active_pid) as active_products, total_stock_cost,
SUM(p.valid_stock) as total_stock_units, total_stock_retail,
SUM(p.valid_stock * p.cost_price) as total_stock_cost, total_revenue,
SUM(p.valid_stock * p.price) as total_stock_retail, avg_margin,
COALESCE(SUM(o.quantity * (o.price - COALESCE(o.discount, 0))), 0) as total_revenue, growth_rate,
CASE last_calculated_at
WHEN SUM(o.quantity * o.price) > 0 )
THEN GREATEST( WITH product_stats AS (
-100.0, SELECT
LEAST( p.brand,
100.0, COUNT(DISTINCT p.pid) as product_count,
( COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts) SUM(p.stock_quantity) as total_stock_units,
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs SUM(p.stock_quantity * p.cost_price) as total_stock_cost,
) * 100.0 / SUM(p.stock_quantity * p.price) as total_stock_retail,
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue SUM(pm.total_revenue) as total_revenue,
AVG(pm.avg_margin_percent) as avg_margin
FROM products p
LEFT JOIN product_metrics pm ON p.pid = pm.pid
WHERE p.brand IN (?)
AND (
p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o
WHERE o.pid = p.pid
AND o.updated > ?
) )
) )
ELSE 0 GROUP BY p.brand
END as avg_margin ),
FROM filtered_products p sales_periods AS (
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false SELECT
GROUP BY p.brand p.brand,
) SUM(CASE
SELECT WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
bd.brand, THEN o.quantity * o.price
bd.product_count, ELSE 0
bd.active_products, END) as current_period_sales,
bd.total_stock_units, SUM(CASE
bd.total_stock_cost, WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY) AND DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
bd.total_stock_retail, THEN o.quantity * o.price
bd.total_revenue, ELSE 0
bd.avg_margin, END) as previous_period_sales
CASE FROM products p
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0 INNER JOIN orders o ON p.pid = o.pid
AND MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) > 0 AND o.canceled = false
THEN 100.0 AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY)
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0 AND o.updated > ?
THEN 0.0 WHERE p.brand IN (?)
ELSE GREATEST( GROUP BY p.brand
-100.0, )
LEAST(
((MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) -
MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END)) /
NULLIF(ABS(MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END)), 0)) * 100.0,
999.99
)
)
END as growth_rate
FROM brand_data bd
LEFT JOIN sales_periods sp ON bd.brand = sp.brand
GROUP BY bd.brand, bd.product_count, bd.active_products, bd.total_stock_units,
bd.total_stock_cost, bd.total_stock_retail, bd.total_revenue, bd.avg_margin
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_stock_units = VALUES(total_stock_units),
total_stock_cost = VALUES(total_stock_cost),
total_stock_retail = VALUES(total_stock_retail),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin),
growth_rate = VALUES(growth_rate),
last_calculated_at = CURRENT_TIMESTAMP
`);
processedCount = Math.floor(totalProducts * 0.97);
outputProgress({
status: 'running',
operation: 'Brand metrics calculated, starting time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate brand time-based metrics with optimized query
await connection.query(`
INSERT INTO brand_time_metrics (
brand,
year,
month,
product_count,
active_products,
total_stock_units,
total_stock_cost,
total_stock_retail,
total_revenue,
avg_margin
)
WITH filtered_products AS (
SELECT SELECT
p.*, ps.brand,
CASE WHEN p.stock_quantity <= 5000 THEN p.pid END as valid_pid, COALESCE(ps.product_count, 0) as product_count,
CASE WHEN p.visible = true AND p.stock_quantity <= 5000 THEN p.pid END as active_pid, COALESCE(ps.active_products, 0) as active_products,
CASE COALESCE(ps.total_stock_units, 0) as total_stock_units,
WHEN p.stock_quantity IS NULL OR p.stock_quantity < 0 OR p.stock_quantity > 5000 THEN 0 COALESCE(ps.total_stock_cost, 0) as total_stock_cost,
ELSE p.stock_quantity COALESCE(ps.total_stock_retail, 0) as total_stock_retail,
END as valid_stock COALESCE(ps.total_revenue, 0) as total_revenue,
FROM products p COALESCE(ps.avg_margin, 0) as avg_margin,
WHERE p.brand IS NOT NULL CASE
), WHEN COALESCE(sp.previous_period_sales, 0) = 0 AND COALESCE(sp.current_period_sales, 0) > 0 THEN 100
monthly_metrics AS ( WHEN COALESCE(sp.previous_period_sales, 0) = 0 THEN 0
SELECT ELSE LEAST(999.99, GREATEST(-100,
p.brand, ((COALESCE(sp.current_period_sales, 0) / sp.previous_period_sales) - 1) * 100
YEAR(o.date) as year, ))
MONTH(o.date) as month, END as growth_rate,
COUNT(DISTINCT p.valid_pid) as product_count, NOW() as last_calculated_at
COUNT(DISTINCT p.active_pid) as active_products, FROM product_stats ps
SUM(p.valid_stock) as total_stock_units, LEFT JOIN sales_periods sp ON ps.brand = sp.brand
SUM(p.valid_stock * p.cost_price) as total_stock_cost, ON DUPLICATE KEY UPDATE
SUM(p.valid_stock * p.price) as total_stock_retail, product_count = VALUES(product_count),
SUM(o.quantity * o.price) as total_revenue, active_products = VALUES(active_products),
CASE total_stock_units = VALUES(total_stock_units),
WHEN SUM(o.quantity * o.price) > 0 total_stock_cost = VALUES(total_stock_cost),
THEN GREATEST( total_stock_retail = VALUES(total_stock_retail),
-100.0, total_revenue = VALUES(total_revenue),
LEAST( avg_margin = VALUES(avg_margin),
100.0, growth_rate = VALUES(growth_rate),
( last_calculated_at = NOW()
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts) `, [
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs batch.map(row => row.brand), // For first IN clause
) * 100.0 / lastCalculationTime, // For p.updated > ?
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue lastCalculationTime, // For o.updated > ? in EXISTS
) lastCalculationTime, // For o.updated > ? in sales_periods
) batch.map(row => row.brand) // For second IN clause
ELSE 0 ]);
END as avg_margin
FROM filtered_products p
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
WHERE o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY p.brand, YEAR(o.date), MONTH(o.date)
)
SELECT *
FROM monthly_metrics
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_stock_units = VALUES(total_stock_units),
total_stock_cost = VALUES(total_stock_cost),
total_stock_retail = VALUES(total_stock_retail),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin)
`);
processedCount = Math.floor(totalProducts * 0.99); lastBrand = batch[batch.length - 1].brand;
outputProgress({ processedCount += batch.length;
status: 'running',
operation: 'Brand time-based metrics calculated', outputProgress({
current: processedCount, status: 'running',
total: totalProducts, operation: 'Processing brand metrics batch',
elapsed: formatElapsedTime(startTime), current: processedCount,
remaining: estimateRemaining(startTime, processedCount, totalProducts), total: totalBrands,
rate: calculateRate(startTime, processedCount), elapsed: formatElapsedTime(startTime),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), remaining: estimateRemaining(startTime, processedCount, totalBrands),
timing: { rate: calculateRate(startTime, processedCount),
start_time: new Date(startTime).toISOString(), percentage: ((processedCount / totalBrands) * 100).toFixed(1),
end_time: new Date().toISOString(), timing: {
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) start_time: new Date(startTime).toISOString(),
} end_time: new Date().toISOString(),
}); elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -299,7 +229,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders: 0, processedPurchaseOrders: 0,
success success
}; };

573
inventory-server/scripts/metrics/category-metrics.js
View File

@@ -4,19 +4,52 @@ const { getConnection } = require('./utils/db');
async function calculateCategoryMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateCategoryMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'category_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of categories needing updates
const [categoryCount] = await connection.query(`
SELECT COUNT(DISTINCT c.cat_id) as count
FROM categories c
JOIN product_categories pc ON c.cat_id = pc.cat_id
LEFT JOIN products p ON pc.pid = p.pid AND p.updated > ?
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE c.status = 'active'
AND (
p.pid IS NOT NULL
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalCategories = categoryCount[0].count;
if (totalCategories === 0) {
console.log('No categories need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
operation: 'Category metrics calculation cancelled', operation: 'Category metrics calculation cancelled',
current: processedCount, current: processedCount,
total: totalProducts, total: totalCategories,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: null, remaining: null,
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalCategories) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -31,69 +64,15 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
}; };
} }
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting category metrics calculation', operation: 'Starting category metrics calculation',
current: processedCount, current: processedCount,
total: totalProducts, total: totalCategories,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts), remaining: estimateRemaining(startTime, processedCount, totalCategories),
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalCategories) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// First, calculate base category metrics
await connection.query(`
INSERT INTO category_metrics (
category_id,
product_count,
active_products,
total_value,
status,
last_calculated_at
)
SELECT
c.cat_id,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
COALESCE(SUM(p.stock_quantity * p.cost_price), 0) as total_value,
c.status,
NOW() as last_calculated_at
FROM categories c
LEFT JOIN product_categories pc ON c.cat_id = pc.cat_id
LEFT JOIN products p ON pc.pid = p.pid
GROUP BY c.cat_id, c.status
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
status = VALUES(status),
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 0.90);
outputProgress({
status: 'running',
operation: 'Base category metrics calculated, updating with margin data',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -101,395 +80,99 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
} }
}); });
if (isCancelled) return { // Process in batches
processedProducts: processedCount, let lastCatId = 0;
processedOrders, while (true) {
processedPurchaseOrders: 0, if (isCancelled) break;
success
};
// Then update with margin and turnover data const [batch] = await connection.query(`
await connection.query(` SELECT DISTINCT c.cat_id
WITH category_sales AS ( FROM categories c
SELECT JOIN product_categories pc ON c.cat_id = pc.cat_id
pc.cat_id, LEFT JOIN products p ON pc.pid = p.pid AND p.updated > ?
SUM(o.quantity * o.price) as total_sales, LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
SUM(o.quantity * (o.price - p.cost_price)) as total_margin, WHERE c.status = 'active'
SUM(o.quantity) as units_sold, AND c.cat_id > ?
AVG(GREATEST(p.stock_quantity, 0)) as avg_stock, AND (
COUNT(DISTINCT DATE(o.date)) as active_days p.pid IS NOT NULL
FROM product_categories pc OR o.id IS NOT NULL
JOIN products p ON pc.pid = p.pid )
JOIN orders o ON p.pid = o.pid ORDER BY c.cat_id
LEFT JOIN turnover_config tc ON LIMIT ?
(tc.category_id = pc.cat_id AND tc.vendor = p.vendor) OR `, [lastCalculationTime, lastCalculationTime, lastCatId, BATCH_SIZE]);
(tc.category_id = pc.cat_id AND tc.vendor IS NULL) OR
(tc.category_id IS NULL AND tc.vendor = p.vendor) OR
(tc.category_id IS NULL AND tc.vendor IS NULL)
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL COALESCE(tc.calculation_period_days, 30) DAY)
GROUP BY pc.cat_id
)
UPDATE category_metrics cm
JOIN category_sales cs ON cm.category_id = cs.cat_id
LEFT JOIN turnover_config tc ON
(tc.category_id = cm.category_id AND tc.vendor IS NULL) OR
(tc.category_id IS NULL AND tc.vendor IS NULL)
SET
cm.avg_margin = COALESCE(cs.total_margin * 100.0 / NULLIF(cs.total_sales, 0), 0),
cm.turnover_rate = CASE
WHEN cs.avg_stock > 0 AND cs.active_days > 0
THEN LEAST(
(cs.units_sold / cs.avg_stock) * (365.0 / cs.active_days),
999.99
)
ELSE 0
END,
cm.last_calculated_at = NOW()
`);
processedCount = Math.floor(totalProducts * 0.95); if (batch.length === 0) break;
outputProgress({
status: 'running',
operation: 'Margin data updated, calculating growth rates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return { // Update category metrics for this batch
processedProducts: processedCount, await connection.query(`
processedOrders, INSERT INTO category_metrics (
processedPurchaseOrders: 0, category_id,
success product_count,
}; active_products,
total_value,
avg_margin,
turnover_rate,
growth_rate,
status,
last_calculated_at
)
SELECT
c.cat_id,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(p.stock_quantity * p.cost_price) as total_value,
AVG(pm.avg_margin_percent) as avg_margin,
AVG(pm.turnover_rate) as turnover_rate,
((SUM(CASE
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END) / NULLIF(SUM(CASE
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY) AND DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END), 0) - 1) * 100) as growth_rate,
c.status,
NOW() as last_calculated_at
FROM categories c
JOIN product_categories pc ON c.cat_id = pc.cat_id
LEFT JOIN products p ON pc.pid = p.pid
LEFT JOIN product_metrics pm ON p.pid = pm.pid
LEFT JOIN orders o ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY)
WHERE c.cat_id IN (?)
GROUP BY c.cat_id, c.status
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
avg_margin = VALUES(avg_margin),
turnover_rate = VALUES(turnover_rate),
growth_rate = VALUES(growth_rate),
status = VALUES(status),
last_calculated_at = NOW()
`, [batch.map(row => row.cat_id)]);
// Finally update growth rates lastCatId = batch[batch.length - 1].cat_id;
await connection.query(` processedCount += batch.length;
WITH current_period AS (
SELECT
pc.cat_id,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0) - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as days
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH)
GROUP BY pc.cat_id
),
previous_period AS (
SELECT
pc.cat_id,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
COUNT(DISTINCT DATE(o.date)) as days
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY pc.cat_id
),
trend_data AS (
SELECT
pc.cat_id,
MONTH(o.date) as month,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
COUNT(DISTINCT DATE(o.date)) as days_in_month
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
GROUP BY pc.cat_id, MONTH(o.date)
),
trend_stats AS (
SELECT
cat_id,
COUNT(*) as n,
AVG(month) as avg_x,
AVG(revenue / NULLIF(days_in_month, 0)) as avg_y,
SUM(month * (revenue / NULLIF(days_in_month, 0))) as sum_xy,
SUM(month * month) as sum_xx
FROM trend_data
GROUP BY cat_id
HAVING COUNT(*) >= 6
),
trend_analysis AS (
SELECT
cat_id,
((n * sum_xy) - (avg_x * n * avg_y)) /
NULLIF((n * sum_xx) - (n * avg_x * avg_x), 0) as trend_slope,
avg_y as avg_daily_revenue
FROM trend_stats
),
margin_calc AS (
SELECT
pc.cat_id,
CASE
WHEN SUM(o.quantity * o.price) > 0 THEN
GREATEST(
-100.0,
LEAST(
100.0,
(
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts)
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs
) * 100.0 /
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue
)
)
ELSE NULL
END as avg_margin
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH)
GROUP BY pc.cat_id
)
UPDATE category_metrics cm
LEFT JOIN current_period cp ON cm.category_id = cp.cat_id
LEFT JOIN previous_period pp ON cm.category_id = pp.cat_id
LEFT JOIN trend_analysis ta ON cm.category_id = ta.cat_id
LEFT JOIN margin_calc mc ON cm.category_id = mc.cat_id
SET
cm.growth_rate = CASE
WHEN pp.revenue = 0 AND COALESCE(cp.revenue, 0) > 0 THEN 100.0
WHEN pp.revenue = 0 OR cp.revenue IS NULL THEN 0.0
WHEN ta.trend_slope IS NOT NULL THEN
GREATEST(
-100.0,
LEAST(
(ta.trend_slope / NULLIF(ta.avg_daily_revenue, 0)) * 365 * 100,
999.99
)
)
ELSE
GREATEST(
-100.0,
LEAST(
((COALESCE(cp.revenue, 0) - pp.revenue) /
NULLIF(ABS(pp.revenue), 0)) * 100.0,
999.99
)
)
END,
cm.avg_margin = COALESCE(mc.avg_margin, cm.avg_margin),
cm.last_calculated_at = NOW()
WHERE cp.cat_id IS NOT NULL OR pp.cat_id IS NOT NULL
`);
processedCount = Math.floor(totalProducts * 0.97); outputProgress({
outputProgress({ status: 'running',
status: 'running', operation: 'Processing category metrics batch',
operation: 'Growth rates calculated, updating time-based metrics', current: processedCount,
current: processedCount, total: totalCategories,
total: totalProducts, elapsed: formatElapsedTime(startTime),
elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalCategories),
remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount),
rate: calculateRate(startTime, processedCount), percentage: ((processedCount / totalCategories) * 100).toFixed(1),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), timing: {
timing: { start_time: new Date(startTime).toISOString(),
start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(),
end_time: new Date().toISOString(), elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) }
} });
}); }
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate time-based metrics
await connection.query(`
INSERT INTO category_time_metrics (
category_id,
year,
month,
product_count,
active_products,
total_value,
total_revenue,
avg_margin,
turnover_rate
)
SELECT
pc.cat_id,
YEAR(o.date) as year,
MONTH(o.date) as month,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(p.stock_quantity * p.cost_price) as total_value,
SUM(o.quantity * o.price) as total_revenue,
CASE
WHEN SUM(o.quantity * o.price) > 0 THEN
LEAST(
GREATEST(
SUM(o.quantity * (o.price - GREATEST(p.cost_price, 0))) * 100.0 /
SUM(o.quantity * o.price),
-100
),
100
)
ELSE 0
END as avg_margin,
COALESCE(
LEAST(
SUM(o.quantity) / NULLIF(AVG(GREATEST(p.stock_quantity, 0)), 0),
999.99
),
0
) as turnover_rate
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY pc.cat_id, YEAR(o.date), MONTH(o.date)
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin),
turnover_rate = VALUES(turnover_rate)
`);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Time-based metrics calculated, updating category-sales metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate category-sales metrics
await connection.query(`
INSERT INTO category_sales_metrics (
category_id,
brand,
period_start,
period_end,
avg_daily_sales,
total_sold,
num_products,
avg_price,
last_calculated_at
)
WITH date_ranges AS (
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY) as period_start,
CURRENT_DATE as period_end
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 31 DAY)
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 180 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 91 DAY)
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 365 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 181 DAY)
),
sales_data AS (
SELECT
pc.cat_id,
COALESCE(p.brand, 'Unknown') as brand,
dr.period_start,
dr.period_end,
COUNT(DISTINCT p.pid) as num_products,
SUM(o.quantity) as total_sold,
SUM(o.quantity * o.price) as total_revenue,
COUNT(DISTINCT DATE(o.date)) as num_days
FROM products p
JOIN product_categories pc ON p.pid = pc.pid
JOIN orders o ON p.pid = o.pid
CROSS JOIN date_ranges dr
WHERE o.canceled = false
AND o.date BETWEEN dr.period_start AND dr.period_end
GROUP BY pc.cat_id, p.brand, dr.period_start, dr.period_end
)
SELECT
cat_id as category_id,
brand,
period_start,
period_end,
CASE
WHEN num_days > 0
THEN total_sold / num_days
ELSE 0
END as avg_daily_sales,
total_sold,
num_products,
CASE
WHEN total_sold > 0
THEN total_revenue / total_sold
ELSE 0
END as avg_price,
NOW() as last_calculated_at
FROM sales_data
ON DUPLICATE KEY UPDATE
avg_daily_sales = VALUES(avg_daily_sales),
total_sold = VALUES(total_sold),
num_products = VALUES(num_products),
avg_price = VALUES(avg_price),
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 1.0);
outputProgress({
status: 'running',
operation: 'Category-sales metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -503,7 +186,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders: 0, processedPurchaseOrders: 0,
success success
}; };

207
inventory-server/scripts/metrics/financial-metrics.js
View File

@@ -4,9 +4,39 @@ const { getConnection } = require('./utils/db');
async function calculateFinancialMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateFinancialMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'financial_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need financial metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
@@ -31,15 +61,6 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
}; };
} }
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH)
`);
processedOrders = orderCount[0].count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting financial metrics calculation', operation: 'Starting financial metrics calculation',
@@ -56,110 +77,76 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
} }
}); });
// Calculate financial metrics with optimized query // Process in batches
await connection.query(` let lastPid = 0;
WITH product_financials AS ( while (true) {
SELECT if (isCancelled) break;
p.pid,
p.cost_price * p.stock_quantity as inventory_value, const [batch] = await connection.query(`
SUM(o.quantity * o.price) as total_revenue, SELECT DISTINCT p.pid
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
MIN(o.date) as first_sale_date,
MAX(o.date) as last_sale_date,
DATEDIFF(MAX(o.date), MIN(o.date)) + 1 as calculation_period_days,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM products p FROM products p
LEFT JOIN orders o ON p.pid = o.pid LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE o.canceled = false WHERE p.pid > ?
AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH) AND (
GROUP BY p.pid p.updated > ?
) OR o.pid IS NOT NULL
UPDATE product_metrics pm )
JOIN product_financials pf ON pm.pid = pf.pid ORDER BY p.pid
SET LIMIT ?
pm.inventory_value = COALESCE(pf.inventory_value, 0), `, [lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
pm.total_revenue = COALESCE(pf.total_revenue, 0),
pm.cost_of_goods_sold = COALESCE(pf.cost_of_goods_sold, 0),
pm.gross_profit = COALESCE(pf.gross_profit, 0),
pm.gmroi = CASE
WHEN COALESCE(pf.inventory_value, 0) > 0 AND pf.active_days > 0 THEN
(COALESCE(pf.gross_profit, 0) * (365.0 / pf.active_days)) / COALESCE(pf.inventory_value, 0)
ELSE 0
END,
pm.last_calculated_at = CURRENT_TIMESTAMP
`);
processedCount = Math.floor(totalProducts * 0.65); if (batch.length === 0) break;
outputProgress({
status: 'running',
operation: 'Base financial metrics calculated, updating time aggregates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return { // Update financial metrics for this batch
processedProducts: processedCount, await connection.query(`
processedOrders, UPDATE product_metrics pm
processedPurchaseOrders: 0, JOIN (
success SELECT
}; p.pid,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
WHERE p.pid IN (?)
GROUP BY p.pid
) fin ON pm.pid = fin.pid
SET
pm.inventory_value = COALESCE(fin.inventory_value, 0),
pm.total_revenue = COALESCE(fin.total_revenue, 0),
pm.cost_of_goods_sold = COALESCE(fin.cost_of_goods_sold, 0),
pm.gross_profit = COALESCE(fin.gross_profit, 0),
pm.gmroi = CASE
WHEN COALESCE(fin.inventory_value, 0) > 0 AND fin.active_days > 0
THEN (COALESCE(fin.gross_profit, 0) * (365.0 / fin.active_days)) / COALESCE(fin.inventory_value, 0)
ELSE 0
END,
pm.last_calculated_at = NOW()
`, [batch.map(row => row.pid)]);
// Update time-based aggregates with optimized query lastPid = batch[batch.length - 1].pid;
await connection.query(` processedCount += batch.length;
WITH monthly_financials AS (
SELECT
p.pid,
YEAR(o.date) as year,
MONTH(o.date) as month,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as active_days,
MIN(o.date) as period_start,
MAX(o.date) as period_end
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
)
UPDATE product_time_aggregates pta
JOIN monthly_financials mf ON pta.pid = mf.pid
AND pta.year = mf.year
AND pta.month = mf.month
SET
pta.inventory_value = COALESCE(mf.inventory_value, 0),
pta.gmroi = CASE
WHEN COALESCE(mf.inventory_value, 0) > 0 AND mf.active_days > 0 THEN
(COALESCE(mf.gross_profit, 0) * (365.0 / mf.active_days)) / COALESCE(mf.inventory_value, 0)
ELSE 0
END
`);
processedCount = Math.floor(totalProducts * 0.70); outputProgress({
outputProgress({ status: 'running',
status: 'running', operation: 'Processing financial metrics batch',
operation: 'Time-based aggregates updated', current: processedCount,
current: processedCount, total: totalProducts,
total: totalProducts, elapsed: formatElapsedTime(startTime),
elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProducts),
remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount),
rate: calculateRate(startTime, processedCount), percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), timing: {
timing: { start_time: new Date(startTime).toISOString(),
start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(),
end_time: new Date().toISOString(), elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) }
} });
}); }
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -173,7 +160,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders: 0, processedPurchaseOrders: 0,
success success
}; };

73
inventory-server/scripts/metrics/product-metrics.js
View File

@@ -16,16 +16,42 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
const BATCH_SIZE = 5000; const BATCH_SIZE = 5000;
try { try {
// Skip flags are inherited from the parent scope // Get last calculation timestamp
const SKIP_PRODUCT_BASE_METRICS = 0; const [lastCalc] = await connection.query(`
const SKIP_PRODUCT_TIME_AGGREGATES = 0; SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'product_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total product count if not provided // Get total product count if not provided
if (!totalProducts) { if (!totalProducts) {
const [productCount] = await connection.query('SELECT COUNT(*) as count FROM products'); const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count; totalProducts = productCount[0].count;
} }
if (totalProducts === 0) {
console.log('No products need updating');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
// Skip flags are inherited from the parent scope
const SKIP_PRODUCT_BASE_METRICS = 0;
const SKIP_PRODUCT_TIME_AGGREGATES = 0;
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
@@ -116,8 +142,15 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
LEFT JOIN orders o ON p.pid = o.pid LEFT JOIN orders o ON p.pid = o.pid
AND o.canceled = false AND o.canceled = false
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY) AND o.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY)
WHERE p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o2
WHERE o2.pid = p.pid
AND o2.canceled = false
AND o2.updated > ?
)
GROUP BY p.pid GROUP BY p.pid
`); `, [lastCalculationTime, lastCalculationTime]);
// Populate temp_purchase_metrics // Populate temp_purchase_metrics
await connection.query(` await connection.query(`
@@ -132,18 +165,34 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
LEFT JOIN purchase_orders po ON p.pid = po.pid LEFT JOIN purchase_orders po ON p.pid = po.pid
AND po.received_date IS NOT NULL AND po.received_date IS NOT NULL
AND po.date >= DATE_SUB(CURDATE(), INTERVAL 365 DAY) AND po.date >= DATE_SUB(CURDATE(), INTERVAL 365 DAY)
WHERE p.updated > ?
OR EXISTS (
SELECT 1 FROM purchase_orders po2
WHERE po2.pid = p.pid
AND po2.updated > ?
)
GROUP BY p.pid GROUP BY p.pid
`); `, [lastCalculationTime, lastCalculationTime]);
// Process updates in batches // Process updates in batches, but only for affected products
let lastPid = 0; let lastPid = 0;
while (true) { while (true) {
if (isCancelled) break; if (isCancelled) break;
const [batch] = await connection.query( const [batch] = await connection.query(`
'SELECT pid FROM products WHERE pid > ? ORDER BY pid LIMIT ?', SELECT DISTINCT p.pid
[lastPid, BATCH_SIZE] FROM products p
); LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ?
WHERE p.pid > ?
AND (
p.updated > ?
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
)
ORDER BY p.pid
LIMIT ?
`, [lastCalculationTime, lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
if (batch.length === 0) break; if (batch.length === 0) break;
@@ -532,7 +581,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
// Update calculate_status // Update calculate_status with current timestamp
await connection.query(` await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp) INSERT INTO calculate_status (module_name, last_calculation_timestamp)
VALUES ('product_metrics', NOW()) VALUES ('product_metrics', NOW())

540
inventory-server/scripts/metrics/sales-forecasts.js
View File

@@ -4,19 +4,50 @@ const { getConnection } = require('./utils/db');
async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'sales_forecasts'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.visible = true
AND (
p.updated > ?
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalProductsToUpdate = productCount[0].count;
if (totalProductsToUpdate === 0) {
console.log('No products need forecast updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
operation: 'Sales forecasts calculation cancelled', operation: 'Sales forecast calculation cancelled',
current: processedCount, current: processedCount,
total: totalProducts, total: totalProductsToUpdate,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: null, remaining: null,
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -31,24 +62,15 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
}; };
} }
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
`);
processedOrders = orderCount[0].count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting sales forecasts calculation', operation: 'Starting sales forecast calculation',
current: processedCount, current: processedCount,
total: totalProducts, total: totalProductsToUpdate,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts), remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -56,365 +78,141 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
} }
}); });
// First, create a temporary table for forecast dates // Process in batches
await connection.query(` let lastPid = '';
CREATE TEMPORARY TABLE IF NOT EXISTS temp_forecast_dates ( while (true) {
forecast_date DATE, if (isCancelled) break;
day_of_week INT,
month INT,
PRIMARY KEY (forecast_date)
)
`);
await connection.query(` const [batch] = await connection.query(`
INSERT INTO temp_forecast_dates SELECT DISTINCT p.pid
SELECT FROM products p
DATE_ADD(CURRENT_DATE, INTERVAL n DAY) as forecast_date, LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
DAYOFWEEK(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as day_of_week, WHERE p.visible = true
MONTH(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as month AND p.pid > ?
FROM ( AND (
SELECT a.N + b.N * 10 as n p.updated > ?
FROM OR o.id IS NOT NULL
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 4 UNION )
SELECT 5 UNION SELECT 6 UNION SELECT 7 UNION SELECT 8 UNION SELECT 9) a, ORDER BY p.pid
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2) b LIMIT ?
ORDER BY n `, [lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
LIMIT 31
) numbers
`);
processedCount = Math.floor(totalProducts * 0.92); if (batch.length === 0) break;
outputProgress({
status: 'running',
operation: 'Forecast dates prepared, calculating daily sales stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return { // Calculate forecasts for this batch
processedProducts: processedCount, await connection.query(`
processedOrders, INSERT INTO sales_forecasts (
processedPurchaseOrders: 0, pid,
success forecast_date,
}; forecast_units,
forecast_revenue,
// Create temporary table for daily sales stats confidence_level,
await connection.query(` last_calculated_at
CREATE TEMPORARY TABLE IF NOT EXISTS temp_daily_sales AS )
SELECT WITH historical_sales AS (
o.pid, SELECT
DAYOFWEEK(o.date) as day_of_week, o.pid,
SUM(o.quantity) as daily_quantity, DATE(o.date) as sale_date,
SUM(o.price * o.quantity) as daily_revenue, SUM(o.quantity) as daily_quantity,
COUNT(DISTINCT DATE(o.date)) as day_count SUM(o.quantity * o.price) as daily_revenue
FROM orders o FROM orders o
WHERE o.canceled = false WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY) AND o.pid IN (?)
GROUP BY o.pid, DAYOFWEEK(o.date) AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 180 DAY)
`); GROUP BY o.pid, DATE(o.date)
),
processedCount = Math.floor(totalProducts * 0.94); sales_stats AS (
outputProgress({ SELECT
status: 'running', pid,
operation: 'Daily sales stats calculated, preparing product stats', AVG(daily_quantity) as avg_daily_units,
current: processedCount, AVG(daily_revenue) as avg_daily_revenue,
total: totalProducts, STDDEV(daily_quantity) as std_daily_units,
elapsed: formatElapsedTime(startTime), COUNT(*) as days_with_sales,
remaining: estimateRemaining(startTime, processedCount, totalProducts), MIN(sale_date) as first_sale,
rate: calculateRate(startTime, processedCount), MAX(sale_date) as last_sale
percentage: ((processedCount / totalProducts) * 100).toFixed(1), FROM historical_sales
timing: { GROUP BY pid
start_time: new Date(startTime).toISOString(), ),
end_time: new Date().toISOString(), recent_trend AS (
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) SELECT
} h.pid,
}); AVG(h.daily_quantity) as recent_avg_units,
AVG(h.daily_revenue) as recent_avg_revenue
if (isCancelled) return { FROM historical_sales h
processedProducts: processedCount, WHERE h.sale_date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
processedOrders, GROUP BY h.pid
processedPurchaseOrders: 0, ),
success confidence_calc AS (
}; SELECT
s.pid,
// Create temporary table for product stats LEAST(100, GREATEST(0, ROUND(
await connection.query(` (s.days_with_sales / 180.0 * 50) + -- Up to 50 points for history length
CREATE TEMPORARY TABLE IF NOT EXISTS temp_product_stats AS (CASE
SELECT WHEN s.std_daily_units = 0 OR s.avg_daily_units = 0 THEN 0
pid, WHEN (s.std_daily_units / s.avg_daily_units) <= 0.5 THEN 30
AVG(daily_revenue) as overall_avg_revenue, WHEN (s.std_daily_units / s.avg_daily_units) <= 1.0 THEN 20
SUM(day_count) as total_days WHEN (s.std_daily_units / s.avg_daily_units) <= 2.0 THEN 10
FROM temp_daily_sales ELSE 0
GROUP BY pid END) + -- Up to 30 points for consistency
`); (CASE
WHEN DATEDIFF(CURRENT_DATE, s.last_sale) <= 7 THEN 20
processedCount = Math.floor(totalProducts * 0.96); WHEN DATEDIFF(CURRENT_DATE, s.last_sale) <= 30 THEN 10
outputProgress({ ELSE 0
status: 'running', END) -- Up to 20 points for recency
operation: 'Product stats prepared, calculating product-level forecasts', ))) as confidence_level
current: processedCount, FROM sales_stats s
total: totalProducts, )
elapsed: formatElapsedTime(startTime), (SELECT
remaining: estimateRemaining(startTime, processedCount, totalProducts), s.pid,
rate: calculateRate(startTime, processedCount), DATE_ADD(CURRENT_DATE, INTERVAL n.days DAY) as forecast_date,
percentage: ((processedCount / totalProducts) * 100).toFixed(1), GREATEST(0, ROUND(
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate product-level forecasts
await connection.query(`
INSERT INTO sales_forecasts (
pid,
forecast_date,
forecast_units,
forecast_revenue,
confidence_level,
last_calculated_at
)
WITH daily_stats AS (
SELECT
ds.pid,
AVG(ds.daily_quantity) as avg_daily_qty,
STDDEV(ds.daily_quantity) as std_daily_qty,
COUNT(DISTINCT ds.day_count) as data_points,
SUM(ds.day_count) as total_days,
AVG(ds.daily_revenue) as avg_daily_revenue,
STDDEV(ds.daily_revenue) as std_daily_revenue,
MIN(ds.daily_quantity) as min_daily_qty,
MAX(ds.daily_quantity) as max_daily_qty,
-- Calculate variance without using LAG
COALESCE(
STDDEV(ds.daily_quantity) / NULLIF(AVG(ds.daily_quantity), 0),
0
) as daily_variance_ratio
FROM temp_daily_sales ds
GROUP BY ds.pid
HAVING AVG(ds.daily_quantity) > 0
)
SELECT
ds.pid,
fd.forecast_date,
GREATEST(0,
ROUND(
ds.avg_daily_qty *
(1 + COALESCE(sf.seasonality_factor, 0)) *
CASE
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 1.5 THEN 0.85
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 1.0 THEN 0.9
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 0.5 THEN 0.95
ELSE 1.0
END,
2
)
) as forecast_units,
GREATEST(0,
ROUND(
COALESCE(
CASE
WHEN ds.data_points >= 4 THEN ds.avg_daily_revenue
ELSE ps.overall_avg_revenue
END *
(1 + COALESCE(sf.seasonality_factor, 0)) *
CASE
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 1.5 THEN 0.85
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 1.0 THEN 0.9
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 0.5 THEN 0.95
ELSE 1.0
END,
0
),
2
)
) as forecast_revenue,
CASE
WHEN ds.total_days >= 60 AND ds.daily_variance_ratio < 0.5 THEN 90
WHEN ds.total_days >= 60 THEN 85
WHEN ds.total_days >= 30 AND ds.daily_variance_ratio < 0.5 THEN 80
WHEN ds.total_days >= 30 THEN 75
WHEN ds.total_days >= 14 AND ds.daily_variance_ratio < 0.5 THEN 70
WHEN ds.total_days >= 14 THEN 65
ELSE 60
END as confidence_level,
NOW() as last_calculated_at
FROM daily_stats ds
JOIN temp_product_stats ps ON ds.pid = ps.pid
CROSS JOIN temp_forecast_dates fd
LEFT JOIN sales_seasonality sf ON fd.month = sf.month
GROUP BY ds.pid, fd.forecast_date, ps.overall_avg_revenue, sf.seasonality_factor
ON DUPLICATE KEY UPDATE
forecast_units = VALUES(forecast_units),
forecast_revenue = VALUES(forecast_revenue),
confidence_level = VALUES(confidence_level),
last_calculated_at = NOW()
`);
processedCount = Math.floor(totalProducts * 0.98);
outputProgress({
status: 'running',
operation: 'Product forecasts calculated, preparing category stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Create temporary table for category stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_sales AS
SELECT
pc.cat_id,
DAYOFWEEK(o.date) as day_of_week,
SUM(o.quantity) as daily_quantity,
SUM(o.price * o.quantity) as daily_revenue,
COUNT(DISTINCT DATE(o.date)) as day_count
FROM orders o
JOIN product_categories pc ON o.pid = pc.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
GROUP BY pc.cat_id, DAYOFWEEK(o.date)
`);
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_stats AS
SELECT
cat_id,
AVG(daily_revenue) as overall_avg_revenue,
SUM(day_count) as total_days
FROM temp_category_sales
GROUP BY cat_id
`);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Category stats prepared, calculating category-level forecasts',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate category-level forecasts
await connection.query(`
INSERT INTO category_forecasts (
category_id,
forecast_date,
forecast_units,
forecast_revenue,
confidence_level,
last_calculated_at
)
SELECT
cs.cat_id as category_id,
fd.forecast_date,
GREATEST(0,
AVG(cs.daily_quantity) *
(1 + COALESCE(sf.seasonality_factor, 0))
) as forecast_units,
GREATEST(0,
COALESCE(
CASE CASE
WHEN SUM(cs.day_count) >= 4 THEN AVG(cs.daily_revenue) WHEN s.days_with_sales >= n.days THEN
ELSE ct.overall_avg_revenue COALESCE(t.recent_avg_units, s.avg_daily_units)
END * ELSE s.avg_daily_units * (s.days_with_sales / n.days)
(1 + COALESCE(sf.seasonality_factor, 0)) * END
(0.95 + (RAND() * 0.1)), )) as forecast_units,
0 GREATEST(0, ROUND(
) CASE
) as forecast_revenue, WHEN s.days_with_sales >= n.days THEN
CASE COALESCE(t.recent_avg_revenue, s.avg_daily_revenue)
WHEN ct.total_days >= 60 THEN 90 ELSE s.avg_daily_revenue * (s.days_with_sales / n.days)
WHEN ct.total_days >= 30 THEN 80 END
WHEN ct.total_days >= 14 THEN 70 , 2)) as forecast_revenue,
ELSE 60 c.confidence_level,
END as confidence_level, NOW() as last_calculated_at
NOW() as last_calculated_at FROM sales_stats s
FROM temp_category_sales cs CROSS JOIN (
JOIN temp_category_stats ct ON cs.cat_id = ct.cat_id SELECT 30 as days UNION SELECT 60 UNION SELECT 90
CROSS JOIN temp_forecast_dates fd ) n
LEFT JOIN sales_seasonality sf ON fd.month = sf.month LEFT JOIN recent_trend t ON s.pid = t.pid
GROUP BY cs.cat_id, fd.forecast_date, ct.overall_avg_revenue, ct.total_days, sf.seasonality_factor LEFT JOIN confidence_calc c ON s.pid = c.pid)
HAVING AVG(cs.daily_quantity) > 0 ON DUPLICATE KEY UPDATE
ON DUPLICATE KEY UPDATE forecast_units = VALUES(forecast_units),
forecast_units = VALUES(forecast_units), forecast_revenue = VALUES(forecast_revenue),
forecast_revenue = VALUES(forecast_revenue), confidence_level = VALUES(confidence_level),
confidence_level = VALUES(confidence_level), last_calculated_at = NOW()
last_calculated_at = NOW() `, [batch.map(row => row.pid)]);
`);
// Clean up temporary tables lastPid = batch[batch.length - 1].pid;
await connection.query(` processedCount += batch.length;
DROP TEMPORARY TABLE IF EXISTS temp_forecast_dates;
DROP TEMPORARY TABLE IF EXISTS temp_daily_sales;
DROP TEMPORARY TABLE IF EXISTS temp_product_stats;
DROP TEMPORARY TABLE IF EXISTS temp_category_sales;
DROP TEMPORARY TABLE IF EXISTS temp_category_stats;
`);
processedCount = Math.floor(totalProducts * 1.0); outputProgress({
outputProgress({ status: 'running',
status: 'running', operation: 'Processing sales forecast batch',
operation: 'Category forecasts calculated and temporary tables cleaned up', current: processedCount,
current: processedCount, total: totalProductsToUpdate,
total: totalProducts, elapsed: formatElapsedTime(startTime),
elapsed: formatElapsedTime(startTime), remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
remaining: estimateRemaining(startTime, processedCount, totalProducts), rate: calculateRate(startTime, processedCount),
rate: calculateRate(startTime, processedCount), percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), timing: {
timing: { start_time: new Date(startTime).toISOString(),
start_time: new Date(startTime).toISOString(), end_time: new Date().toISOString(),
end_time: new Date().toISOString(), elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) }
} });
}); }
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -428,7 +226,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders: 0, processedPurchaseOrders: 0,
success success
}; };

290
inventory-server/scripts/metrics/time-aggregates.js
View File

@@ -4,9 +4,39 @@ const { getConnection } = require('./utils/db');
async function calculateTimeAggregates(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateTimeAggregates(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'time_aggregates'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need time aggregate updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
@@ -31,14 +61,6 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
}; };
} }
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting time aggregates calculation', operation: 'Starting time aggregates calculation',
@@ -55,180 +77,98 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
} }
}); });
// Initial insert of time-based aggregates // Process in batches
await connection.query(` let lastPid = 0;
INSERT INTO product_time_aggregates ( while (true) {
pid, if (isCancelled) break;
year,
month, const [batch] = await connection.query(`
total_quantity_sold, SELECT DISTINCT p.pid
total_revenue, FROM products p
total_cost, LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
order_count, WHERE p.pid > ?
stock_received, AND (
stock_ordered, p.updated > ?
avg_price, OR o.id IS NOT NULL
profit_margin, )
inventory_value, ORDER BY p.pid
gmroi LIMIT ?
) `, [lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
WITH monthly_sales AS (
SELECT if (batch.length === 0) break;
o.pid,
YEAR(o.date) as year, // Calculate and update time aggregates for this batch
MONTH(o.date) as month, await connection.query(`
SUM(o.quantity) as total_quantity_sold, INSERT INTO product_time_aggregates (
SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) as total_revenue,
SUM(COALESCE(p.cost_price, 0) * o.quantity) as total_cost,
COUNT(DISTINCT o.order_number) as order_count,
AVG(o.price - COALESCE(o.discount, 0)) as avg_price,
CASE
WHEN SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) > 0
THEN ((SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) - SUM(COALESCE(p.cost_price, 0) * o.quantity))
/ SUM((o.price - COALESCE(o.discount, 0)) * o.quantity)) * 100
ELSE 0
END as profit_margin,
p.cost_price * p.stock_quantity as inventory_value,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM orders o
JOIN products p ON o.pid = p.pid
WHERE o.canceled = false
GROUP BY o.pid, YEAR(o.date), MONTH(o.date)
),
monthly_stock AS (
SELECT
pid, pid,
YEAR(date) as year, year,
MONTH(date) as month, month,
SUM(received) as stock_received, total_quantity_sold,
SUM(ordered) as stock_ordered total_revenue,
FROM purchase_orders total_cost,
GROUP BY pid, YEAR(date), MONTH(date) order_count,
) avg_price,
SELECT profit_margin,
s.pid, inventory_value,
s.year, gmroi
s.month, )
s.total_quantity_sold,
s.total_revenue,
s.total_cost,
s.order_count,
COALESCE(ms.stock_received, 0) as stock_received,
COALESCE(ms.stock_ordered, 0) as stock_ordered,
s.avg_price,
s.profit_margin,
s.inventory_value,
CASE
WHEN s.inventory_value > 0 THEN
(s.total_revenue - s.total_cost) / s.inventory_value
ELSE 0
END as gmroi
FROM monthly_sales s
LEFT JOIN monthly_stock ms
ON s.pid = ms.pid
AND s.year = ms.year
AND s.month = ms.month
UNION
SELECT
p.pid,
p.year,
p.month,
0 as total_quantity_sold,
0 as total_revenue,
0 as total_cost,
0 as order_count,
p.stock_received,
p.stock_ordered,
0 as avg_price,
0 as profit_margin,
(SELECT cost_price * stock_quantity FROM products WHERE pid = p.pid) as inventory_value,
0 as gmroi
FROM monthly_stock p
LEFT JOIN monthly_sales s
ON p.pid = s.pid
AND p.year = s.year
AND p.month = s.month
WHERE s.pid IS NULL
ON DUPLICATE KEY UPDATE
total_quantity_sold = VALUES(total_quantity_sold),
total_revenue = VALUES(total_revenue),
total_cost = VALUES(total_cost),
order_count = VALUES(order_count),
stock_received = VALUES(stock_received),
stock_ordered = VALUES(stock_ordered),
avg_price = VALUES(avg_price),
profit_margin = VALUES(profit_margin),
inventory_value = VALUES(inventory_value),
gmroi = VALUES(gmroi)
`);
processedCount = Math.floor(totalProducts * 0.60);
outputProgress({
status: 'running',
operation: 'Base time aggregates calculated, updating financial metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Update with financial metrics
await connection.query(`
UPDATE product_time_aggregates pta
JOIN (
SELECT SELECT
p.pid, p.pid,
YEAR(o.date) as year, YEAR(o.date) as year,
MONTH(o.date) as month, MONTH(o.date) as month,
SUM(o.quantity) as total_quantity_sold,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * p.cost_price) as total_cost,
COUNT(DISTINCT o.order_number) as order_count,
AVG(o.price) as avg_price,
CASE
WHEN SUM(o.quantity * o.price) > 0
THEN ((SUM(o.quantity * o.price) - SUM(o.quantity * p.cost_price)) / SUM(o.quantity * o.price)) * 100
ELSE 0
END as profit_margin,
p.cost_price * p.stock_quantity as inventory_value, p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit, CASE
COUNT(DISTINCT DATE(o.date)) as days_in_period WHEN p.cost_price * p.stock_quantity > 0
THEN (SUM(o.quantity * (o.price - p.cost_price))) / (p.cost_price * p.stock_quantity)
ELSE 0
END as gmroi
FROM products p FROM products p
LEFT JOIN orders o ON p.pid = o.pid INNER JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
WHERE p.pid IN (?)
GROUP BY p.pid, YEAR(o.date), MONTH(o.date) GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
) fin ON pta.pid = fin.pid HAVING year IS NOT NULL AND month IS NOT NULL
AND pta.year = fin.year ON DUPLICATE KEY UPDATE
AND pta.month = fin.month total_quantity_sold = VALUES(total_quantity_sold),
SET total_revenue = VALUES(total_revenue),
pta.inventory_value = COALESCE(fin.inventory_value, 0), total_cost = VALUES(total_cost),
pta.gmroi = CASE order_count = VALUES(order_count),
WHEN COALESCE(fin.inventory_value, 0) > 0 AND fin.days_in_period > 0 THEN avg_price = VALUES(avg_price),
(COALESCE(fin.gross_profit, 0) * (365.0 / fin.days_in_period)) / COALESCE(fin.inventory_value, 0) profit_margin = VALUES(profit_margin),
ELSE 0 inventory_value = VALUES(inventory_value),
END gmroi = VALUES(gmroi)
`); `, [batch.map(row => row.pid)]);
processedCount = Math.floor(totalProducts * 0.65); lastPid = batch[batch.length - 1].pid;
outputProgress({ processedCount += batch.length;
status: 'running',
operation: 'Financial metrics updated', outputProgress({
current: processedCount, status: 'running',
total: totalProducts, operation: 'Processing time aggregates batch',
elapsed: formatElapsedTime(startTime), current: processedCount,
remaining: estimateRemaining(startTime, processedCount, totalProducts), total: totalProducts,
rate: calculateRate(startTime, processedCount), elapsed: formatElapsedTime(startTime),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), remaining: estimateRemaining(startTime, processedCount, totalProducts),
timing: { rate: calculateRate(startTime, processedCount),
start_time: new Date(startTime).toISOString(), percentage: ((processedCount / totalProducts) * 100).toFixed(1),
end_time: new Date().toISOString(), timing: {
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) start_time: new Date(startTime).toISOString(),
} end_time: new Date().toISOString(),
}); elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -242,7 +182,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders: 0, processedPurchaseOrders: 0,
success success
}; };

453
inventory-server/scripts/metrics/vendor-metrics.js
View File

@@ -4,20 +4,51 @@ const { getConnection } = require('./utils/db');
async function calculateVendorMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) { async function calculateVendorMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection(); const connection = await getConnection();
let success = false; let success = false;
let processedOrders = 0; const BATCH_SIZE = 5000;
let processedPurchaseOrders = 0;
try { try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'vendor_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of vendors needing updates
const [vendorCount] = await connection.query(`
SELECT COUNT(DISTINCT v.vendor) as count
FROM vendor_details v
LEFT JOIN products p ON v.vendor = p.vendor AND p.updated > ?
LEFT JOIN purchase_orders po ON v.vendor = po.vendor AND po.updated > ?
WHERE v.status = 'active'
AND (
p.pid IS NOT NULL
OR po.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalVendors = vendorCount[0].count;
if (totalVendors === 0) {
console.log('No vendors need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) { if (isCancelled) {
outputProgress({ outputProgress({
status: 'cancelled', status: 'cancelled',
operation: 'Vendor metrics calculation cancelled', operation: 'Vendor metrics calculation cancelled',
current: processedCount, current: processedCount,
total: totalProducts, total: totalVendors,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: null, remaining: null,
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalVendors) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -26,37 +57,21 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
}); });
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders, processedPurchaseOrders: 0,
success success
}; };
} }
// Get counts of records that will be processed
const [[orderCount], [poCount]] = await Promise.all([
connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`),
connection.query(`
SELECT COUNT(*) as count
FROM purchase_orders po
WHERE po.status != 0
`)
]);
processedOrders = orderCount.count;
processedPurchaseOrders = poCount.count;
outputProgress({ outputProgress({
status: 'running', status: 'running',
operation: 'Starting vendor metrics calculation', operation: 'Starting vendor metrics calculation',
current: processedCount, current: processedCount,
total: totalProducts, total: totalVendors,
elapsed: formatElapsedTime(startTime), elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts), remaining: estimateRemaining(startTime, processedCount, totalVendors),
rate: calculateRate(startTime, processedCount), rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), percentage: ((processedCount / totalVendors) * 100).toFixed(1),
timing: { timing: {
start_time: new Date(startTime).toISOString(), start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(), end_time: new Date().toISOString(),
@@ -64,278 +79,130 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
} }
}); });
// First ensure all vendors exist in vendor_details // Process in batches
await connection.query(` let lastVendor = '';
INSERT IGNORE INTO vendor_details (vendor, status, created_at, updated_at) while (true) {
SELECT DISTINCT if (isCancelled) break;
vendor,
'active' as status,
NOW() as created_at,
NOW() as updated_at
FROM products
WHERE vendor IS NOT NULL
`);
processedCount = Math.floor(totalProducts * 0.8); const [batch] = await connection.query(`
outputProgress({ SELECT DISTINCT v.vendor
status: 'running', FROM vendor_details v
operation: 'Vendor details updated, calculating metrics', LEFT JOIN products p ON v.vendor = p.vendor AND p.updated > ?
current: processedCount, LEFT JOIN purchase_orders po ON v.vendor = po.vendor AND po.updated > ?
total: totalProducts, WHERE v.status = 'active'
elapsed: formatElapsedTime(startTime), AND v.vendor > ?
remaining: estimateRemaining(startTime, processedCount, totalProducts), AND (
rate: calculateRate(startTime, processedCount), p.pid IS NOT NULL
percentage: ((processedCount / totalProducts) * 100).toFixed(1), OR po.id IS NOT NULL
timing: { )
start_time: new Date(startTime).toISOString(), ORDER BY v.vendor
end_time: new Date().toISOString(), LIMIT ?
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) `, [lastCalculationTime, lastCalculationTime, lastVendor, BATCH_SIZE]);
}
});
if (isCancelled) return { if (batch.length === 0) break;
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
success
};
// Now calculate vendor metrics // Update vendor metrics for this batch
await connection.query(` await connection.query(`
INSERT INTO vendor_metrics ( INSERT INTO vendor_metrics (
vendor,
total_revenue,
total_orders,
total_late_orders,
avg_lead_time_days,
on_time_delivery_rate,
order_fill_rate,
avg_order_value,
active_products,
total_products,
total_purchase_value,
avg_margin_percent,
status,
last_calculated_at
)
WITH vendor_sales AS (
SELECT
p.vendor,
SUM(o.quantity * o.price) as total_revenue,
COUNT(DISTINCT o.id) as total_orders,
COUNT(DISTINCT p.pid) as active_products,
SUM(o.quantity * (o.price - p.cost_price)) as total_margin
FROM products p
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY p.vendor
),
vendor_po AS (
SELECT
p.vendor,
COUNT(DISTINCT CASE WHEN po.receiving_status = 40 THEN po.id END) as received_orders,
COUNT(DISTINCT po.id) as total_orders,
AVG(CASE
WHEN po.receiving_status = 40
THEN DATEDIFF(po.received_date, po.date)
END) as avg_lead_time_days,
SUM(po.ordered * po.po_cost_price) as total_purchase_value
FROM products p
JOIN purchase_orders po ON p.pid = po.pid
WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY p.vendor
),
vendor_products AS (
SELECT
vendor, vendor,
COUNT(DISTINCT pid) as total_products avg_lead_time_days,
FROM products on_time_delivery_rate,
GROUP BY vendor order_fill_rate,
) total_orders,
SELECT total_late_orders,
vs.vendor, total_purchase_value,
COALESCE(vs.total_revenue, 0) as total_revenue, avg_order_value,
COALESCE(vp.total_orders, 0) as total_orders, active_products,
COALESCE(vp.total_orders - vp.received_orders, 0) as total_late_orders, total_products,
COALESCE(vp.avg_lead_time_days, 0) as avg_lead_time_days, total_revenue,
CASE avg_margin_percent,
WHEN vp.total_orders > 0 status,
THEN (vp.received_orders / vp.total_orders) * 100 last_calculated_at
ELSE 0 )
END as on_time_delivery_rate, WITH purchase_stats AS (
CASE SELECT
WHEN vp.total_orders > 0 po.vendor,
THEN (vp.received_orders / vp.total_orders) * 100 AVG(DATEDIFF(po.received_date, po.date)) as avg_lead_time_days,
ELSE 0 COUNT(DISTINCT po.po_id) as total_orders,
END as order_fill_rate, COUNT(CASE WHEN DATEDIFF(po.received_date, po.date) > 30 THEN 1 END) as total_late_orders,
CASE SUM(po.ordered * po.po_cost_price) as total_purchase_value,
WHEN vs.total_orders > 0 AVG(po.ordered * po.po_cost_price) as avg_order_value,
THEN vs.total_revenue / vs.total_orders (COUNT(CASE WHEN DATEDIFF(po.received_date, po.date) <= 30 THEN 1 END) / COUNT(*)) * 100 as on_time_delivery_rate,
ELSE 0 (SUM(LEAST(po.received, po.ordered)) / NULLIF(SUM(po.ordered), 0)) * 100 as order_fill_rate
END as avg_order_value, FROM purchase_orders po
COALESCE(vs.active_products, 0) as active_products, WHERE po.vendor IN (?)
COALESCE(vpr.total_products, 0) as total_products, AND po.received_date IS NOT NULL
COALESCE(vp.total_purchase_value, 0) as total_purchase_value, AND po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 365 DAY)
CASE AND po.updated > ?
WHEN vs.total_revenue > 0 GROUP BY po.vendor
THEN (vs.total_margin / vs.total_revenue) * 100 ),
ELSE 0 product_stats AS (
END as avg_margin_percent, SELECT
'active' as status, p.vendor,
NOW() as last_calculated_at COUNT(DISTINCT p.pid) as total_products,
FROM vendor_sales vs COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
LEFT JOIN vendor_po vp ON vs.vendor = vp.vendor AVG(pm.avg_margin_percent) as avg_margin_percent,
LEFT JOIN vendor_products vpr ON vs.vendor = vpr.vendor SUM(pm.total_revenue) as total_revenue
WHERE vs.vendor IS NOT NULL FROM products p
ON DUPLICATE KEY UPDATE LEFT JOIN product_metrics pm ON p.pid = pm.pid
total_revenue = VALUES(total_revenue), WHERE p.vendor IN (?)
total_orders = VALUES(total_orders), AND p.updated > ?
total_late_orders = VALUES(total_late_orders), GROUP BY p.vendor
avg_lead_time_days = VALUES(avg_lead_time_days), )
on_time_delivery_rate = VALUES(on_time_delivery_rate),
order_fill_rate = VALUES(order_fill_rate),
avg_order_value = VALUES(avg_order_value),
active_products = VALUES(active_products),
total_products = VALUES(total_products),
total_purchase_value = VALUES(total_purchase_value),
avg_margin_percent = VALUES(avg_margin_percent),
status = VALUES(status),
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 0.9);
outputProgress({
status: 'running',
operation: 'Vendor metrics calculated, updating time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
success
};
// Calculate time-based metrics
await connection.query(`
INSERT INTO vendor_time_metrics (
vendor,
year,
month,
total_orders,
late_orders,
avg_lead_time_days,
total_purchase_value,
total_revenue,
avg_margin_percent
)
WITH monthly_orders AS (
SELECT SELECT
p.vendor, v.vendor,
YEAR(o.date) as year, COALESCE(ps.avg_lead_time_days, 0) as avg_lead_time_days,
MONTH(o.date) as month, COALESCE(ps.on_time_delivery_rate, 0) as on_time_delivery_rate,
COUNT(DISTINCT o.id) as total_orders, COALESCE(ps.order_fill_rate, 0) as order_fill_rate,
SUM(o.quantity * o.price) as total_revenue, COALESCE(ps.total_orders, 0) as total_orders,
SUM(o.quantity * (o.price - p.cost_price)) as total_margin COALESCE(ps.total_late_orders, 0) as total_late_orders,
FROM products p COALESCE(ps.total_purchase_value, 0) as total_purchase_value,
JOIN orders o ON p.pid = o.pid COALESCE(ps.avg_order_value, 0) as avg_order_value,
WHERE o.canceled = false COALESCE(prs.active_products, 0) as active_products,
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) COALESCE(prs.total_products, 0) as total_products,
AND p.vendor IS NOT NULL COALESCE(prs.total_revenue, 0) as total_revenue,
GROUP BY p.vendor, YEAR(o.date), MONTH(o.date) COALESCE(prs.avg_margin_percent, 0) as avg_margin_percent,
), v.status,
monthly_po AS ( NOW() as last_calculated_at
SELECT FROM vendor_details v
p.vendor, LEFT JOIN purchase_stats ps ON v.vendor = ps.vendor
YEAR(po.date) as year, LEFT JOIN product_stats prs ON v.vendor = prs.vendor
MONTH(po.date) as month, WHERE v.vendor IN (?)
COUNT(DISTINCT po.id) as total_po, ON DUPLICATE KEY UPDATE
COUNT(DISTINCT CASE avg_lead_time_days = VALUES(avg_lead_time_days),
WHEN po.receiving_status = 40 AND po.received_date > po.expected_date on_time_delivery_rate = VALUES(on_time_delivery_rate),
THEN po.id order_fill_rate = VALUES(order_fill_rate),
END) as late_orders, total_orders = VALUES(total_orders),
AVG(CASE total_late_orders = VALUES(total_late_orders),
WHEN po.receiving_status = 40 total_purchase_value = VALUES(total_purchase_value),
THEN DATEDIFF(po.received_date, po.date) avg_order_value = VALUES(avg_order_value),
END) as avg_lead_time_days, active_products = VALUES(active_products),
SUM(po.ordered * po.po_cost_price) as total_purchase_value total_products = VALUES(total_products),
FROM products p total_revenue = VALUES(total_revenue),
JOIN purchase_orders po ON p.pid = po.pid avg_margin_percent = VALUES(avg_margin_percent),
WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) status = VALUES(status),
AND p.vendor IS NOT NULL last_calculated_at = NOW()
GROUP BY p.vendor, YEAR(po.date), MONTH(po.date) `, [batch.map(row => row.vendor), lastCalculationTime, batch.map(row => row.vendor), lastCalculationTime, batch.map(row => row.vendor)]);
)
SELECT
mo.vendor,
mo.year,
mo.month,
COALESCE(mp.total_po, 0) as total_orders,
COALESCE(mp.late_orders, 0) as late_orders,
COALESCE(mp.avg_lead_time_days, 0) as avg_lead_time_days,
COALESCE(mp.total_purchase_value, 0) as total_purchase_value,
mo.total_revenue,
CASE
WHEN mo.total_revenue > 0
THEN (mo.total_margin / mo.total_revenue) * 100
ELSE 0
END as avg_margin_percent
FROM monthly_orders mo
LEFT JOIN monthly_po mp ON mo.vendor = mp.vendor
AND mo.year = mp.year
AND mo.month = mp.month
UNION
SELECT
mp.vendor,
mp.year,
mp.month,
mp.total_po as total_orders,
mp.late_orders,
mp.avg_lead_time_days,
mp.total_purchase_value,
0 as total_revenue,
0 as avg_margin_percent
FROM monthly_po mp
LEFT JOIN monthly_orders mo ON mp.vendor = mo.vendor
AND mp.year = mo.year
AND mp.month = mo.month
WHERE mo.vendor IS NULL
ON DUPLICATE KEY UPDATE
total_orders = VALUES(total_orders),
late_orders = VALUES(late_orders),
avg_lead_time_days = VALUES(avg_lead_time_days),
total_purchase_value = VALUES(total_purchase_value),
total_revenue = VALUES(total_revenue),
avg_margin_percent = VALUES(avg_margin_percent)
`);
processedCount = Math.floor(totalProducts * 0.95); lastVendor = batch[batch.length - 1].vendor;
outputProgress({ processedCount += batch.length;
status: 'running',
operation: 'Time-based vendor metrics calculated', outputProgress({
current: processedCount, status: 'running',
total: totalProducts, operation: 'Processing vendor metrics batch',
elapsed: formatElapsedTime(startTime), current: processedCount,
remaining: estimateRemaining(startTime, processedCount, totalProducts), total: totalVendors,
rate: calculateRate(startTime, processedCount), elapsed: formatElapsedTime(startTime),
percentage: ((processedCount / totalProducts) * 100).toFixed(1), remaining: estimateRemaining(startTime, processedCount, totalVendors),
timing: { rate: calculateRate(startTime, processedCount),
start_time: new Date(startTime).toISOString(), percentage: ((processedCount / totalVendors) * 100).toFixed(1),
end_time: new Date().toISOString(), timing: {
elapsed_seconds: Math.round((Date.now() - startTime) / 1000) start_time: new Date(startTime).toISOString(),
} end_time: new Date().toISOString(),
}); elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully // If we get here, everything completed successfully
success = true; success = true;
@@ -349,8 +216,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
return { return {
processedProducts: processedCount, processedProducts: processedCount,
processedOrders, processedOrders: 0,
processedPurchaseOrders, processedPurchaseOrders: 0,
success success
}; };