matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply gemini suggested calculate improvements

Browse Source
This commit is contained in:
Matt
2025-02-10 15:20:22 -05:00
parent 09f7103472
commit 90379386d6
8 changed files with 211 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
inventory-server/scripts/calculate-metrics.js
View File

@@ -83,6 +83,7 @@ process.on('SIGTERM', cancelCalculation);
async function calculateMetrics() {
let connection;
const startTime = Date.now();
// Initialize all counts to 0
let processedProducts = 0;
let processedOrders = 0;
let processedPurchaseOrders = 0;
@@ -90,7 +91,7 @@ async function calculateMetrics() {
let totalOrders = 0;
let totalPurchaseOrders = 0;
let calculateHistoryId;
try {
// Clean up any previously running calculations
connection = await getConnection();
@@ -140,6 +141,7 @@ async function calculateMetrics() {
totalProducts = productCount.total;
totalOrders = orderCount.total;
totalPurchaseOrders = poCount.total;
connection.release();
// If nothing needs updating, we can exit early
if (totalProducts === 0 && totalOrders === 0 && totalPurchaseOrders === 0) {
@@ -153,6 +155,7 @@ async function calculateMetrics() {
}
// Create history record for this calculation
connection = await getConnection(); // Re-establish connection
const [historyResult] = await connection.query(`
INSERT INTO calculate_history (
start_time,
@@ -183,7 +186,7 @@ async function calculateMetrics() {
totalPurchaseOrders,
SKIP_PRODUCT_METRICS,
SKIP_TIME_AGGREGATES,
SKIP_FINANCIAL_METRICS,
SKIP_FINANCIAL_METRICS,
SKIP_VENDOR_METRICS,
SKIP_CATEGORY_METRICS,
SKIP_BRAND_METRICS,
@@ -209,7 +212,7 @@ async function calculateMetrics() {
}
isCancelled = false;
connection = await getConnection();
connection = await getConnection(); // Get a new connection for the main processing
try {
global.outputProgress({
@@ -228,14 +231,12 @@ async function calculateMetrics() {
}
});
// Update progress periodically
// Update progress periodically - REFACTORED
const updateProgress = async (products = null, orders = null, purchaseOrders = null) => {
// Ensure all values are valid numbers or default to previous value
if (products !== null) processedProducts = Number(products) || processedProducts || 0;
if (orders !== null) processedOrders = Number(orders) || processedOrders || 0;
if (purchaseOrders !== null) processedPurchaseOrders = Number(purchaseOrders) || processedPurchaseOrders || 0;
// Ensure we never send NaN to the database
const safeProducts = Number(processedProducts) || 0;
const safeOrders = Number(processedOrders) || 0;
const safePurchaseOrders = Number(processedPurchaseOrders) || 0;
@@ -250,14 +251,14 @@ async function calculateMetrics() {
`, [safeProducts, safeOrders, safePurchaseOrders, calculateHistoryId]);
};
// Helper function to ensure valid progress numbers
// Helper function to ensure valid progress numbers - this is fine
const ensureValidProgress = (current, total) => ({
current: Number(current) || 0,
total: Number(total) || 1, // Default to 1 to avoid division by zero
percentage: (((Number(current) || 0) / (Number(total) || 1)) * 100).toFixed(1)
});
// Initial progress
// Initial progress - this is fine
const initialProgress = ensureValidProgress(0, totalProducts);
global.outputProgress({
status: 'running',
@@ -275,37 +276,28 @@ async function calculateMetrics() {
}
});
// --- Call each module, passing totals and accumulating processed counts ---
if (!SKIP_PRODUCT_METRICS) {
const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); // Update with accumulated values
if (!result.success) {
throw new Error('Product metrics calculation failed');
}
} else {
console.log('Skipping product metrics calculation...');
processedProducts = Math.floor(totalProducts * 0.6);
await updateProgress(processedProducts);
global.outputProgress({
status: 'running',
operation: 'Skipping product metrics calculation',
current: processedProducts,
total: totalProducts,
elapsed: global.formatElapsedTime(startTime),
remaining: global.estimateRemaining(startTime, processedProducts, totalProducts),
rate: global.calculateRate(startTime, processedProducts),
percentage: '60',
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Don't artificially inflate processedProducts if skipping
}
// Calculate time-based aggregates
if (!SKIP_TIME_AGGREGATES) {
const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Time aggregates calculation failed');
}
@@ -313,21 +305,25 @@ async function calculateMetrics() {
console.log('Skipping time aggregates calculation');
}
// Calculate financial metrics
if (!SKIP_FINANCIAL_METRICS) {
const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Financial metrics calculation failed');
}
} else {
console.log('Skipping financial metrics calculation');
}
// Calculate vendor metrics
if (!SKIP_VENDOR_METRICS) {
const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Vendor metrics calculation failed');
}
@@ -335,10 +331,12 @@ async function calculateMetrics() {
console.log('Skipping vendor metrics calculation');
}
// Calculate category metrics
if (!SKIP_CATEGORY_METRICS) {
const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Category metrics calculation failed');
}
@@ -346,10 +344,12 @@ async function calculateMetrics() {
console.log('Skipping category metrics calculation');
}
// Calculate brand metrics
if (!SKIP_BRAND_METRICS) {
const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Brand metrics calculation failed');
}
@@ -357,10 +357,12 @@ async function calculateMetrics() {
console.log('Skipping brand metrics calculation');
}
// Calculate sales forecasts
if (!SKIP_SALES_FORECASTS) {
const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Sales forecasts calculation failed');
}
@@ -368,23 +370,7 @@ async function calculateMetrics() {
console.log('Skipping sales forecasts calculation');
}
// Calculate ABC classification
outputProgress({
status: 'running',
operation: 'Starting ABC classification',
current: processedProducts || 0,
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// --- ABC Classification (Refactored) ---
if (isCancelled) return {
processedProducts: processedProducts || 0,
processedOrders: processedOrders || 0,
@@ -402,21 +388,26 @@ async function calculateMetrics() {
pid BIGINT NOT NULL,
total_revenue DECIMAL(10,3),
rank_num INT,
dense_rank_num INT,
percentile DECIMAL(5,2),
total_count INT,
PRIMARY KEY (pid),
INDEX (rank_num)
INDEX (rank_num),
INDEX (dense_rank_num),
INDEX (percentile)
) ENGINE=MEMORY
`);
let processedCount = processedProducts;
outputProgress({
status: 'running',
operation: 'Creating revenue rankings',
current: processedProducts || 0,
total: totalProducts || 0,
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -431,49 +422,35 @@ async function calculateMetrics() {
success: false
};
// Calculate rankings with proper tie handling and get total count in one go.
await connection.query(`
INSERT INTO temp_revenue_ranks
WITH revenue_data AS (
SELECT
pid,
total_revenue,
COUNT(*) OVER () as total_count,
PERCENT_RANK() OVER (ORDER BY total_revenue DESC) * 100 as percentile,
RANK() OVER (ORDER BY total_revenue DESC) as rank_num,
DENSE_RANK() OVER (ORDER BY total_revenue DESC) as dense_rank_num
FROM product_metrics
WHERE total_revenue > 0
)
SELECT
pid,
total_revenue,
@rank := @rank + 1 as rank_num,
@total_count := @rank as total_count
FROM (
SELECT pid, total_revenue
FROM product_metrics
WHERE total_revenue > 0
ORDER BY total_revenue DESC
) ranked,
(SELECT @rank := 0) r
rank_num,
dense_rank_num,
percentile,
total_count
FROM revenue_data
`);
// Get total count for percentage calculation
const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
const totalCount = rankingCount[0].total_count || 1;
const max_rank = totalCount; // Store max_rank for use in classification
outputProgress({
status: 'running',
operation: 'Updating ABC classifications',
current: processedProducts || 0,
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedProducts || 0,
processedOrders: processedOrders || 0,
processedPurchaseOrders: 0,
success: false
};
// Get total count for percentage calculation (already done in the above query)
// No need for this separate query:
// const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
// const totalCount = rankingCount[0].total_count || 1;
// const max_rank = totalCount; // Store max_rank for use in classification
// ABC classification progress tracking
let abcProcessedCount = 0;
@@ -489,7 +466,7 @@ async function calculateMetrics() {
success: false
};
// First get a batch of PIDs that need updating
// Get a batch of PIDs that need updating - REFACTORED to use percentile
const [pids] = await connection.query(`
SELECT pm.pid
FROM product_metrics pm
@@ -497,41 +474,38 @@ async function calculateMetrics() {
WHERE pm.abc_class IS NULL
OR pm.abc_class !=
CASE
WHEN tr.rank_num IS NULL THEN 'C'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
WHEN tr.pid IS NULL THEN 'C'
WHEN tr.percentile <= ? THEN 'A'
WHEN tr.percentile <= ? THEN 'B'
ELSE 'C'
END
LIMIT ?
`, [max_rank, abcThresholds.a_threshold,
max_rank, abcThresholds.b_threshold,
batchSize]);
`, [abcThresholds.a_threshold, abcThresholds.b_threshold, batchSize]);
if (pids.length === 0) {
break;
}
// Then update just those PIDs
const [result] = await connection.query(`
// Update just those PIDs - REFACTORED to use percentile
await connection.query(`
UPDATE product_metrics pm
LEFT JOIN temp_revenue_ranks tr ON pm.pid = tr.pid
SET pm.abc_class =
CASE
WHEN tr.rank_num IS NULL THEN 'C'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
WHEN tr.pid IS NULL THEN 'C'
WHEN tr.percentile <= ? THEN 'A'
WHEN tr.percentile <= ? THEN 'B'
ELSE 'C'
END,
pm.last_calculated_at = NOW()
WHERE pm.pid IN (?)
`, [max_rank, abcThresholds.a_threshold,
max_rank, abcThresholds.b_threshold,
pids.map(row => row.pid)]);
`, [abcThresholds.a_threshold, abcThresholds.b_threshold, pids.map(row => row.pid)]);
abcProcessedCount += pids.length; // Use pids.length, more accurate
processedProducts += pids.length; // Add to the main processedProducts
abcProcessedCount += result.affectedRows;
// Calculate progress ensuring valid numbers
const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalCount || 1)) * 0.01));
const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalProducts || 1)) * 0.01));
processedProducts = Number(currentProgress) || processedProducts || 0;
// Only update progress at most once per second