matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply gemini suggested calculate improvements

Browse Source
This commit is contained in:
Matt
2025-02-10 15:20:22 -05:00
parent 09f7103472
commit 90379386d6
8 changed files with 211 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
inventory-server/scripts/calculate-metrics.js
View File

@@ -83,6 +83,7 @@ process.on('SIGTERM', cancelCalculation);
async function calculateMetrics() {
let connection;
const startTime = Date.now();
// Initialize all counts to 0
let processedProducts = 0;
let processedOrders = 0;
let processedPurchaseOrders = 0;
@@ -90,7 +91,7 @@ async function calculateMetrics() {
let totalOrders = 0;
let totalPurchaseOrders = 0;
let calculateHistoryId;
try {
// Clean up any previously running calculations
connection = await getConnection();
@@ -140,6 +141,7 @@ async function calculateMetrics() {
totalProducts = productCount.total;
totalOrders = orderCount.total;
totalPurchaseOrders = poCount.total;
connection.release();
// If nothing needs updating, we can exit early
if (totalProducts === 0 && totalOrders === 0 && totalPurchaseOrders === 0) {
@@ -153,6 +155,7 @@ async function calculateMetrics() {
}
// Create history record for this calculation
connection = await getConnection(); // Re-establish connection
const [historyResult] = await connection.query(`
INSERT INTO calculate_history (
start_time,
@@ -183,7 +186,7 @@ async function calculateMetrics() {
totalPurchaseOrders,
SKIP_PRODUCT_METRICS,
SKIP_TIME_AGGREGATES,
SKIP_FINANCIAL_METRICS,
SKIP_FINANCIAL_METRICS,
SKIP_VENDOR_METRICS,
SKIP_CATEGORY_METRICS,
SKIP_BRAND_METRICS,
@@ -209,7 +212,7 @@ async function calculateMetrics() {
}
isCancelled = false;
connection = await getConnection();
connection = await getConnection(); // Get a new connection for the main processing
try {
global.outputProgress({
@@ -228,14 +231,12 @@ async function calculateMetrics() {
}
});
// Update progress periodically
// Update progress periodically - REFACTORED
const updateProgress = async (products = null, orders = null, purchaseOrders = null) => {
// Ensure all values are valid numbers or default to previous value
if (products !== null) processedProducts = Number(products) || processedProducts || 0;
if (orders !== null) processedOrders = Number(orders) || processedOrders || 0;
if (purchaseOrders !== null) processedPurchaseOrders = Number(purchaseOrders) || processedPurchaseOrders || 0;
// Ensure we never send NaN to the database
const safeProducts = Number(processedProducts) || 0;
const safeOrders = Number(processedOrders) || 0;
const safePurchaseOrders = Number(processedPurchaseOrders) || 0;
@@ -250,14 +251,14 @@ async function calculateMetrics() {
`, [safeProducts, safeOrders, safePurchaseOrders, calculateHistoryId]);
};
// Helper function to ensure valid progress numbers
// Helper function to ensure valid progress numbers - this is fine
const ensureValidProgress = (current, total) => ({
current: Number(current) || 0,
total: Number(total) || 1, // Default to 1 to avoid division by zero
percentage: (((Number(current) || 0) / (Number(total) || 1)) * 100).toFixed(1)
});
// Initial progress
// Initial progress - this is fine
const initialProgress = ensureValidProgress(0, totalProducts);
global.outputProgress({
status: 'running',
@@ -275,37 +276,28 @@ async function calculateMetrics() {
}
});
// --- Call each module, passing totals and accumulating processed counts ---
if (!SKIP_PRODUCT_METRICS) {
const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders); // Update with accumulated values
if (!result.success) {
throw new Error('Product metrics calculation failed');
}
} else {
console.log('Skipping product metrics calculation...');
processedProducts = Math.floor(totalProducts * 0.6);
await updateProgress(processedProducts);
global.outputProgress({
status: 'running',
operation: 'Skipping product metrics calculation',
current: processedProducts,
total: totalProducts,
elapsed: global.formatElapsedTime(startTime),
remaining: global.estimateRemaining(startTime, processedProducts, totalProducts),
rate: global.calculateRate(startTime, processedProducts),
percentage: '60',
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Don't artificially inflate processedProducts if skipping
}
// Calculate time-based aggregates
if (!SKIP_TIME_AGGREGATES) {
const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateTimeAggregates(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Time aggregates calculation failed');
}
@@ -313,21 +305,25 @@ async function calculateMetrics() {
console.log('Skipping time aggregates calculation');
}
// Calculate financial metrics
if (!SKIP_FINANCIAL_METRICS) {
const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateFinancialMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Financial metrics calculation failed');
}
} else {
console.log('Skipping financial metrics calculation');
}
// Calculate vendor metrics
if (!SKIP_VENDOR_METRICS) {
const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateVendorMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Vendor metrics calculation failed');
}
@@ -335,10 +331,12 @@ async function calculateMetrics() {
console.log('Skipping vendor metrics calculation');
}
// Calculate category metrics
if (!SKIP_CATEGORY_METRICS) {
const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateCategoryMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Category metrics calculation failed');
}
@@ -346,10 +344,12 @@ async function calculateMetrics() {
console.log('Skipping category metrics calculation');
}
// Calculate brand metrics
if (!SKIP_BRAND_METRICS) {
const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateBrandMetrics(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Brand metrics calculation failed');
}
@@ -357,10 +357,12 @@ async function calculateMetrics() {
console.log('Skipping brand metrics calculation');
}
// Calculate sales forecasts
if (!SKIP_SALES_FORECASTS) {
const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
const result = await calculateSalesForecasts(startTime, totalProducts, processedProducts, isCancelled); // Pass totals
processedProducts += result.processedProducts; // Accumulate
processedOrders += result.processedOrders;
processedPurchaseOrders += result.processedPurchaseOrders;
await updateProgress(processedProducts, processedOrders, processedPurchaseOrders);
if (!result.success) {
throw new Error('Sales forecasts calculation failed');
}
@@ -368,23 +370,7 @@ async function calculateMetrics() {
console.log('Skipping sales forecasts calculation');
}
// Calculate ABC classification
outputProgress({
status: 'running',
operation: 'Starting ABC classification',
current: processedProducts || 0,
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// --- ABC Classification (Refactored) ---
if (isCancelled) return {
processedProducts: processedProducts || 0,
processedOrders: processedOrders || 0,
@@ -402,21 +388,26 @@ async function calculateMetrics() {
pid BIGINT NOT NULL,
total_revenue DECIMAL(10,3),
rank_num INT,
dense_rank_num INT,
percentile DECIMAL(5,2),
total_count INT,
PRIMARY KEY (pid),
INDEX (rank_num)
INDEX (rank_num),
INDEX (dense_rank_num),
INDEX (percentile)
) ENGINE=MEMORY
`);
let processedCount = processedProducts;
outputProgress({
status: 'running',
operation: 'Creating revenue rankings',
current: processedProducts || 0,
total: totalProducts || 0,
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -431,49 +422,35 @@ async function calculateMetrics() {
success: false
};
// Calculate rankings with proper tie handling and get total count in one go.
await connection.query(`
INSERT INTO temp_revenue_ranks
WITH revenue_data AS (
SELECT
pid,
total_revenue,
COUNT(*) OVER () as total_count,
PERCENT_RANK() OVER (ORDER BY total_revenue DESC) * 100 as percentile,
RANK() OVER (ORDER BY total_revenue DESC) as rank_num,
DENSE_RANK() OVER (ORDER BY total_revenue DESC) as dense_rank_num
FROM product_metrics
WHERE total_revenue > 0
)
SELECT
pid,
total_revenue,
@rank := @rank + 1 as rank_num,
@total_count := @rank as total_count
FROM (
SELECT pid, total_revenue
FROM product_metrics
WHERE total_revenue > 0
ORDER BY total_revenue DESC
) ranked,
(SELECT @rank := 0) r
rank_num,
dense_rank_num,
percentile,
total_count
FROM revenue_data
`);
// Get total count for percentage calculation
const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
const totalCount = rankingCount[0].total_count || 1;
const max_rank = totalCount; // Store max_rank for use in classification
outputProgress({
status: 'running',
operation: 'Updating ABC classifications',
current: processedProducts || 0,
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedProducts || 0, totalProducts || 0),
rate: calculateRate(startTime, processedProducts || 0),
percentage: (((processedProducts || 0) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedProducts || 0,
processedOrders: processedOrders || 0,
processedPurchaseOrders: 0,
success: false
};
// Get total count for percentage calculation (already done in the above query)
// No need for this separate query:
// const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
// const totalCount = rankingCount[0].total_count || 1;
// const max_rank = totalCount; // Store max_rank for use in classification
// ABC classification progress tracking
let abcProcessedCount = 0;
@@ -489,7 +466,7 @@ async function calculateMetrics() {
success: false
};
// First get a batch of PIDs that need updating
// Get a batch of PIDs that need updating - REFACTORED to use percentile
const [pids] = await connection.query(`
SELECT pm.pid
FROM product_metrics pm
@@ -497,41 +474,38 @@ async function calculateMetrics() {
WHERE pm.abc_class IS NULL
OR pm.abc_class !=
CASE
WHEN tr.rank_num IS NULL THEN 'C'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
WHEN tr.pid IS NULL THEN 'C'
WHEN tr.percentile <= ? THEN 'A'
WHEN tr.percentile <= ? THEN 'B'
ELSE 'C'
END
LIMIT ?
`, [max_rank, abcThresholds.a_threshold,
max_rank, abcThresholds.b_threshold,
batchSize]);
`, [abcThresholds.a_threshold, abcThresholds.b_threshold, batchSize]);
if (pids.length === 0) {
break;
}
// Then update just those PIDs
const [result] = await connection.query(`
// Update just those PIDs - REFACTORED to use percentile
await connection.query(`
UPDATE product_metrics pm
LEFT JOIN temp_revenue_ranks tr ON pm.pid = tr.pid
SET pm.abc_class =
CASE
WHEN tr.rank_num IS NULL THEN 'C'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
WHEN tr.pid IS NULL THEN 'C'
WHEN tr.percentile <= ? THEN 'A'
WHEN tr.percentile <= ? THEN 'B'
ELSE 'C'
END,
pm.last_calculated_at = NOW()
WHERE pm.pid IN (?)
`, [max_rank, abcThresholds.a_threshold,
max_rank, abcThresholds.b_threshold,
pids.map(row => row.pid)]);
`, [abcThresholds.a_threshold, abcThresholds.b_threshold, pids.map(row => row.pid)]);
abcProcessedCount += pids.length; // Use pids.length, more accurate
processedProducts += pids.length; // Add to the main processedProducts
abcProcessedCount += result.affectedRows;
// Calculate progress ensuring valid numbers
const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalCount || 1)) * 0.01));
const currentProgress = Math.floor(totalProducts * (0.99 + (abcProcessedCount / (totalProducts || 1)) * 0.01));
processedProducts = Number(currentProgress) || processedProducts || 0;
// Only update progress at most once per second

38
inventory-server/scripts/metrics/brand-metrics.js
View File

@@ -5,7 +5,8 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
const connection = await getConnection();
let success = false;
const BATCH_SIZE = 5000;
let myProcessedProducts = 0; // Not *directly* processing products, tracking brands
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
@@ -26,12 +27,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalBrands = brandCount[0].count;
const totalBrands = brandCount[0].count; // Track total *brands*
if (totalBrands === 0) {
console.log('No brands need metric updates');
return {
processedProducts: 0,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
@@ -42,12 +43,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
outputProgress({
status: 'cancelled',
operation: 'Brand metrics calculation cancelled',
current: processedCount,
total: totalBrands,
current: processedCount, // Use passed-in value
total: totalBrands, // Report total *brands*
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
percentage: ((processedCount / totalBrands) * 100).toFixed(1), // Base on brands
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -55,7 +56,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
}
});
return {
processedProducts: processedCount,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -65,12 +66,12 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
outputProgress({
status: 'running',
operation: 'Starting brand metrics calculation',
current: processedCount,
total: totalBrands,
current: processedCount, // Use passed-in value
total: totalBrands, // Report total *brands*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalBrands),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
percentage: ((processedCount / totalBrands) * 100).toFixed(1), // Base on brands
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -80,6 +81,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
// Process in batches
let lastBrand = '';
let processedBrands = 0; // Track processed brands
while (true) {
if (isCancelled) break;
@@ -243,17 +245,17 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
lastBrand = batch[batch.length - 1].brand;
processedCount += batch.length;
processedBrands += batch.length; // Increment processed *brands*
outputProgress({
status: 'running',
operation: 'Processing brand metrics batch',
current: processedCount,
total: totalBrands,
current: processedCount + processedBrands, // Use cumulative brand count
total: totalBrands, // Report total *brands*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalBrands),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount + processedBrands, totalBrands),
rate: calculateRate(startTime, processedCount + processedBrands),
percentage: (((processedCount + processedBrands) / totalBrands) * 100).toFixed(1), // Base on brands
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -264,7 +266,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
// If we get here, everything completed successfully
success = true;
// Update calculate_status
await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp)
@@ -273,7 +275,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
`);
return {
processedProducts: processedCount,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success

38
inventory-server/scripts/metrics/category-metrics.js
View File

@@ -5,7 +5,8 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
const connection = await getConnection();
let success = false;
const BATCH_SIZE = 5000;
let myProcessedProducts = 0; // Not *directly* processing products, but tracking categories
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
@@ -28,12 +29,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalCategories = categoryCount[0].count;
const totalCategories = categoryCount[0].count; // Track total *categories*
if (totalCategories === 0) {
console.log('No categories need metric updates');
return {
processedProducts: 0,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
@@ -44,12 +45,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
outputProgress({
status: 'cancelled',
operation: 'Category metrics calculation cancelled',
current: processedCount,
total: totalCategories,
current: processedCount, // Use passed-in value
total: totalCategories, // Report total *categories*
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
percentage: ((processedCount / totalCategories) * 100).toFixed(1), // Base on categories
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -57,7 +58,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
}
});
return {
processedProducts: processedCount,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -67,12 +68,12 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
outputProgress({
status: 'running',
operation: 'Starting category metrics calculation',
current: processedCount,
total: totalCategories,
current: processedCount, // Use passed-in value
total: totalCategories, // Report total *categories*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalCategories),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
percentage: ((processedCount / totalCategories) * 100).toFixed(1), // Base on categories
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -82,6 +83,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
// Process in batches
let lastCatId = 0;
let processedCategories = 0; // Track processed categories
while (true) {
if (isCancelled) break;
@@ -247,17 +249,17 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
lastCatId = batch[batch.length - 1].cat_id;
processedCount += batch.length;
processedCategories += batch.length; // Increment processed *categories*
outputProgress({
status: 'running',
operation: 'Processing category metrics batch',
current: processedCount,
total: totalCategories,
current: processedCount + processedCategories, // Use cumulative category count
total: totalCategories, // Report total *categories*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalCategories),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount + processedCategories, totalCategories),
rate: calculateRate(startTime, processedCount + processedCategories),
percentage: (((processedCount + processedCategories) / totalCategories) * 100).toFixed(1), // Base on categories
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -268,7 +270,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
// If we get here, everything completed successfully
success = true;
// Update calculate_status
await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp)
@@ -277,7 +279,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
`);
return {
processedProducts: processedCount,
processedProducts: 0, // Not directly processing products
processedOrders: 0,
processedPurchaseOrders: 0,
success

15
inventory-server/scripts/metrics/financial-metrics.js
View File

@@ -5,6 +5,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
const connection = await getConnection();
let success = false;
const BATCH_SIZE = 5000;
let myProcessedProducts = 0; // Track products processed *within this module*
try {
// Get last calculation timestamp
@@ -54,7 +55,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
}
});
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts,
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -133,17 +134,17 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
`, [batch.map(row => row.pid)]);
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
myProcessedProducts += batch.length;
outputProgress({
status: 'running',
operation: 'Processing financial metrics batch',
current: processedCount,
current: processedCount + myProcessedProducts,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount + myProcessedProducts, totalProducts),
rate: calculateRate(startTime, processedCount + myProcessedProducts),
percentage: (((processedCount + myProcessedProducts) / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -163,7 +164,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
`);
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts,
processedOrders: 0,
processedPurchaseOrders: 0,
success

58
inventory-server/scripts/metrics/product-metrics.js
View File

@@ -13,6 +13,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
const connection = await getConnection();
let success = false;
let processedOrders = 0;
let myProcessedProducts = 0; // Track products processed *within this module*
const BATCH_SIZE = 5000;
try {
@@ -24,24 +25,10 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total product count if not provided
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need updating');
return {
processedProducts: 0,
processedProducts: myProcessedProducts,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
@@ -69,7 +56,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
}
});
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts,
processedOrders,
processedPurchaseOrders: 0,
success
@@ -290,6 +277,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
myProcessedProducts += batch.length; // Increment the *module's* count
outputProgress({
status: 'running',
@@ -361,12 +349,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
outputProgress({
status: 'running',
operation: 'Starting product time aggregates calculation',
current: processedCount || 0,
total: totalProducts || 0,
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0),
rate: calculateRate(startTime, processedCount || 0),
percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -428,12 +416,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
outputProgress({
status: 'running',
operation: 'Product time aggregates calculated',
current: processedCount || 0,
total: totalProducts || 0,
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0),
rate: calculateRate(startTime, processedCount || 0),
percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -445,12 +433,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
outputProgress({
status: 'running',
operation: 'Skipping product time aggregates calculation',
current: processedCount || 0,
total: totalProducts || 0,
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount || 0, totalProducts || 0),
rate: calculateRate(startTime, processedCount || 0),
percentage: (((processedCount || 0) / (totalProducts || 1)) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: (((processedCount) / (totalProducts || 1)) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -479,7 +467,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0, // This module doesn't process POs
processedPurchaseOrders: 0,
success
};
@@ -540,7 +528,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0, // This module doesn't process POs
processedPurchaseOrders: 0,
success
};
@@ -620,9 +608,9 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
`);
return {
processedProducts: processedCount || 0,
processedProducts: processedCount,
processedOrders: processedOrders || 0,
processedPurchaseOrders: 0, // This module doesn't process POs
processedPurchaseOrders: 0,
success
};

17
inventory-server/scripts/metrics/sales-forecasts.js
View File

@@ -4,6 +4,7 @@ const { getConnection } = require('./utils/db');
async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let myProcessedProducts = 0; // Track products processed *within this module*
const BATCH_SIZE = 5000;
try {
@@ -43,7 +44,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
status: 'cancelled',
operation: 'Sales forecast calculation cancelled',
current: processedCount,
total: totalProductsToUpdate,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
@@ -55,7 +56,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
}
});
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts,
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -66,9 +67,9 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
status: 'running',
operation: 'Starting sales forecast calculation',
current: processedCount,
total: totalProductsToUpdate,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: {
@@ -255,15 +256,15 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_confidence_calc');
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
myProcessedProducts += batch.length;
outputProgress({
status: 'running',
operation: 'Processing sales forecast batch',
current: processedCount,
total: totalProductsToUpdate,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: {
@@ -285,7 +286,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
`);
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts,
processedOrders: 0,
processedPurchaseOrders: 0,
success

35
inventory-server/scripts/metrics/time-aggregates.js
View File

@@ -5,7 +5,8 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
const connection = await getConnection();
let success = false;
const BATCH_SIZE = 5000;
let myProcessedProducts = 0; // Track products processed *within this module*
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
@@ -15,17 +16,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
// We now receive totalProducts as an argument, so we don't need to query for it here.
if (totalProducts === 0) {
console.log('No products need time aggregate updates');
@@ -41,7 +32,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
outputProgress({
status: 'cancelled',
operation: 'Time aggregates calculation cancelled',
current: processedCount,
current: processedCount, // Use passed-in value
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: null,
@@ -54,7 +45,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
}
});
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts, // Return only what *this* module processed
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -64,7 +55,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
outputProgress({
status: 'running',
operation: 'Starting time aggregates calculation',
current: processedCount,
current: processedCount, // Use passed-in value
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
@@ -253,17 +244,17 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_time_aggregates');
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
myProcessedProducts += batch.length; // Increment *this module's* count
outputProgress({
status: 'running',
operation: 'Processing time aggregates batch',
current: processedCount,
current: processedCount + myProcessedProducts, // Show cumulative progress
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount + myProcessedProducts, totalProducts),
rate: calculateRate(startTime, processedCount + myProcessedProducts),
percentage: (((processedCount + myProcessedProducts) / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -274,7 +265,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
// If we get here, everything completed successfully
success = true;
// Update calculate_status
await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp)
@@ -283,7 +274,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
`);
return {
processedProducts: processedCount,
processedProducts: myProcessedProducts, // Return only what *this* module processed
processedOrders: 0,
processedPurchaseOrders: 0,
success

40
inventory-server/scripts/metrics/vendor-metrics.js
View File

@@ -5,7 +5,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
const connection = await getConnection();
let success = false;
const BATCH_SIZE = 5000;
let myProcessedProducts = 0; // Not directly processing products, but we'll track vendors
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
@@ -33,12 +34,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
)
)
`, [lastCalculationTime, lastCalculationTime]);
const totalVendors = vendorCount[0].count;
const totalVendors = vendorCount[0].count; // Track total *vendors*
if (totalVendors === 0) {
console.log('No vendors need metric updates');
return {
processedProducts: 0,
processedProducts: 0, // No products directly processed
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
@@ -49,12 +50,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
outputProgress({
status: 'cancelled',
operation: 'Vendor metrics calculation cancelled',
current: processedCount,
total: totalVendors,
current: processedCount, // Use passed-in value (for consistency)
total: totalVendors, // Report total *vendors*
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
percentage: ((processedCount / totalVendors) * 100).toFixed(1), // Base on vendors
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -62,7 +63,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
}
});
return {
processedProducts: processedCount,
processedProducts: 0, // No products directly processed
processedOrders: 0,
processedPurchaseOrders: 0,
success
@@ -72,12 +73,12 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
outputProgress({
status: 'running',
operation: 'Starting vendor metrics calculation',
current: processedCount,
total: totalVendors,
current: processedCount, // Use passed-in value
total: totalVendors, // Report total *vendors*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalVendors),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
percentage: ((processedCount / totalVendors) * 100).toFixed(1), // Base on vendors
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -87,6 +88,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
// Process in batches
let lastVendor = '';
let processedVendors = 0; // Track processed vendors
while (true) {
if (isCancelled) break;
@@ -119,7 +121,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
// Create temporary tables with optimized structure and indexes
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_purchase_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query(`
CREATE TEMPORARY TABLE temp_purchase_stats (
vendor VARCHAR(100) NOT NULL,
@@ -253,17 +255,17 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
lastVendor = batch[batch.length - 1].vendor;
processedCount += batch.length;
processedVendors += batch.length; // Increment processed *vendors*
outputProgress({
status: 'running',
operation: 'Processing vendor metrics batch',
current: processedCount,
total: totalVendors,
current: processedCount + processedVendors, // Use cumulative vendor count
total: totalVendors, // Report total *vendors*
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalVendors),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
remaining: estimateRemaining(startTime, processedCount + processedVendors, totalVendors),
rate: calculateRate(startTime, processedCount + processedVendors),
percentage: (((processedCount + processedVendors) / totalVendors) * 100).toFixed(1), // Base on vendors
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -274,7 +276,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
// If we get here, everything completed successfully
success = true;
// Update calculate_status
await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp)
@@ -283,7 +285,7 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
`);
return {
processedProducts: processedCount,
processedProducts: 0, // No products directly processed
processedOrders: 0,
processedPurchaseOrders: 0,
success