matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes and enhancements for calculate metrics script

Browse Source
This commit is contained in:
Matt
2025-01-12 18:13:05 -05:00
parent 4d3f956a03
commit 35105f9474
3 changed files with 337 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
inventory-server/scripts/calculate-metrics.js
View File

@@ -1,6 +1,7 @@
const mysql = require('mysql2/promise');
const path = require('path');
require('dotenv').config({ path: path.resolve(__dirname, '..', '.env') });
const fs = require('fs');
// Helper function to format elapsed time
function formatElapsedTime(startTime) {
@@ -43,22 +44,59 @@ function calculateRate(startTime, current) {
// Helper function to output progress
function outputProgress(data) {
// Save progress to file for resumption
saveProgress(data);
// Format as SSE event
const event = {
progress: data
};
// Always send to stdout for frontend
process.stdout.write(JSON.stringify(event) + '\n');
// Log significant events to disk
const isSignificant =
// Operation starts
(data.operation && !data.current) ||
// Operation completions and errors
data.status === 'complete' ||
data.status === 'error' ||
// Major phase changes
data.operation?.includes('Starting ABC classification') ||
data.operation?.includes('Starting time-based aggregates') ||
data.operation?.includes('Starting vendor metrics');
if (isSignificant) {
logImport(`${data.operation || 'Operation'}${data.message ? ': ' + data.message : ''}${data.error ? ' Error: ' + data.error : ''}${data.status ? ' Status: ' + data.status : ''}`);
}
}
// Set up logging
const LOG_DIR = path.join(__dirname, '../logs');
const ERROR_LOG = path.join(LOG_DIR, 'import-errors.log');
const IMPORT_LOG = path.join(LOG_DIR, 'import.log');
// Ensure log directory exists
if (!fs.existsSync(LOG_DIR)) {
fs.mkdirSync(LOG_DIR, { recursive: true });
}
// Helper function to log errors
function logError(error, context) {
console.error(JSON.stringify({
progress: {
status: 'error',
error: error.message || error,
context
}
}));
function logError(error, context = '') {
const timestamp = new Date().toISOString();
const errorMessage = `[${timestamp}] ${context}\nError: ${error.message}\nStack: ${error.stack}\n\n`;
// Log to error file
fs.appendFileSync(ERROR_LOG, errorMessage);
// Also log to console
console.error(`\n${context}\nError: ${error.message}`);
}
// Helper function to log import progress
function logImport(message, isSignificant = true) {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] ${message}\n`;
fs.appendFileSync(IMPORT_LOG, logMessage);
}
// Database configuration
@@ -75,8 +113,52 @@ const dbConfig = {
// Add cancel handler
let isCancelled = false;
// Add status file handling for progress resumption
const STATUS_FILE = path.join(__dirname, '..', 'logs', 'metrics-status.json');
function saveProgress(progress) {
try {
fs.writeFileSync(STATUS_FILE, JSON.stringify({
...progress,
timestamp: Date.now()
}));
} catch (err) {
console.error('Failed to save progress:', err);
}
}
function clearProgress() {
try {
if (fs.existsSync(STATUS_FILE)) {
fs.unlinkSync(STATUS_FILE);
}
} catch (err) {
console.error('Failed to clear progress:', err);
}
}
function getProgress() {
try {
if (fs.existsSync(STATUS_FILE)) {
const progress = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf8'));
// Check if the progress is still valid (less than 1 hour old)
if (progress.timestamp && Date.now() - progress.timestamp < 3600000) {
return progress;
} else {
// Clear old progress
clearProgress();
}
}
} catch (err) {
console.error('Failed to read progress:', err);
clearProgress();
}
return null;
}
function cancelCalculation() {
isCancelled = true;
clearProgress();
// Format as SSE event
const event = {
progress: {
@@ -86,18 +168,22 @@ function cancelCalculation() {
total: 0,
elapsed: null,
remaining: null,
rate: 0
rate: 0,
timestamp: Date.now()
}
};
process.stdout.write(JSON.stringify(event) + '\n');
process.exit(0);
}
// Handle SIGTERM signal for cancellation
process.on('SIGTERM', cancelCalculation);
async function calculateMetrics() {
let pool;
const startTime = Date.now();
let processedCount = 0;
let totalProducts = 0; // Initialize at the top
let totalProducts = 0;
try {
isCancelled = false;
@@ -431,7 +517,13 @@ async function calculateMetrics() {
throw err;
});
const threshold = thresholds[0] || { critical_days: 7, reorder_days: 14, overstock_days: 90 };
const threshold = thresholds[0] || {
critical_days: 7,
reorder_days: 14,
overstock_days: 90,
safety_stock_days: 14, // Add default safety stock days
service_level: 95.0 // Add default service level
};
// Calculate metrics
const metrics = salesMetrics[0] || {};
@@ -452,13 +544,13 @@ async function calculateMetrics() {
// Calculate stock status using configurable thresholds with proper handling of zero sales
const stock_status = daily_sales_avg === 0 ? 'New' :
stock.stock_quantity <= Math.max(1, Math.ceil(daily_sales_avg * config.critical_days)) ? 'Critical' :
stock.stock_quantity <= Math.max(1, Math.ceil(daily_sales_avg * config.reorder_days)) ? 'Reorder' :
stock.stock_quantity > Math.max(1, daily_sales_avg * config.overstock_days) ? 'Overstocked' : 'Healthy';
stock.stock_quantity <= Math.max(1, Math.ceil(daily_sales_avg * threshold.critical_days)) ? 'Critical' :
stock.stock_quantity <= Math.max(1, Math.ceil(daily_sales_avg * threshold.reorder_days)) ? 'Reorder' :
stock.stock_quantity > Math.max(1, daily_sales_avg * threshold.overstock_days) ? 'Overstocked' : 'Healthy';
// Calculate safety stock using configured values
// Calculate safety stock using configured values with proper defaults
const safety_stock = daily_sales_avg > 0 ?
Math.max(1, Math.ceil(daily_sales_avg * config.safety_stock_days * (config.service_level / 100))) :
Math.max(1, Math.ceil(daily_sales_avg * (threshold.safety_stock_days || 14) * ((threshold.service_level || 95.0) / 100))) :
null;
// Add to batch update
@@ -473,8 +565,8 @@ async function calculateMetrics() {
metrics.last_sale_date || null,
daily_sales_avg > 0 ? stock.stock_quantity / daily_sales_avg : null,
weekly_sales_avg > 0 ? stock.stock_quantity / weekly_sales_avg : null,
daily_sales_avg > 0 ? Math.max(1, Math.ceil(daily_sales_avg * config.reorder_days)) : null,
daily_sales_avg > 0 ? Math.max(1, Math.ceil(daily_sales_avg * config.critical_days)) : null,
daily_sales_avg > 0 ? Math.max(1, Math.ceil(daily_sales_avg * threshold.reorder_days)) : null,
daily_sales_avg > 0 ? Math.max(1, Math.ceil(daily_sales_avg * threshold.critical_days)) : null,
margin_percent,
metrics.total_revenue || 0,
inventory_value || 0,
@@ -542,18 +634,38 @@ async function calculateMetrics() {
}
// Update progress for ABC classification
if (isCancelled) {
throw new Error('Operation cancelled');
}
outputProgress({
status: 'running',
operation: 'Calculating ABC classification',
current: totalProducts,
operation: 'Starting ABC classification',
current: Math.floor(totalProducts * 0.7), // Start from 70% after product processing
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, totalProducts, totalProducts),
rate: calculateRate(startTime, totalProducts),
percentage: '100'
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.7), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.7)),
percentage: '70'
});
// Calculate ABC classification using configured thresholds
if (isCancelled) {
throw new Error('Operation cancelled');
}
const [abcConfig] = await connection.query('SELECT a_threshold, b_threshold FROM abc_classification_config WHERE id = 1');
const abcThresholds = abcConfig[0] || { a_threshold: 20, b_threshold: 50 };
outputProgress({
status: 'running',
operation: 'Calculating ABC rankings',
current: Math.floor(totalProducts * 0.8),
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.8), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.8)),
percentage: '80'
});
await connection.query(`
WITH revenue_rankings AS (
SELECT
@@ -576,23 +688,40 @@ async function calculateMetrics() {
JOIN classification_update cu ON pm.product_id = cu.product_id
SET pm.abc_class = cu.abc_class,
pm.last_calculated_at = NOW()
`, [config.abc_a_threshold, config.abc_b_threshold]);
`, [abcThresholds.a_threshold, abcThresholds.b_threshold]);
// Update progress for time-based aggregates
if (isCancelled) {
throw new Error('Operation cancelled');
}
outputProgress({
status: 'running',
operation: 'Calculating time-based aggregates',
current: totalProducts,
operation: 'Starting time-based aggregates calculation',
current: Math.floor(totalProducts * 0.85),
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, totalProducts, totalProducts),
rate: calculateRate(startTime, totalProducts),
percentage: '100'
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.85), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.85)),
percentage: '85'
});
// Calculate time-based aggregates
if (isCancelled) {
throw new Error('Operation cancelled');
}
await connection.query('TRUNCATE TABLE product_time_aggregates;');
outputProgress({
status: 'running',
operation: 'Calculating sales aggregates',
current: Math.floor(totalProducts * 0.9),
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.9), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.9)),
percentage: '90'
});
await connection.query(`
INSERT INTO product_time_aggregates (
product_id,
@@ -677,10 +806,9 @@ async function calculateMetrics() {
WHERE s.product_id IS NULL
`);
// Update progress for vendor metrics
outputProgress({
status: 'running',
operation: 'Calculating vendor metrics',
operation: 'Time-based aggregates complete',
current: totalProducts,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
@@ -689,7 +817,25 @@ async function calculateMetrics() {
percentage: '100'
});
// Calculate vendor metrics
// Update progress for vendor metrics
if (isCancelled) {
throw new Error('Operation cancelled');
}
outputProgress({
status: 'running',
operation: 'Starting vendor metrics calculation',
current: Math.floor(totalProducts * 0.95),
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.95), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.95)),
percentage: '95'
});
// Calculate vendor metrics with fixed order fill rate calculation
if (isCancelled) {
throw new Error('Operation cancelled');
}
await connection.query(`
INSERT INTO vendor_metrics (
vendor,
@@ -704,8 +850,14 @@ async function calculateMetrics() {
vendor,
NOW() as last_calculated_at,
COALESCE(AVG(DATEDIFF(received_date, date)), 0) as avg_lead_time_days,
COALESCE((COUNT(CASE WHEN DATEDIFF(received_date, date) <= 14 THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0)), 0) as on_time_delivery_rate,
COALESCE((SUM(received) * 100.0 / NULLIF(SUM(ordered), 0)), 0) as order_fill_rate,
COALESCE(
(COUNT(CASE WHEN DATEDIFF(received_date, date) <= 14 THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0)),
0
) as on_time_delivery_rate,
CASE
WHEN SUM(ordered) = 0 THEN 0
ELSE LEAST(100, GREATEST(0, (SUM(CASE WHEN received >= 0 THEN received ELSE 0 END) * 100.0 / SUM(ordered))))
END as order_fill_rate,
COUNT(DISTINCT po_id) as total_orders,
COUNT(CASE WHEN DATEDIFF(received_date, date) > 14 THEN 1 END) as total_late_orders
FROM purchase_orders
@@ -720,6 +872,17 @@ async function calculateMetrics() {
total_late_orders = VALUES(total_late_orders)
`);
outputProgress({
status: 'running',
operation: 'Vendor metrics complete',
current: Math.floor(totalProducts * 0.98),
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, Math.floor(totalProducts * 0.98), totalProducts),
rate: calculateRate(startTime, Math.floor(totalProducts * 0.98)),
percentage: '98'
});
// Final success message
outputProgress({
status: 'complete',
@@ -732,13 +895,16 @@ async function calculateMetrics() {
percentage: '100'
});
// Clear progress file on successful completion
clearProgress();
} catch (error) {
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Calculation cancelled',
current: processedCount,
total: totalProducts || 0, // Use 0 if not yet defined
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
@@ -749,7 +915,7 @@ async function calculateMetrics() {
status: 'error',
operation: 'Error: ' + error.message,
current: processedCount,
total: totalProducts || 0, // Use 0 if not yet defined
total: totalProducts || 0,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
@@ -767,9 +933,10 @@ async function calculateMetrics() {
}
}
// Export both functions
// Export both functions and progress checker
module.exports = calculateMetrics;
module.exports.cancelCalculation = cancelCalculation;
module.exports.getProgress = getProgress;
// Run directly if called from command line
if (require.main === module) {