Clean up, fix file path issues with import scripts, adjust data management page for new metrics calcs

inventory-server/db/config-schema-new.sql

@@ -111,3 +111,53 @@ FROM
     public.products p
 
 ON CONFLICT (pid) DO NOTHING; -- IMPORTANT: Do not overwrite existing product-specific settings
+
+
+-- History and status tables
+CREATE TABLE IF NOT EXISTS calculate_history (
+    id BIGSERIAL PRIMARY KEY,
+    start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    end_time TIMESTAMP WITH TIME ZONE NULL,
+    duration_seconds INTEGER,
+    duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
+    total_products INTEGER DEFAULT 0,
+    total_orders INTEGER DEFAULT 0,
+    total_purchase_orders INTEGER DEFAULT 0,
+    processed_products INTEGER DEFAULT 0,
+    processed_orders INTEGER DEFAULT 0,
+    processed_purchase_orders INTEGER DEFAULT 0,
+    status calculation_status DEFAULT 'running',
+    error_message TEXT,
+    additional_info JSONB
+);
+
+CREATE TABLE IF NOT EXISTS calculate_status (
+    module_name module_name PRIMARY KEY,
+    last_calculation_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS sync_status (
+    table_name TEXT PRIMARY KEY,
+    last_sync_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    last_sync_id BIGINT
+);
+
+CREATE TABLE IF NOT EXISTS import_history (
+    id BIGSERIAL PRIMARY KEY,
+    table_name VARCHAR(50) NOT NULL,
+    start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    end_time TIMESTAMP WITH TIME ZONE NULL,
+    duration_seconds INTEGER,
+    duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
+    records_added INTEGER DEFAULT 0,
+    records_updated INTEGER DEFAULT 0,
+    is_incremental BOOLEAN DEFAULT FALSE,
+    status calculation_status DEFAULT 'running',
+    error_message TEXT,
+    additional_info JSONB
+);
+
+-- Create all indexes after tables are fully created
+CREATE INDEX IF NOT EXISTS idx_last_calc ON calculate_status(last_calculation_timestamp);
+CREATE INDEX IF NOT EXISTS idx_last_sync ON sync_status(last_sync_timestamp);
+CREATE INDEX IF NOT EXISTS idx_table_time ON import_history(table_name, start_time); 

inventory-server/old/config-schema.sql

@@ -256,52 +256,3 @@
 --     END,
 --     c.name,
 --     st.vendor;
-
--- History and status tables
-CREATE TABLE IF NOT EXISTS calculate_history (
-    id BIGSERIAL PRIMARY KEY,
-    start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    end_time TIMESTAMP WITH TIME ZONE NULL,
-    duration_seconds INTEGER,
-    duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
-    total_products INTEGER DEFAULT 0,
-    total_orders INTEGER DEFAULT 0,
-    total_purchase_orders INTEGER DEFAULT 0,
-    processed_products INTEGER DEFAULT 0,
-    processed_orders INTEGER DEFAULT 0,
-    processed_purchase_orders INTEGER DEFAULT 0,
-    status calculation_status DEFAULT 'running',
-    error_message TEXT,
-    additional_info JSONB
-);
-
-CREATE TABLE IF NOT EXISTS calculate_status (
-    module_name module_name PRIMARY KEY,
-    last_calculation_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE TABLE IF NOT EXISTS sync_status (
-    table_name TEXT PRIMARY KEY,
-    last_sync_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    last_sync_id BIGINT
-);
-
-CREATE TABLE IF NOT EXISTS import_history (
-    id BIGSERIAL PRIMARY KEY,
-    table_name VARCHAR(50) NOT NULL,
-    start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    end_time TIMESTAMP WITH TIME ZONE NULL,
-    duration_seconds INTEGER,
-    duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
-    records_added INTEGER DEFAULT 0,
-    records_updated INTEGER DEFAULT 0,
-    is_incremental BOOLEAN DEFAULT FALSE,
-    status calculation_status DEFAULT 'running',
-    error_message TEXT,
-    additional_info JSONB
-);
-
--- Create all indexes after tables are fully created
-CREATE INDEX IF NOT EXISTS idx_last_calc ON calculate_status(last_calculation_timestamp);
-CREATE INDEX IF NOT EXISTS idx_last_sync ON sync_status(last_sync_timestamp);
-CREATE INDEX IF NOT EXISTS idx_table_time ON import_history(table_name, start_time); 

inventory-server/scripts/calculate-metrics-new.js

@@ -246,6 +246,89 @@ process.on('unhandledRejection', (reason, promise) => {
 
 // --- Core Logic ---
 
+/**
+ * Ensures all products have entries in the settings_product table
+ * This is important after importing new products
+ */
+async function syncSettingsProductTable() {
+    let conn = null;
+    try {
+        currentStep = 'Syncing settings_product table';
+        progressUtils.outputProgress({
+            operation: 'Syncing product settings',
+            message: 'Ensuring all products have settings entries'
+        });
+
+        conn = await getConnection();
+        
+        // Get counts before sync
+        const beforeCounts = await conn.query(`
+            SELECT 
+                (SELECT COUNT(*) FROM products) AS products_count,
+                (SELECT COUNT(*) FROM settings_product) AS settings_count
+        `);
+        
+        const productsCount = parseInt(beforeCounts.rows[0].products_count);
+        const settingsCount = parseInt(beforeCounts.rows[0].settings_count);
+        
+        progressUtils.outputProgress({
+            operation: 'Settings product sync',
+            message: `Found ${productsCount} products and ${settingsCount} settings entries`
+        });
+        
+        // Insert missing product settings
+        const result = await conn.query(`
+            INSERT INTO settings_product (
+                pid,
+                lead_time_days,
+                days_of_stock,
+                safety_stock,
+                forecast_method,
+                exclude_from_forecast
+            )
+            SELECT
+                p.pid,
+                CAST(NULL AS INTEGER),
+                CAST(NULL AS INTEGER),
+                COALESCE((SELECT setting_value::int FROM settings_global WHERE setting_key = 'default_safety_stock_units'), 0),
+                CAST(NULL AS VARCHAR),
+                FALSE
+            FROM
+                public.products p
+            WHERE
+                NOT EXISTS (
+                    SELECT 1 FROM settings_product sp WHERE sp.pid = p.pid
+                )
+            ON CONFLICT (pid) DO NOTHING
+        `);
+        
+        // Get counts after sync
+        const afterCounts = await conn.query(`
+            SELECT COUNT(*) AS settings_count FROM settings_product
+        `);
+        
+        const newSettingsCount = parseInt(afterCounts.rows[0].settings_count);
+        const addedCount = newSettingsCount - settingsCount;
+        
+        progressUtils.outputProgress({
+            operation: 'Settings product sync',
+            message: `Added ${addedCount} new settings entries. Now have ${newSettingsCount} total entries.`,
+            status: 'complete'
+        });
+        
+        conn.release();
+        return addedCount;
+    } catch (err) {
+        progressUtils.outputProgress({
+            status: 'error',
+            operation: 'Settings product sync failed',
+            error: err.message
+        });
+        if (conn) conn.release();
+        throw err;
+    }
+}
+
 /**
  * Executes a single SQL calculation step.
  * @param {object} config - Configuration for the step.
@@ -552,6 +635,26 @@ async function runAllCalculations() {
     let overallSuccess = true;
 
     try {
+        // First, sync the settings_product table to ensure all products have entries
+        progressUtils.outputProgress({
+            operation: 'Starting metrics calculation',
+            message: 'Preparing product settings...'
+        });
+        
+        try {
+            const addedCount = await syncSettingsProductTable();
+            
+            progressUtils.outputProgress({
+                operation: 'Preparation complete',
+                message: `Added ${addedCount} missing product settings entries`,
+                status: 'complete'
+            });
+        } catch (syncError) {
+            console.error('Warning: Failed to sync product settings, continuing with metrics calculations:', syncError);
+            // Don't fail the entire process if settings sync fails
+        }
+
+        // Now run the calculation steps
         for (const step of steps) {
             if (step.run) {
                 if (isCancelled) {
@@ -599,6 +702,7 @@ if (require.main === module) {
     module.exports = {
         runAllCalculations,
         cancelCalculation,
+        syncSettingsProductTable,
         // Expose individual steps if useful, wrapping them slightly
         runDailySnapshots: () => executeSqlStep({ name: 'Daily Snapshots Update', sqlFile: 'update_daily_snapshots.sql', historyType: 'daily_snapshots', statusModule: 'daily_snapshots' }, progressUtils),
         runProductMetrics: () => executeSqlStep({ name: 'Product Metrics Update', sqlFile: 'update_product_metrics.sql', historyType: 'product_metrics', statusModule: 'product_metrics' }, progressUtils),

inventory-server/scripts/full-reset.js

@@ -88,7 +88,7 @@ async function fullReset() {
             operation: 'Starting metrics calculation',
             message: 'Step 3/3: Calculating metrics...'
         });
-        await runScript(path.join(__dirname, 'calculate-metrics.js'));
+        await runScript(path.join(__dirname, 'calculate-metrics-new.js'));
         
         // Final completion message
         outputProgress({

inventory-server/scripts/full-update.js

@@ -68,7 +68,7 @@ async function fullUpdate() {
             operation: 'Starting metrics calculation',
             message: 'Step 2/2: Calculating metrics...'
         });
-        await runScript(path.join(__dirname, 'calculate-metrics.js'));
+        await runScript(path.join(__dirname, 'calculate-metrics-new.js'));
         outputProgress({
             status: 'complete',
             operation: 'Metrics step complete',

inventory-server/scripts/import-from-prod.js

@@ -1,6 +1,6 @@
 const dotenv = require("dotenv");
 const path = require("path");
-const { outputProgress, formatElapsedTime } = require('./metrics/utils/progress');
+const { outputProgress, formatElapsedTime } = require('./metrics-new/utils/progress');
 const { setupConnections, closeConnections } = require('./import/utils');
 const importCategories = require('./import/categories');
 const { importProducts } = require('./import/products');

inventory-server/scripts/import/categories.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime } = require('../metrics/utils/progress');
+const { outputProgress, formatElapsedTime } = require('../metrics-new/utils/progress');
 
 async function importCategories(prodConnection, localConnection) {
   outputProgress({

inventory-server/scripts/import/historical-data.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
+const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
 const fs = require('fs');
 const path = require('path');
 const { pipeline } = require('stream');

inventory-server/scripts/import/orders.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
+const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
 const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
 
 /**

inventory-server/scripts/import/products.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
+const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
 const BATCH_SIZE = 1000; // Smaller batch size for better progress tracking
 const MAX_RETRIES = 3;
 const RETRY_DELAY = 5000; // 5 seconds

inventory-server/scripts/import/purchase-orders.js

@@ -1,4 +1,4 @@
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
+const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
 
 /**
  * Validates a date from MySQL before inserting it into PostgreSQL

inventory-server/scripts/reset-db.js

@@ -384,7 +384,7 @@ async function resetDatabase() {
             message: 'Creating configuration tables...'
         });
         const configSchemaSQL = fs.readFileSync(
-            path.join(__dirname, '../db/config-schema.sql'),
+            path.join(__dirname, '../db/config-schema-new.sql'),
             'utf8'
         );
         
@@ -433,7 +433,7 @@ async function resetDatabase() {
             message: 'Creating metrics tables...'
         });
         const metricsSchemaSQL = fs.readFileSync(
-            path.join(__dirname, '../db/metrics-schema.sql'),
+            path.join(__dirname, '../db/metrics-schema-new.sql'),
             'utf8'
         );
         

inventory-server/src/routes/csv.js

@@ -843,14 +843,10 @@ router.get('/status/table-counts', async (req, res) => {
         const tables = [
             // Core tables
             'products', 'categories', 'product_categories', 'orders', 'purchase_orders',
-            // Metrics tables
+            // New metrics tables
-            'product_metrics', 'product_time_aggregates', 'vendor_metrics', 'category_metrics',
+            'product_metrics', 'daily_product_snapshots',
-            'vendor_time_metrics', 'category_time_metrics', 'category_sales_metrics',
-            'brand_metrics', 'brand_time_metrics', 'sales_forecasts', 'category_forecasts',
             // Config tables
-            'stock_thresholds', 'lead_time_thresholds', 'sales_velocity_config',
+            'settings_global', 'settings_vendor', 'settings_product'
-            'abc_classification_config', 'safety_stock_config', 'turnover_config',
-            'sales_seasonality', 'financial_calc_config'
         ];
 
         const counts = await Promise.all(
@@ -871,16 +867,8 @@ router.get('/status/table-counts', async (req, res) => {
         // Group tables by type
         const groupedCounts = {
             core: counts.filter(c => ['products', 'categories', 'product_categories', 'orders', 'purchase_orders'].includes(c.table_name)),
-            metrics: counts.filter(c => [
+            metrics: counts.filter(c => ['product_metrics', 'daily_product_snapshots'].includes(c.table_name)),
-                'product_metrics', 'product_time_aggregates', 'vendor_metrics', 'category_metrics',
+            config: counts.filter(c => ['settings_global', 'settings_vendor', 'settings_product'].includes(c.table_name))
-                'vendor_time_metrics', 'category_time_metrics', 'category_sales_metrics',
-                'brand_metrics', 'brand_time_metrics', 'sales_forecasts', 'category_forecasts'
-            ].includes(c.table_name)),
-            config: counts.filter(c => [
-                'stock_thresholds', 'lead_time_thresholds', 'sales_velocity_config',
-                'abc_classification_config', 'safety_stock_config', 'turnover_config',
-                'sales_seasonality', 'financial_calc_config'
-            ].includes(c.table_name))
         };
 
         res.json(groupedCounts);

inventory/src/components/settings/DataManagement.tsx

@@ -767,11 +767,11 @@ export function DataManagement() {
     return new Intl.NumberFormat().format(num);
   };
 
-  // Update renderTableCountsSection to use skeletons
+  // Update renderTableCountsSection to show new metrics tables
   const renderTableCountsSection = () => {
     const renderTableCountsSkeleton = () => (
       <div>
-        {Array.from({ length: 18 }).map((_, i) => (
+        {Array.from({ length: 11 }).map((_, i) => (
           <div key={i} className="flex justify-between text-sm items-center py-2 border-b last:border-0">
             <Skeleton className="h-4 w-[120px]" />
             <Skeleton className="h-4 w-[60px]" />
@@ -801,12 +801,12 @@ export function DataManagement() {
     );
 
     return (
-      <Card className="md:col-start-2 md:row-span-2 h-[670px]">
+      <Card className="md:col-start-2 md:row-span-2 h-[550px]">
         <CardHeader className="pb-3">
           <CardTitle>Table Record Counts</CardTitle>
         </CardHeader>
         <CardContent>
-          {isLoading ? (
+          {isLoading && !tableCounts ? (
             <div className="px-2">
               {renderTableCountsSkeleton()}
             </div>
@@ -820,7 +820,8 @@ export function DataManagement() {
           ) : (
             <div>
               <div className="bg-sky-50/50 rounded-t-md px-2">{renderTableGroup('Core Tables', tableCounts?.core || [])}</div>
-              <div className="bg-green-50/50 rounded-b-md px-2">{renderTableGroup('Metrics Tables', tableCounts?.metrics || [])}</div>
+              <div className="bg-green-50/50 px-2">{renderTableGroup('Metrics Tables', tableCounts?.metrics || [])}</div>
+              <div className="bg-amber-50/50 rounded-b-md px-2">{renderTableGroup('Config Tables', tableCounts?.config || [])}</div>
             </div>
           )}
         </CardContent>
@@ -952,12 +953,12 @@ export function DataManagement() {
 
         <div className="grid gap-4 md:grid-cols-2">
           {/* Table Status */}
-          <div className="space-y-4 flex flex-col h-[670px]">
+          <div className="space-y-4 flex flex-col h-[550px]">
             <Card className="flex-1">
               <CardHeader className="pb-3">
                 <CardTitle>Last Import Times</CardTitle>
               </CardHeader>
-              <CardContent className="h-[calc(50%)]">
+              <CardContent className="h-auto">
                 <div className="">
                   {isLoading && !tableStatus.length ? (
                     <div>
@@ -999,11 +1000,11 @@ export function DataManagement() {
               <CardHeader className="pb-3">
                 <CardTitle>Last Calculation Times</CardTitle>
               </CardHeader>
-              <CardContent className="h-[calc(50%)]">
+              <CardContent className="h-auto">
                 <div className="">
                   {isLoading && !moduleStatus.length ? (
                     <div>
-                      {Array.from({ length: 7 }).map((_, i) => (
+                      {Array.from({ length: 3 }).map((_, i) => (
                         <div key={i} className="flex justify-between text-sm items-center py-2 border-b last:border-0">
                           <Skeleton className="h-4 w-[120px]" />
                           <Skeleton className="h-4 w-[60px]" />