Fix time out error on data import and fix regression on progress display + sort out core and metrics schemas better

inventory-server/db/metrics-schema.sql

@@ -108,3 +108,23 @@ LEFT JOIN
     product_metrics pm ON p.product_id = pm.product_id
 WHERE 
     p.managing_stock = true;
+
+-- Create view for sales trends analysis
+CREATE OR REPLACE VIEW product_sales_trends AS
+SELECT 
+    p.product_id,
+    p.SKU,
+    p.title,
+    COALESCE(SUM(o.quantity), 0) as total_sold,
+    COALESCE(AVG(o.quantity), 0) as avg_quantity_per_order,
+    COALESCE(COUNT(DISTINCT o.order_number), 0) as number_of_orders,
+    MIN(o.date) as first_sale_date,
+    MAX(o.date) as last_sale_date
+FROM 
+    products p
+LEFT JOIN 
+    orders o ON p.product_id = o.product_id
+WHERE 
+    o.canceled = false
+GROUP BY 
+    p.product_id, p.SKU, p.title; 

inventory-server/db/schema.sql

@@ -110,21 +110,4 @@ CREATE TABLE purchase_orders (
 SET FOREIGN_KEY_CHECKS = 1;
 
 -- Create views for common calculations
-CREATE OR REPLACE VIEW product_sales_trends AS
+-- product_sales_trends view moved to metrics-schema.sql 
-SELECT 
-    p.product_id,
-    p.SKU,
-    p.title,
-    COALESCE(SUM(o.quantity), 0) as total_sold,
-    COALESCE(AVG(o.quantity), 0) as avg_quantity_per_order,
-    COALESCE(COUNT(DISTINCT o.order_number), 0) as number_of_orders,
-    MIN(o.date) as first_sale_date,
-    MAX(o.date) as last_sale_date
-FROM 
-    products p
-LEFT JOIN 
-    orders o ON p.product_id = o.product_id
-WHERE 
-    o.canceled = false
-GROUP BY 
-    p.product_id, p.SKU, p.title; 

inventory-server/scripts/import-csv.js

@@ -54,20 +54,23 @@ function logImport(message) {
     fs.appendFileSync(IMPORT_LOG, logMessage);
 }
 
-// Helper function to output progress in JSON format
+// Helper function to format duration
+function formatDuration(seconds) {
+    const hours = Math.floor(seconds / 3600);
+    const minutes = Math.floor((seconds % 3600) / 60);
+    seconds = Math.floor(seconds % 60);
+    
+    const parts = [];
+    if (hours > 0) parts.push(`${hours}h`);
+    if (minutes > 0) parts.push(`${minutes}m`);
+    if (seconds > 0 || parts.length === 0) parts.push(`${seconds}s`);
+    
+    return parts.join(' ');
+}
+
+// Helper function to output progress
 function outputProgress(data) {
-    if (!data.status) {
+    process.stdout.write(JSON.stringify(data) + '\n');
-        data = {
-            status: 'running',
-            ...data
-        };
-    }
-    
-    // Log progress to import log
-    logImport(JSON.stringify(data));
-    
-    // Output to console
-    console.log(JSON.stringify(data));
 }
 
 // Helper function to count total rows in a CSV file
@@ -82,14 +85,6 @@ async function countRows(filePath) {
     });
 }
 
-// Helper function to format time duration
-function formatDuration(seconds) {
-    if (seconds < 60) return `${Math.round(seconds)}s`;
-    const minutes = Math.floor(seconds / 60);
-    seconds = Math.round(seconds % 60);
-    return `${minutes}m ${seconds}s`;
-}
-
 // Helper function to update progress with time estimate
 function updateProgress(current, total, operation, startTime) {
     const elapsed = (Date.now() - startTime) / 1000;
@@ -401,118 +396,6 @@ async function calculateVendorMetrics(connection) {
     }
 }
 
-// Helper function to calculate metrics in batches
-async function calculateMetricsInBatch(connection) {
-    try {
-        // Clear temporary tables
-        await connection.query('TRUNCATE TABLE temp_sales_metrics');
-        await connection.query('TRUNCATE TABLE temp_purchase_metrics');
-
-        // Calculate sales metrics for all products in one go
-        await connection.query(`
-            INSERT INTO temp_sales_metrics
-            SELECT 
-                o.product_id,
-                COUNT(*) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) as daily_sales_avg,
-                SUM(o.quantity) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) * 7 as weekly_sales_avg,
-                SUM(o.quantity) / NULLIF(DATEDIFF(MAX(o.date), MIN(o.date)), 0) * 30 as monthly_sales_avg,
-                SUM(o.price * o.quantity) as total_revenue,
-                AVG((o.price - p.cost_price) / o.price * 100) as avg_margin_percent,
-                MIN(o.date) as first_sale_date,
-                MAX(o.date) as last_sale_date
-            FROM orders o
-            JOIN products p ON o.product_id = p.product_id
-            WHERE o.canceled = false
-            GROUP BY o.product_id
-        `);
-
-        // Calculate purchase metrics for all products in one go
-        await connection.query(`
-            INSERT INTO temp_purchase_metrics
-            SELECT 
-                product_id,
-                AVG(DATEDIFF(received_date, date)) as avg_lead_time_days,
-                MAX(date) as last_purchase_date,
-                MAX(received_date) as last_received_date
-            FROM purchase_orders
-            WHERE status = 'closed'
-            GROUP BY product_id
-        `);
-
-        // Update product_metrics table with all metrics at once
-        await connection.query(`
-            INSERT INTO product_metrics (
-                product_id, daily_sales_avg, weekly_sales_avg, monthly_sales_avg,
-                days_of_inventory, weeks_of_inventory, safety_stock, reorder_point,
-                avg_margin_percent, total_revenue, avg_lead_time_days,
-                last_purchase_date, last_received_date
-            )
-            SELECT 
-                p.product_id,
-                COALESCE(s.daily_sales_avg, 0),
-                COALESCE(s.weekly_sales_avg, 0),
-                COALESCE(s.monthly_sales_avg, 0),
-                CASE 
-                    WHEN s.daily_sales_avg > 0 THEN FLOOR(p.stock_quantity / s.daily_sales_avg)
-                    ELSE 999
-                END as days_of_inventory,
-                CASE 
-                    WHEN s.daily_sales_avg > 0 THEN FLOOR(p.stock_quantity / s.daily_sales_avg / 7)
-                    ELSE 999
-                END as weeks_of_inventory,
-                CEIL(COALESCE(s.daily_sales_avg, 0) * 14) as safety_stock,
-                CEIL(COALESCE(s.daily_sales_avg, 0) * 21) as reorder_point,
-                COALESCE(s.avg_margin_percent, 0),
-                COALESCE(s.total_revenue, 0),
-                COALESCE(pm.avg_lead_time_days, 0),
-                pm.last_purchase_date,
-                pm.last_received_date
-            FROM products p
-            LEFT JOIN temp_sales_metrics s ON p.product_id = s.product_id
-            LEFT JOIN temp_purchase_metrics pm ON p.product_id = pm.product_id
-            ON DUPLICATE KEY UPDATE
-                daily_sales_avg = VALUES(daily_sales_avg),
-                weekly_sales_avg = VALUES(weekly_sales_avg),
-                monthly_sales_avg = VALUES(monthly_sales_avg),
-                days_of_inventory = VALUES(days_of_inventory),
-                weeks_of_inventory = VALUES(weeks_of_inventory),
-                safety_stock = VALUES(safety_stock),
-                reorder_point = VALUES(reorder_point),
-                avg_margin_percent = VALUES(avg_margin_percent),
-                total_revenue = VALUES(total_revenue),
-                avg_lead_time_days = VALUES(avg_lead_time_days),
-                last_purchase_date = VALUES(last_purchase_date),
-                last_received_date = VALUES(last_received_date),
-                last_calculated_at = CURRENT_TIMESTAMP
-        `);
-
-        // Calculate ABC classification in one go
-        await connection.query(`
-            WITH revenue_ranks AS (
-                SELECT 
-                    product_id,
-                    total_revenue,
-                    total_revenue / SUM(total_revenue) OVER () * 100 as revenue_percent,
-                    ROW_NUMBER() OVER (ORDER BY total_revenue DESC) as rank
-                FROM product_metrics
-                WHERE total_revenue > 0
-            )
-            UPDATE product_metrics pm
-            JOIN revenue_ranks r ON pm.product_id = r.product_id
-            SET abc_class = 
-                CASE 
-                    WHEN r.revenue_percent >= 20 THEN 'A'
-                    WHEN r.revenue_percent >= 5 THEN 'B'
-                    ELSE 'C'
-                END
-        `);
-
-    } catch (error) {
-        logError(error, 'Error in batch metrics calculation');
-        throw error;
-    }
-}
-
 async function importProducts(pool, filePath) {
     const parser = fs.createReadStream(filePath).pipe(csv.parse({ columns: true, trim: true }));
     const totalRows = PRODUCTS_TEST_LIMIT > 0 ? Math.min(await countRows(filePath), PRODUCTS_TEST_LIMIT) : await countRows(filePath);
@@ -1085,7 +968,7 @@ async function main() {
             connection.release();
         }
 
-        // Step 1: Import all data first
+        // Import all data
         try {
             // Import products first since they're referenced by other tables
             await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv'));
@@ -1101,34 +984,16 @@ async function main() {
                 importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'))
             ]);
 
-            // Step 2: Calculate all metrics after imports are complete
             outputProgress({
-                operation: 'Starting metrics calculation',
+                status: 'complete',
-                message: 'Calculating metrics for all products and vendors...'
+                operation: 'Import process completed',
+                duration: formatDuration((Date.now() - startTime) / 1000)
             });
 
-            const connection = await pool.getConnection();
-            try {
-                // Calculate metrics in batches
-                await calculateMetricsInBatch(connection);
-
-                // Calculate vendor metrics
-                await calculateVendorMetrics(connection);
-
-            } finally {
-                connection.release();
-            }
-
         } catch (error) {
-            logError(error, 'Error during import/metrics calculation');
+            logError(error, 'Error during import');
             throw error;
         }
-        
-        outputProgress({
-            status: 'complete',
-            operation: 'Import process completed',
-            duration: formatDuration((Date.now() - startTime) / 1000)
-        });
     } catch (error) {
         logError(error, 'Fatal error during import process');
         outputProgress({

inventory-server/scripts/reset-metrics.js

@@ -84,6 +84,14 @@ async function resetMetrics() {
         // Disable foreign key checks first
         await connection.query('SET FOREIGN_KEY_CHECKS = 0');
 
+        // Drop the metrics views first
+        outputProgress({
+            status: 'running',
+            operation: 'Dropping metrics views',
+            percentage: '15'
+        });
+        await connection.query('DROP VIEW IF EXISTS inventory_health, product_sales_trends');
+
         // Drop only the metrics tables if they exist
         const [existing] = await connection.query(`
             SELECT GROUP_CONCAT(table_name) as tables 

inventory/src/pages/Settings.tsx

@@ -94,119 +94,102 @@ export function Settings() {
       setEventSource(null);
     }
 
-    try {
+    let retryCount = 0;
-      console.log(`Creating new EventSource for ${config.apiUrl}/csv/${type}/progress`);
+    const MAX_RETRIES = 3;
-      const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
+    const RETRY_DELAY = 2000; // 2 seconds
-        withCredentials: true
-      });
 
-      // Set up handlers before setting state
+    const setupConnection = () => {
-      source.onopen = () => {
+      try {
-        console.log('EventSource connected successfully');
+        console.log(`Creating new EventSource for ${config.apiUrl}/csv/${type}/progress`);
-        // Set event source state only after successful connection
+        const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
-        setEventSource(source);
+          withCredentials: true
-      };
+        });
         
-      source.onerror = (event) => {
+        source.onopen = () => {
-        console.error('EventSource failed:', event);
+          console.log('EventSource connected successfully');
-        source.close();
+          retryCount = 0; // Reset retry count on successful connection
+          setEventSource(source);
+        };
 
-        // Reset states based on type
+        source.onerror = async (event) => {
-        switch (type) {
+          console.error('EventSource error:', event);
-          case 'update':
+          source.close();
-            setIsUpdating(false);
-            setUpdateProgress(null);
-            break;
-          case 'import':
-            setIsImporting(false);
-            setImportProgress(null);
-            break;
-          case 'reset':
-            setIsResetting(false);
-            setResetProgress(null);
-            break;
-          case 'reset-metrics':
-            setIsResettingMetrics(false);
-            setResetMetricsProgress(null);
-            break;
-          case 'calculate-metrics':
-            setIsCalculatingMetrics(false);
-            setMetricsProgress(null);
-            break;
-        }
           
-        setEventSource(null);
+          // Only retry if we haven't exceeded max retries and we're still in the active state
-        handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Lost connection to server');
+          const isActive = type === 'import' ? isImporting : 
-      };
+                          type === 'update' ? isUpdating :
+                          type === 'reset' ? isResetting :
+                          type === 'reset-metrics' ? isResettingMetrics :
+                          type === 'calculate-metrics' ? isCalculatingMetrics : false;
 
-      source.onmessage = (event) => {
+          if (retryCount < MAX_RETRIES && isActive) {
-        try {
+            console.log(`Retrying connection (${retryCount + 1}/${MAX_RETRIES})...`);
-          console.log(`Received message for ${type}:`, event.data);
+            retryCount++;
-          
+            // Wait before retrying
-          // First parse the outer message
+            await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
-          const data = JSON.parse(event.data);
+            setupConnection();
-          
+          } else if (retryCount >= MAX_RETRIES) {
-          // If we have a progress field that's a string containing multiple JSON objects
+            console.log('Max retries exceeded, but operation may still be running...');
-          if (data.progress && typeof data.progress === 'string') {
+            // Don't reset states or show error, just log it
-            // Split the progress string into separate JSON objects
+            console.warn(`Lost connection to ${type} progress stream after ${MAX_RETRIES} retries`);
-            const progressMessages = data.progress
-              .split('\n')
-              .filter(Boolean)
-              .map((message: string) => {
-                try {
-                  return JSON.parse(message);
-                } catch (err) {
-                  console.warn('Failed to parse progress message:', message, err);
-                  return null;
-                }
-              })
-              .filter((msg: unknown): msg is ImportProgress => msg !== null);
-
-            // Process each progress message
-            progressMessages.forEach((progressData: ImportProgress) => {
-              handleProgressUpdate(type, progressData, source);
-            });
-          } else {
-            // Handle single message case
-            const progressData = data.progress || data;
-            handleProgressUpdate(type, progressData, source);
           }
-        } catch (error) {
+          
-          console.error('Error parsing event data:', error, event.data);
+          setEventSource(null);
-          handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Failed to parse server response');
+        };
+
+        source.onmessage = (event) => {
+          try {
+            console.log(`Received message for ${type}:`, event.data);
+            
+            // First parse the outer message
+            const data = JSON.parse(event.data);
+            
+            // If we have a progress field that's a string containing multiple JSON objects
+            if (data.progress && typeof data.progress === 'string') {
+              // Split the progress string into separate JSON objects
+              const progressMessages = data.progress
+                .split('\n')
+                .filter(Boolean)
+                .map((message: string) => {
+                  try {
+                    return JSON.parse(message);
+                  } catch (err) {
+                    console.warn('Failed to parse progress message:', message, err);
+                    return null;
+                  }
+                })
+                .filter((msg: unknown): msg is ImportProgress => msg !== null);
+
+              // Process each progress message
+              progressMessages.forEach((progressData: ImportProgress) => {
+                handleProgressUpdate(type, progressData, source);
+              });
+            } else {
+              // Handle single message case
+              const progressData = data.progress || data;
+              handleProgressUpdate(type, progressData, source);
+            }
+          } catch (error) {
+            console.error('Error parsing event data:', error, event.data);
+            // Don't show error to user for parsing issues, just log them
+            console.warn('Failed to parse server response:', error);
+          }
+        };
+
+      } catch (error) {
+        console.error('Failed to set up EventSource:', error);
+        if (retryCount < MAX_RETRIES) {
+          console.log(`Retrying connection (${retryCount + 1}/${MAX_RETRIES})...`);
+          retryCount++;
+          setTimeout(setupConnection, RETRY_DELAY);
+        } else {
+          console.log('Max retries exceeded, but operation may still be running...');
         }
-      };
-
-    } catch (error) {
-      console.error('Failed to set up EventSource:', error);
-      
-      // Reset operation state
-      switch (type) {
-        case 'update':
-          setIsUpdating(false);
-          setUpdateProgress(null);
-          break;
-        case 'import':
-          setIsImporting(false);
-          setImportProgress(null);
-          break;
-        case 'reset':
-          setIsResetting(false);
-          setResetProgress(null);
-          break;
-        case 'reset-metrics':
-          setIsResettingMetrics(false);
-          setResetMetricsProgress(null);
-          break;
-        case 'calculate-metrics':
-          setIsCalculatingMetrics(false);
-          setMetricsProgress(null);
-          break;
       }
+    };
 
-      handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Failed to connect to server');
+    setupConnection();
-    }
+  }, [eventSource, handleComplete, handleError, isImporting, isUpdating, isResetting, isResettingMetrics, isCalculatingMetrics]);
-  }, [eventSource, handleComplete, handleError]);
 
   // Helper function to process a single progress update
   const handleProgressUpdate = (
@@ -225,6 +208,18 @@ export function Settings() {
       status: progressData.status || 'running'
     };
 
+    // For import type, handle orders and purchase orders separately
+    if (type === 'import' && progressData.operation) {
+      const operation = progressData.operation.toLowerCase();
+      if (operation.includes('purchase orders')) {
+        setPurchaseOrdersProgress(prev => ({
+          ...prev,
+          ...processedData
+        }));
+        return; // Don't update main import progress for PO updates
+      }
+    }
+
     // Update progress state based on type
     switch (type) {
       case 'update':
@@ -261,6 +256,29 @@ export function Settings() {
     // Handle completion
     if (progressData.status === 'complete') {
       console.log(`Operation ${type} completed`);
+      
+      // For import, only close connection when both operations are complete
+      if (type === 'import') {
+        const operation = progressData.operation?.toLowerCase() || '';
+        if (operation.includes('purchase orders')) {
+          setPurchaseOrdersProgress(null);
+        } else {
+          setImportProgress(null);
+        }
+        
+        // Only fully complete if both are done
+        if (!importProgress || !purchaseOrdersProgress) {
+          source.close();
+          setEventSource(null);
+          setIsImporting(false);
+          if (!progressData.operation?.includes('cancelled')) {
+            handleComplete(`${type.charAt(0).toUpperCase() + type.slice(1)}`);
+          }
+        }
+        return;
+      }
+      
+      // For other operations, close immediately
       source.close();
       setEventSource(null);
       
@@ -270,11 +288,6 @@ export function Settings() {
           setIsUpdating(false);
           setUpdateProgress(null);
           break;
-        case 'import':
-          setIsImporting(false);
-          setImportProgress(null);
-          setPurchaseOrdersProgress(null);
-          break;
         case 'reset':
           setIsResetting(false);
           setResetProgress(null);
@@ -306,6 +319,8 @@ export function Settings() {
           break;
         case 'import':
           setIsImporting(false);
+          setImportProgress(null);
+          setPurchaseOrdersProgress(null);
           break;
         case 'reset':
           setIsResetting(false);
@@ -489,7 +504,7 @@ export function Settings() {
       // Connect to SSE for progress updates first
       connectToEventSource('import');
 
-      // Make the import request
+      // Make the import request - no timeout needed since this just starts the process
       const response = await fetch(`${config.apiUrl}/csv/import`, {
         method: 'POST',
         headers: {
@@ -497,8 +512,22 @@ export function Settings() {
         },
         credentials: 'include',
         body: JSON.stringify(limits)
+      }).catch(error => {
+        // If we already have progress, ignore network errors
+        if ((importProgress?.current || purchaseOrdersProgress?.current) && 
+            (error.name === 'TypeError')) {
+          console.log('Request error but import is in progress:', error);
+          return null;
+        }
+        throw error;
       });
       
+      // If response is null, it means we caught an error but have progress
+      if (!response) {
+        console.log('Continuing with existing progress...');
+        return;
+      }
+      
       if (!response.ok) {
         const data = await response.json().catch(() => ({}));
         // Only handle error if we don't have any progress yet
@@ -507,17 +536,23 @@ export function Settings() {
         }
       }
     } catch (error) {
-      // Only clean up if we don't have any progress
+      console.log('Import error:', error);
-      if (!importProgress?.current && !purchaseOrdersProgress?.current) {
+      
-        if (eventSource) {
+      // If we have any progress, assume the import is still running
-          eventSource.close();
+      if (importProgress?.current || purchaseOrdersProgress?.current) {
-          setEventSource(null);
+        console.log('Error occurred but import appears to be running');
-        }
+        return;
-        setIsImporting(false);
-        setImportProgress(null);
-        setPurchaseOrdersProgress(null);
-        handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
       }
+
+      // Only clean up if we don't have any progress
+      if (eventSource) {
+        eventSource.close();
+        setEventSource(null);
+      }
+      setIsImporting(false);
+      setImportProgress(null);
+      setPurchaseOrdersProgress(null);
+      handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
     }
   };
 
@@ -783,12 +818,12 @@ export function Settings() {
               <Button 
                 className="flex-1"
                 onClick={handleImportCSV}
-                disabled={isUpdating || isImporting}
+                disabled={isImporting || isUpdating || isResetting}
               >
                 {isImporting ? (
                   <>
                     <Loader2 className="mr-2 h-4 w-4 animate-spin" />
-                    Importing...
+                    Importing Data...
                   </>
                 ) : (
                   <>
@@ -810,49 +845,8 @@ export function Settings() {
 
             {isImporting && (
               <div className="space-y-4">
-                {/* Show products progress */}
+                {importProgress && renderProgress(importProgress)}
-                {importProgress?.operation?.toLowerCase().includes('products') && (
+                {purchaseOrdersProgress && renderProgress(purchaseOrdersProgress)}
-                  <div>
-                    {renderProgress(importProgress)}
-                  </div>
-                )}
-                {/* Show orders progress */}
-                {importProgress?.operation?.toLowerCase().includes('orders import') && 
-                 !importProgress.operation.toLowerCase().includes('purchase') && (
-                  <div>
-                    {renderProgress(importProgress)}
-                  </div>
-                )}
-                {/* Show purchase orders progress */}
-                {purchaseOrdersProgress && (
-                  <div>
-                    {renderProgress(purchaseOrdersProgress)}
-                  </div>
-                )}
-                {/* Show metrics calculation progress */}
-                {importProgress?.operation?.toLowerCase().includes('metrics') && (
-                  <div>
-                    <Progress value={Number(importProgress.percentage)} className="mb-2" />
-                    <p className="text-sm text-muted-foreground">
-                      {importProgress.message || importProgress.operation || 'Calculating metrics...'}
-                      {importProgress.current && importProgress.total && (
-                        <> ({importProgress.current} of {importProgress.total})</>
-                      )}
-                    </p>
-                    {importProgress.elapsed && (
-                      <p className="text-xs text-muted-foreground">
-                        Elapsed: {importProgress.elapsed}
-                        {importProgress.remaining && <> • Remaining: {importProgress.remaining}</>}
-                      </p>
-                    )}
-                  </div>
-                )}
-                {/* Show vendor metrics progress */}
-                {importProgress?.operation?.toLowerCase().includes('vendor metrics') && (
-                  <div>
-                    {renderProgress(importProgress)}
-                  </div>
-                )}
               </div>
             )}
           </CardContent>