More import optimizations + reset optimizations

inventory-server/scripts/import-csv.js

@@ -124,7 +124,14 @@ async function importProducts(pool, filePath) {
     });
     
     function convertDate(dateStr) {
-        if (!dateStr) return null;
+        if (!dateStr) {
+            // Default to current date for missing dates
+            const now = new Date();
+            const year = now.getFullYear();
+            const month = String(now.getMonth() + 1).padStart(2, '0');
+            const day = String(now.getDate()).padStart(2, '0');
+            return `${year}-${month}-${day}`;
+        }
         const [day, month, year] = dateStr.split('-');
         return `${year}-${month}-${day}`;
     }
@@ -268,8 +275,16 @@ async function importProducts(pool, filePath) {
                 
                 // Update stats
                 if (result.affectedRows > 0) {
-                    updated += result.affectedRows - result.insertId;
+                    // For INSERT ... ON DUPLICATE KEY UPDATE:
-                    added += result.insertId;
+                    // - If a row is inserted, affectedRows = 1
+                    // - If a row is updated, affectedRows = 2
+                    // So we can calculate:
+                    // - Number of inserts = number of rows where affectedRows = 1
+                    // - Number of updates = number of rows where affectedRows = 2
+                    const insertCount = result.affectedRows - result.changedRows;
+                    const updateCount = result.changedRows;
+                    added += insertCount;
+                    updated += updateCount;
                 }
 
                 // Process categories within the same transaction
@@ -304,7 +319,14 @@ async function importOrders(pool, filePath) {
     });
     
     function convertDate(dateStr) {
-        if (!dateStr) return null;
+        if (!dateStr) {
+            // Default to current date for missing dates
+            const now = new Date();
+            const year = now.getFullYear();
+            const month = String(now.getMonth() + 1).padStart(2, '0');
+            const day = String(now.getDate()).padStart(2, '0');
+            return `${year}-${month}-${day}`;
+        }
         const [day, month, year] = dateStr.split('-');
         return `${year}-${month}-${day}`;
     }
@@ -430,8 +452,16 @@ async function importOrders(pool, filePath) {
             
             // Update stats
             if (result.affectedRows > 0) {
-                updated += result.affectedRows - result.insertId;
+                // For INSERT ... ON DUPLICATE KEY UPDATE:
-                added += result.insertId;
+                // - If a row is inserted, affectedRows = 1
+                // - If a row is updated, affectedRows = 2
+                // So we can calculate:
+                // - Number of inserts = number of rows where affectedRows = 1
+                // - Number of updates = number of rows where affectedRows = 2
+                const insertCount = result.affectedRows - result.changedRows;
+                const updateCount = result.changedRows;
+                added += insertCount;
+                updated += updateCount;
             }
         } catch (error) {
             console.error(`\nError processing batch:`, error.message);
@@ -456,7 +486,14 @@ async function importPurchaseOrders(pool, filePath) {
     });
     
     function convertDate(dateStr) {
-        if (!dateStr) return null;
+        if (!dateStr) {
+            // Default to current date for missing dates
+            const now = new Date();
+            const year = now.getFullYear();
+            const month = String(now.getMonth() + 1).padStart(2, '0');
+            const day = String(now.getDate()).padStart(2, '0');
+            return `${year}-${month}-${day}`;
+        }
         const [day, month, year] = dateStr.split('-');
         return `${year}-${month}-${day}`;
     }
@@ -583,8 +620,16 @@ async function importPurchaseOrders(pool, filePath) {
             
             // Update stats
             if (result.affectedRows > 0) {
-                updated += result.affectedRows - result.insertId;
+                // For INSERT ... ON DUPLICATE KEY UPDATE:
-                added += result.insertId;
+                // - If a row is inserted, affectedRows = 1
+                // - If a row is updated, affectedRows = 2
+                // So we can calculate:
+                // - Number of inserts = number of rows where affectedRows = 1
+                // - Number of updates = number of rows where affectedRows = 2
+                const insertCount = result.affectedRows - result.changedRows;
+                const updateCount = result.changedRows;
+                added += insertCount;
+                updated += updateCount;
             }
         } catch (error) {
             console.error(`\nError processing batch:`, error.message);
@@ -617,8 +662,17 @@ async function main() {
         
         // Import products first since they're referenced by other tables
         await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv'));
-        await importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv'));
+
-        await importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'));
+        // Process orders and purchase orders in parallel
+        outputProgress({
+            operation: 'Starting parallel import',
+            message: 'Processing orders and purchase orders simultaneously...'
+        });
+
+        await Promise.all([
+            importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')),
+            importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'))
+        ]);
         
         outputProgress({
             status: 'complete',

inventory-server/scripts/reset-db.js

@@ -1,7 +1,7 @@
 const mysql = require('mysql2/promise');
 const path = require('path');
 const dotenv = require('dotenv');
-const { spawn } = require('child_process');
+const fs = require('fs');
 
 dotenv.config({ path: path.join(__dirname, '../.env') });
 
@@ -33,77 +33,48 @@ async function resetDatabase() {
     const connection = await mysql.createConnection(dbConfig);
     
     try {
-        // Get list of all tables
+        // Get list of all tables efficiently
         outputProgress({
             operation: 'Getting table list',
             message: 'Retrieving all table names...'
         });
 
-        const [tables] = await connection.query(
+        // More efficient query to get table names
-            'SELECT table_name FROM information_schema.tables WHERE table_schema = ?',
+        const [tables] = await connection.query(`
-            [dbConfig.database]
+            SELECT GROUP_CONCAT(table_name) as tables 
+            FROM information_schema.tables 
+            WHERE table_schema = DATABASE()`
         );
 
-        if (tables.length === 0) {
+        if (!tables[0].tables) {
             outputProgress({
                 operation: 'No tables found',
                 message: 'Database is already empty'
             });
         } else {
-            // Disable foreign key checks to allow dropping tables with dependencies
+            // Disable foreign key checks and drop all tables in one query
+            outputProgress({
+                operation: 'Dropping tables',
+                message: 'Dropping all tables...'
+            });
+
             await connection.query('SET FOREIGN_KEY_CHECKS = 0');
             
-            // Drop each table
+            // Create DROP TABLE statements for all tables at once
-            for (let i = 0; i < tables.length; i++) {
+            const dropQuery = `DROP TABLE IF EXISTS ${tables[0].tables.split(',').map(table => '`' + table + '`').join(', ')}`;
-                const tableName = tables[i].TABLE_NAME;
+            await connection.query(dropQuery);
-                outputProgress({
-                    operation: 'Dropping tables',
-                    message: `Dropping table: ${tableName}`,
-                    current: i + 1,
-                    total: tables.length,
-                    percentage: (((i + 1) / tables.length) * 100).toFixed(1)
-                });
-                await connection.query(`DROP TABLE IF EXISTS \`${tableName}\``);
-            }
             
-            // Re-enable foreign key checks
             await connection.query('SET FOREIGN_KEY_CHECKS = 1');
         }
 
-        // Run setup-db.js
+        // Read and execute schema directly instead of spawning a process
         outputProgress({
             operation: 'Running database setup',
             message: 'Creating new tables...'
         });
 
-        const setupScript = path.join(__dirname, 'setup-db.js');
+        const schemaSQL = fs.readFileSync(path.join(__dirname, '../db/schema.sql'), 'utf8');
-        const setupProcess = spawn('node', [setupScript]);
+        await connection.query(schemaSQL);
-
-        setupProcess.stdout.on('data', (data) => {
-            const output = data.toString().trim();
-            outputProgress({
-                operation: 'Database setup',
-                message: output
-            });
-        });
-
-        setupProcess.stderr.on('data', (data) => {
-            const error = data.toString().trim();
-            outputProgress({
-                status: 'error',
-                error
-            });
-        });
-
-        await new Promise((resolve, reject) => {
-            setupProcess.on('close', (code) => {
-                if (code === 0) {
-                    resolve();
-                } else {
-                    reject(new Error(`Setup process exited with code ${code}`));
-                }
-            });
-        });
 
         outputProgress({
             status: 'complete',

inventory/src/pages/Settings.tsx

@@ -50,6 +50,7 @@ export function Settings() {
   const [isResetting, setIsResetting] = useState(false);
   const [updateProgress, setUpdateProgress] = useState<ImportProgress | null>(null);
   const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);
+  const [purchaseOrdersProgress, setPurchaseOrdersProgress] = useState<ImportProgress | null>(null);
   const [resetProgress, setResetProgress] = useState<ImportProgress | null>(null);
   const [eventSource, setEventSource] = useState<EventSource | null>(null);
   const [limits, setLimits] = useState<ImportLimits>({
@@ -58,49 +59,48 @@ export function Settings() {
     purchaseOrders: 0
   });
 
+  // Helper function to update progress state
+  const updateProgressState = (progressData: any) => {
+    const operation = progressData.operation?.toLowerCase() || '';
+    const progressUpdate = {
+      status: progressData.status || 'running',
+      operation: progressData.operation,
+      current: progressData.current !== undefined ? Number(progressData.current) : undefined,
+      total: progressData.total !== undefined ? Number(progressData.total) : undefined,
+      rate: progressData.rate !== undefined ? Number(progressData.rate) : undefined,
+      percentage: progressData.percentage,
+      elapsed: progressData.elapsed,
+      remaining: progressData.remaining,
+      error: progressData.error,
+      message: progressData.message,
+      added: progressData.added,
+      updated: progressData.updated,
+      skipped: progressData.skipped,
+      duration: progressData.duration
+    };
+
+    if (operation.includes('products import completed')) {
+      setImportProgress(null);
+    } else if (operation.includes('products import')) {
+      setImportProgress(prev => ({ ...prev, ...progressUpdate }));
+    } else if (operation.includes('orders import') && !operation.includes('purchase')) {
+      setImportProgress(prev => ({ ...prev, ...progressUpdate }));
+    } else if (operation.includes('purchase orders import')) {
+      setPurchaseOrdersProgress(prev => ({ ...prev, ...progressUpdate }));
+    }
+  };
+
   // Helper to connect to event source
   const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset') => {
     if (eventSource) {
       eventSource.close();
     }
 
-    console.log('Connecting to event source:', type); // Debug log
     const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
       withCredentials: true
     });
     setEventSource(source);
 
-    source.onopen = () => {
-      console.log('Event source connected:', type); // Debug log
-    };
-    
-    source.onerror = () => {
-      console.log('Event source error:', type, source.readyState); // Debug log
-      if (source.readyState === EventSource.CLOSED) {
-        source.close();
-        setEventSource(null);
-        
-        // Only reset states if we're not in a completed state
-        const progress = type === 'update' ? updateProgress :
-                        type === 'import' ? importProgress :
-                        resetProgress;
-                        
-        if (progress?.status !== 'complete') {
-          // Reset the appropriate state based on type
-          if (type === 'update') {
-            setIsUpdating(false);
-            setUpdateProgress(null);
-          } else if (type === 'import') {
-            setIsImporting(false);
-            setImportProgress(null);
-          } else if (type === 'reset') {
-            setIsResetting(false);
-            setResetProgress(null);
-          }
-        }
-      }
-    };
-    
     source.onmessage = (event) => {
       try {
         const data = JSON.parse(event.data);
@@ -108,44 +108,18 @@ export function Settings() {
           (typeof data.progress === 'string' ? JSON.parse(data.progress) : data.progress) 
           : data;
 
-        // Update the appropriate progress state based on type
+        // Handle different types of progress
-        const setProgress = type === 'update' ? setUpdateProgress :
+        if (type === 'import') {
-                          type === 'import' ? setImportProgress :
+          updateProgressState(progressData);
-                          setResetProgress;
+        } else {
-
+          // For non-import operations, use the existing logic
-        // Also set the operation state if we're getting progress
+          const setProgress = type === 'update' ? setUpdateProgress :
-        if (progressData.status === 'running') {
+                            type === 'reset' ? setResetProgress :
-          if (type === 'update') setIsUpdating(true);
+                            setImportProgress;
-          else if (type === 'import') setIsImporting(true);
+          setProgress(prev => ({
-          else if (type === 'reset') setIsResetting(true);
-        }
-
-        setProgress(prev => {
-          // If we're getting a new operation, clear out old messages
-          if (progressData.operation && progressData.operation !== prev?.operation) {
-            return {
-              status: progressData.status || 'running',
-              operation: progressData.operation,
-              current: progressData.current !== undefined ? Number(progressData.current) : undefined,
-              total: progressData.total !== undefined ? Number(progressData.total) : undefined,
-              rate: progressData.rate !== undefined ? Number(progressData.rate) : undefined,
-              percentage: progressData.percentage,
-              elapsed: progressData.elapsed,
-              remaining: progressData.remaining,
-              message: progressData.message,
-              error: progressData.error,
-              added: progressData.added,
-              updated: progressData.updated,
-              skipped: progressData.skipped,
-              duration: progressData.duration
-            };
-          }
-          
-          // Otherwise update existing state
-          return {
             ...prev,
-            status: progressData.status || prev?.status || 'running',
+            status: progressData.status || 'running',
-            operation: progressData.operation || prev?.operation,
+            operation: progressData.operation,
             current: progressData.current !== undefined ? Number(progressData.current) : prev?.current,
             total: progressData.total !== undefined ? Number(progressData.total) : prev?.total,
             rate: progressData.rate !== undefined ? Number(progressData.rate) : prev?.rate,
@@ -158,8 +132,15 @@ export function Settings() {
             updated: progressData.updated !== undefined ? progressData.updated : prev?.updated,
             skipped: progressData.skipped !== undefined ? progressData.skipped : prev?.skipped,
             duration: progressData.duration || prev?.duration
-          };
+          }));
-        });
+        }
+
+        // Set operation state if we're getting progress
+        if (progressData.status === 'running') {
+          if (type === 'update') setIsUpdating(true);
+          else if (type === 'import') setIsImporting(true);
+          else if (type === 'reset') setIsResetting(true);
+        }
 
         if (progressData.status === 'complete') {
           source.close();
@@ -172,6 +153,7 @@ export function Settings() {
           } else if (type === 'import') {
             setIsImporting(false);
             setImportProgress(null);
+            setPurchaseOrdersProgress(null);
           } else if (type === 'reset') {
             setIsResetting(false);
             setResetProgress(null);
@@ -196,7 +178,7 @@ export function Settings() {
           handleError(`${type.charAt(0).toUpperCase() + type.slice(1)}`, progressData.error || 'Unknown error');
         }
       } catch (error) {
-        console.error('Error parsing event data:', error); // Debug log
+        console.error('Error parsing event data:', error);
       }
     };
   }, []);  // Remove dependencies that might prevent initial connection
@@ -339,7 +321,7 @@ export function Settings() {
     setImportProgress({ status: 'running', operation: 'Starting import process' });
     
     try {
-      // Connect to SSE for progress updates
+      // Connect to SSE for progress updates first
       connectToEventSource('import');
 
       // Make the import request
@@ -354,20 +336,23 @@ export function Settings() {
       
       if (!response.ok) {
         const data = await response.json().catch(() => ({}));
-        if (data.error === 'Import already in progress') {
+        // Only handle error if we don't have any progress yet
-          // If there's already an import, just let the SSE connection handle showing progress
+        if (!importProgress?.current && !purchaseOrdersProgress?.current) {
-          return;
+          throw new Error(data.error || 'Failed to start CSV import');
         }
-        throw new Error(data.error || 'Failed to start CSV import');
       }
     } catch (error) {
-      if (eventSource) {
+      // Only clean up if we don't have any progress
-        eventSource.close();
+      if (!importProgress?.current && !purchaseOrdersProgress?.current) {
-        setEventSource(null);
+        if (eventSource) {
+          eventSource.close();
+          setEventSource(null);
+        }
+        setIsImporting(false);
+        setImportProgress(null);
+        setPurchaseOrdersProgress(null);
+        handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
       }
-      setIsImporting(false);
-      setImportProgress(null);
-      handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
     }
   };
 
@@ -479,7 +464,11 @@ export function Settings() {
     if (error.includes('cancelled') || 
         error.includes('Process exited with code 143') || 
         error.includes('Operation cancelled') ||
-        error.includes('500 Internal Server Error')) {
+        error.includes('500 Internal Server Error') ||
+        // Skip "Failed to start" errors if we have active progress
+        (error.includes('Failed to start CSV import') && (importProgress || purchaseOrdersProgress)) ||
+        // Skip connection errors if we have active progress
+        (error.includes('Failed to fetch') && (importProgress || purchaseOrdersProgress))) {
       return;
     }
     toast.error(`${operation} failed: ${error}`);
@@ -615,7 +604,29 @@ export function Settings() {
               )}
             </div>
 
-            {isImporting && renderProgress(importProgress)}
+            {isImporting && (
+              <div className="space-y-4">
+                {/* Show products progress */}
+                {importProgress?.operation?.toLowerCase().includes('products') && (
+                  <div>
+                    {renderProgress(importProgress)}
+                  </div>
+                )}
+                {/* Show orders progress */}
+                {importProgress?.operation?.toLowerCase().includes('orders import') && 
+                 !importProgress.operation.toLowerCase().includes('purchase') && (
+                  <div>
+                    {renderProgress(importProgress)}
+                  </div>
+                )}
+                {/* Show purchase orders progress */}
+                {purchaseOrdersProgress && (
+                  <div>
+                    {renderProgress(purchaseOrdersProgress)}
+                  </div>
+                )}
+              </div>
+            )}
           </CardContent>
         </Card>