Optimize match columns step

inventory-server/src/routes/import.js

@@ -13,6 +13,26 @@ fs.mkdirSync(uploadsDir, { recursive: true });
 // Create a Map to track image upload times and their scheduled deletion
 const imageUploadMap = new Map();
 
+// Connection pooling and cache configuration
+const connectionCache = {
+  ssh: null,
+  dbConnection: null,
+  lastUsed: 0,
+  isConnecting: false,
+  connectionPromise: null,
+  // Cache expiration time in milliseconds (5 minutes)
+  expirationTime: 5 * 60 * 1000,
+  // Cache for query results (key: query string, value: {data, timestamp})
+  queryCache: new Map(),
+  // Cache duration for different query types in milliseconds
+  cacheDuration: {
+    'field-options': 30 * 60 * 1000, // 30 minutes for field options
+    'product-lines': 10 * 60 * 1000, // 10 minutes for product lines
+    'sublines': 10 * 60 * 1000,      // 10 minutes for sublines
+    'default': 60 * 1000             // 1 minute default
+  }
+};
+
 // Function to schedule image deletion after 24 hours
 const scheduleImageDeletion = (filename, filePath) => {
   // Delete any existing timeout for this file
@@ -165,7 +185,114 @@ const upload = multer({
   }
 });
 
-// Helper function to setup SSH tunnel
+// Modified function to get a database connection with connection pooling
+async function getDbConnection() {
+  const now = Date.now();
+  
+  // Check if we need to refresh the connection due to inactivity
+  const needsRefresh = !connectionCache.ssh || 
+                       !connectionCache.dbConnection || 
+                       (now - connectionCache.lastUsed > connectionCache.expirationTime);
+  
+  // If connection is still valid, update last used time and return existing connection
+  if (!needsRefresh) {
+    connectionCache.lastUsed = now;
+    return {
+      ssh: connectionCache.ssh,
+      connection: connectionCache.dbConnection
+    };
+  }
+  
+  // If another request is already establishing a connection, wait for that promise
+  if (connectionCache.isConnecting && connectionCache.connectionPromise) {
+    try {
+      await connectionCache.connectionPromise;
+      return {
+        ssh: connectionCache.ssh,
+        connection: connectionCache.dbConnection
+      };
+    } catch (error) {
+      // If that connection attempt failed, we'll try again below
+      console.error('Error waiting for existing connection:', error);
+    }
+  }
+  
+  // Close existing connections if they exist
+  if (connectionCache.dbConnection) {
+    try {
+      await connectionCache.dbConnection.end();
+    } catch (error) {
+      console.error('Error closing existing database connection:', error);
+    }
+  }
+  
+  if (connectionCache.ssh) {
+    try {
+      connectionCache.ssh.end();
+    } catch (error) {
+      console.error('Error closing existing SSH connection:', error);
+    }
+  }
+  
+  // Mark that we're establishing a new connection
+  connectionCache.isConnecting = true;
+  
+  // Create a new promise for this connection attempt
+  connectionCache.connectionPromise = setupSshTunnel().then(tunnel => {
+    const { ssh, stream, dbConfig } = tunnel;
+    
+    return mysql.createConnection({
+      ...dbConfig,
+      stream
+    }).then(connection => {
+      // Store the new connections
+      connectionCache.ssh = ssh;
+      connectionCache.dbConnection = connection;
+      connectionCache.lastUsed = Date.now();
+      connectionCache.isConnecting = false;
+      
+      return {
+        ssh,
+        connection
+      };
+    });
+  }).catch(error => {
+    connectionCache.isConnecting = false;
+    throw error;
+  });
+  
+  // Wait for the connection to be established
+  return connectionCache.connectionPromise;
+}
+
+// Helper function to get cached query results or execute query if not cached
+async function getCachedQuery(cacheKey, queryType, queryFn) {
+  // Get cache duration based on query type
+  const cacheDuration = connectionCache.cacheDuration[queryType] || connectionCache.cacheDuration.default;
+  
+  // Check if we have a valid cached result
+  const cachedResult = connectionCache.queryCache.get(cacheKey);
+  const now = Date.now();
+  
+  if (cachedResult && (now - cachedResult.timestamp < cacheDuration)) {
+    console.log(`Cache hit for ${queryType} query: ${cacheKey}`);
+    return cachedResult.data;
+  }
+  
+  // No valid cache found, execute the query
+  console.log(`Cache miss for ${queryType} query: ${cacheKey}`);
+  const result = await queryFn();
+  
+  // Cache the result
+  connectionCache.queryCache.set(cacheKey, {
+    data: result,
+    timestamp: now
+  });
+  
+  return result;
+}
+
+// Helper function to setup SSH tunnel - ONLY USED BY getDbConnection NOW
 async function setupSshTunnel() {
   const sshConfig = {
     host: process.env.PROD_SSH_HOST,
@@ -303,20 +430,12 @@ router.delete('/delete-image', (req, res) => {
 
 // Get all options for import fields
 router.get('/field-options', async (req, res) => {
-    let ssh;
-    let connection;
-    
     try {
-        // Setup SSH tunnel and get database connection
+        // Use cached connection
-        const tunnel = await setupSshTunnel();
+        const { connection } = await getDbConnection();
-        ssh = tunnel.ssh;
-        
-        // Create MySQL connection over SSH tunnel
-        connection = await mysql.createConnection({
-            ...tunnel.dbConfig,
-            stream: tunnel.stream
-        });
 
+        const cacheKey = 'field-options';
+        const result = await getCachedQuery(cacheKey, 'field-options', async () => {
             // Fetch companies (type 1)
             const [companies] = await connection.query(`
                 SELECT cat_id, name 
@@ -412,7 +531,8 @@ router.get('/field-options', async (req, res) => {
                 ORDER BY tax_code_id = 0 DESC, name
             `);
 
-        res.json({
+            // Format and return all options
+            return {
                 companies: companies.map(c => ({ label: c.name, value: c.cat_id.toString() })),
                 artists: artists.map(a => ({ label: a.name, value: a.cat_id.toString() })),
                 sizes: sizes.map(s => ({ label: s.name, value: s.cat_id.toString() })),
@@ -443,81 +563,69 @@ router.get('/field-options', async (req, res) => {
                     { label: "No FedEx 2 Day", value: "4" },
                     { label: "North America Only", value: "5" }
                 ]
+            };
         });
+
+        res.json(result);
     } catch (error) {
         console.error('Error fetching import field options:', error);
         res.status(500).json({ error: 'Failed to fetch import field options' });
-    } finally {
-        if (connection) await connection.end();
-        if (ssh) ssh.end();
     }
 });
 
 // Get product lines for a specific company
 router.get('/product-lines/:companyId', async (req, res) => {
-    let ssh;
-    let connection;
-    
     try {
-        // Setup SSH tunnel and get database connection
+        // Use cached connection
-        const tunnel = await setupSshTunnel();
+        const { connection } = await getDbConnection();
-        ssh = tunnel.ssh;
 
-        // Create MySQL connection over SSH tunnel
+        const companyId = req.params.companyId;
-        connection = await mysql.createConnection({
+        const cacheKey = `product-lines-${companyId}`;
-            ...tunnel.dbConfig,
-            stream: tunnel.stream
-        });
         
-        const [lines] = await connection.query(`
+        const lines = await getCachedQuery(cacheKey, 'product-lines', async () => {
+            const [queryResult] = await connection.query(`
                 SELECT cat_id as value, name as label 
                 FROM product_categories 
                 WHERE type = 2 
                 AND master_cat_id = ? 
                 ORDER BY name
-        `, [req.params.companyId]);
+            `, [companyId]);
             
-        res.json(lines.map(l => ({ label: l.label, value: l.value.toString() })));
+            return queryResult.map(l => ({ label: l.label, value: l.value.toString() }));
+        });
+
+        res.json(lines);
     } catch (error) {
         console.error('Error fetching product lines:', error);
         res.status(500).json({ error: 'Failed to fetch product lines' });
-    } finally {
-        if (connection) await connection.end();
-        if (ssh) ssh.end();
     }
 });
 
 // Get sublines for a specific product line
 router.get('/sublines/:lineId', async (req, res) => {
-    let ssh;
-    let connection;
-    
     try {
-        // Setup SSH tunnel and get database connection
+        // Use cached connection
-        const tunnel = await setupSshTunnel();
+        const { connection } = await getDbConnection();
-        ssh = tunnel.ssh;
 
-        // Create MySQL connection over SSH tunnel
+        const lineId = req.params.lineId;
-        connection = await mysql.createConnection({
+        const cacheKey = `sublines-${lineId}`;
-            ...tunnel.dbConfig,
-            stream: tunnel.stream
-        });
         
-        const [sublines] = await connection.query(`
+        const sublines = await getCachedQuery(cacheKey, 'sublines', async () => {
+            const [queryResult] = await connection.query(`
                 SELECT cat_id as value, name as label 
                 FROM product_categories 
                 WHERE type = 3 
                 AND master_cat_id = ? 
                 ORDER BY name
-        `, [req.params.lineId]);
+            `, [lineId]);
             
-        res.json(sublines.map(s => ({ label: s.label, value: s.value.toString() })));
+            return queryResult.map(s => ({ label: s.label, value: s.value.toString() }));
+        });
+
+        res.json(sublines);
     } catch (error) {
         console.error('Error fetching sublines:', error);
         res.status(500).json({ error: 'Failed to fetch sublines' });
-    } finally {
-        if (connection) await connection.end();
-        if (ssh) ssh.end();
     }
 });