Break up prod import script into pieces and move csv scripts into folder

inventory-server/scripts/import-from-prod.js

@@ -1,7 +1,7 @@
-const mysql = require("mysql2/promise");
 const dotenv = require("dotenv");
 const path = require("path");
-const { setupSshTunnel, outputProgress, formatElapsedTime, prodDbConfig, localDbConfig } = require('./import/utils');
+const { outputProgress, formatElapsedTime } = require('./metrics/utils/progress');
+const { setupConnections, closeConnections } = require('./import/utils');
 const importCategories = require('./import/categories');
 const { importProducts } = require('./import/products');
 const importOrders = require('./import/orders');
@@ -15,6 +15,38 @@ const IMPORT_PRODUCTS = true;
 const IMPORT_ORDERS = true;
 const IMPORT_PURCHASE_ORDERS = true;
 
+// SSH configuration
+const sshConfig = {
+  ssh: {
+    host: process.env.PROD_SSH_HOST,
+    port: process.env.PROD_SSH_PORT || 22,
+    username: process.env.PROD_SSH_USER,
+    privateKey: process.env.PROD_SSH_KEY_PATH
+      ? require("fs").readFileSync(process.env.PROD_SSH_KEY_PATH)
+      : undefined,
+  },
+  // Production database configuration
+  prodDbConfig: {
+    host: process.env.PROD_DB_HOST || "localhost",
+    user: process.env.PROD_DB_USER,
+    password: process.env.PROD_DB_PASSWORD,
+    database: process.env.PROD_DB_NAME,
+    port: process.env.PROD_DB_PORT || 3306,
+  },
+  // Local database configuration
+  localDbConfig: {
+    host: process.env.DB_HOST,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASSWORD,
+    database: process.env.DB_NAME,
+    multipleStatements: true,
+    waitForConnections: true,
+    connectionLimit: 10,
+    queueLimit: 0,
+    namedPlaceholders: true,
+  }
+};
+
 let isImportCancelled = false;
 
 // Add cancel function
@@ -32,12 +64,9 @@ function cancelImport() {
     });
 }
 
-// Modify main function to handle cancellation and avoid process.exit
 async function main() {
-  let ssh;
-  let prodConnection;
-  let localConnection;
   const startTime = Date.now();
+  let connections;
 
   try {
     // Initial progress update
@@ -50,96 +79,39 @@ async function main() {
       elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
     });
 
-    const tunnel = await setupSshTunnel();
-    ssh = tunnel.ssh;
-
-    outputProgress({
-      status: "running",
-      operation: "Import process",
-      message: "Connecting to production database...",
-      current: 0,
-      total: 4,
-      elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-    });
-
-    prodConnection = await mysql.createConnection({
-      ...prodDbConfig,
-      stream: tunnel.stream,
-    });
-
-    outputProgress({
-      status: "running",
-      operation: "Import process",
-      message: "Connecting to local database...",
-      current: 0,
-      total: 4,
-      elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-    });
-
-    localConnection = await mysql.createPool({
-      ...localDbConfig,
-      waitForConnections: true,
-      connectionLimit: 10,
-      queueLimit: 0
-    });
+    connections = await setupConnections(sshConfig);
+    const { prodConnection, localConnection } = connections;
 
     if (isImportCancelled) throw new Error("Import cancelled");
 
-    let currentStep = 0;
+    const results = {
+      categories: null,
+      products: null,
+      orders: null,
+      purchaseOrders: null
+    };
 
     // Run each import based on constants
     if (IMPORT_CATEGORIES) {
-      outputProgress({
-        status: "running",
-        operation: "Import process",
-        message: "Starting categories import...",
-        current: currentStep,
-        total: 4,
-        elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-      });
-      await importCategories(prodConnection, localConnection);
+      results.categories = await importCategories(prodConnection, localConnection);
       if (isImportCancelled) throw new Error("Import cancelled");
       currentStep++;
     }
 
     if (IMPORT_PRODUCTS) {
-      outputProgress({
-        status: "running",
-        operation: "Import process",
-        message: "Starting products import...",
-        current: currentStep,
-        total: 4,
-        elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-      });
-      await importProducts(prodConnection, localConnection);
+      results.products = await importProducts(prodConnection, localConnection);
       if (isImportCancelled) throw new Error("Import cancelled");
       currentStep++;
     }
 
     if (IMPORT_ORDERS) {
-      outputProgress({
-        status: "running",
-        operation: "Import process",
-        message: "Starting orders import...",
-        current: currentStep,
-        total: 4,
-        elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-      });
-      await importOrders(prodConnection, localConnection);
+      results.orders = await importOrders(prodConnection, localConnection);
       if (isImportCancelled) throw new Error("Import cancelled");
       currentStep++;
     }
 
     if (IMPORT_PURCHASE_ORDERS) {
-      outputProgress({
-        status: "running",
-        operation: "Import process",
-        message: "Starting purchase orders import...",
-        current: currentStep,
-        total: 4,
-        elapsed: formatElapsedTime((Date.now() - startTime) / 1000)
-      });
-      await importPurchaseOrders(prodConnection, localConnection);
+      results.purchaseOrders = await importPurchaseOrders(prodConnection, localConnection);
       if (isImportCancelled) throw new Error("Import cancelled");
       currentStep++;
     }
@@ -157,8 +129,11 @@ async function main() {
         end_time: new Date(endTime).toISOString(),
         elapsed_time: formatElapsedTime((endTime - startTime) / 1000),
         elapsed_seconds: Math.round((endTime - startTime) / 1000)
-      }
+      },
+      results
     });
+
+    return results;
   } catch (error) {
     const endTime = Date.now();
     console.error("Error during import process:", error);
@@ -179,23 +154,8 @@ async function main() {
     });
     throw error;
   } finally {
-    try {
-      // Close connections in order
-      if (prodConnection) await prodConnection.end();
-      if (localConnection) await localConnection.end();
-      
-      // Wait a bit for any pending data to be written before closing SSH
-      await new Promise(resolve => setTimeout(resolve, 100));
-      
-      if (ssh) {
-        // Properly close the SSH connection
-        ssh.on('close', () => {
-          console.log('SSH connection closed cleanly');
-        });
-        ssh.end();
-      }
-    } catch (err) {
-      console.error('Error during cleanup:', err);
+    if (connections) {
+      await closeConnections(connections);
     }
   }
 }
@@ -211,6 +171,5 @@ if (require.main === module) {
 // Export the functions needed by the route
 module.exports = {
   main,
-  outputProgress,
   cancelImport,
 };