Break up prod import script into pieces and move csv scripts into folder

inventory-server/scripts/import/utils.js

@@ -2,53 +2,14 @@ const mysql = require("mysql2/promise");
 const { Client } = require("ssh2");
 const dotenv = require("dotenv");
 const path = require("path");
-const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
 
-dotenv.config({ path: path.join(__dirname, "../../.env") });
-
-// SSH configuration
-const sshConfig = {
-  host: process.env.PROD_SSH_HOST,
-  port: process.env.PROD_SSH_PORT || 22,
-  username: process.env.PROD_SSH_USER,
-  privateKey: process.env.PROD_SSH_KEY_PATH
-    ? require("fs").readFileSync(process.env.PROD_SSH_KEY_PATH)
-    : undefined,
-};
-
-// Production database configuration
-const prodDbConfig = {
-  host: process.env.PROD_DB_HOST || "localhost",
-  user: process.env.PROD_DB_USER,
-  password: process.env.PROD_DB_PASSWORD,
-  database: process.env.PROD_DB_NAME,
-  port: process.env.PROD_DB_PORT || 3306,
-};
-
-// Local database configuration
-const localDbConfig = {
-  host: process.env.DB_HOST,
-  user: process.env.DB_USER,
-  password: process.env.DB_PASSWORD,
-  database: process.env.DB_NAME,
-  multipleStatements: true,
-  waitForConnections: true,
-  connectionLimit: 10,
-  queueLimit: 0,
-  namedPlaceholders: true,
-};
-
-// Constants
-const BATCH_SIZE = 1000;
-const PROGRESS_INTERVAL = 1000; // Update progress every second
-
-async function setupSshTunnel() {
+// Helper function to setup SSH tunnel
+async function setupSshTunnel(sshConfig) {
   return new Promise((resolve, reject) => {
     const ssh = new Client();
 
     ssh.on('error', (err) => {
       console.error('SSH connection error:', err);
-      // Don't reject here, just log the error
     });
 
     ssh.on('end', () => {
@@ -64,39 +25,64 @@ async function setupSshTunnel() {
         ssh.forwardOut(
           "127.0.0.1",
           0,
-          prodDbConfig.host,
-          prodDbConfig.port,
+          sshConfig.prodDbConfig.host,
+          sshConfig.prodDbConfig.port,
           async (err, stream) => {
             if (err) reject(err);
             resolve({ ssh, stream });
           }
         );
       })
-      .connect(sshConfig);
+      .connect(sshConfig.ssh);
   });
 }
 
-// Helper function to update progress with time estimate
-function updateProgress(current, total, operation, startTime) {
-    outputProgress({
-        status: 'running',
-        operation,
-        current,
-        total,
-        rate: calculateRate(startTime, current),
-        elapsed: formatElapsedTime(startTime),
-        remaining: estimateRemaining(startTime, current, total),
-        percentage: ((current / total) * 100).toFixed(1)
-    });
+// Helper function to setup database connections
+async function setupConnections(sshConfig) {
+  const tunnel = await setupSshTunnel(sshConfig);
+  
+  const prodConnection = await mysql.createConnection({
+    ...sshConfig.prodDbConfig,
+    stream: tunnel.stream,
+  });
+
+  const localConnection = await mysql.createPool({
+    ...sshConfig.localDbConfig,
+    waitForConnections: true,
+    connectionLimit: 10,
+    queueLimit: 0
+  });
+
+  return {
+    ssh: tunnel.ssh,
+    prodConnection,
+    localConnection
+  };
+}
+
+// Helper function to close connections
+async function closeConnections(connections) {
+  const { ssh, prodConnection, localConnection } = connections;
+  
+  try {
+    if (prodConnection) await prodConnection.end();
+    if (localConnection) await localConnection.end();
+    
+    // Wait a bit for any pending data to be written before closing SSH
+    await new Promise(resolve => setTimeout(resolve, 100));
+    
+    if (ssh) {
+      ssh.on('close', () => {
+        console.log('SSH connection closed cleanly');
+      });
+      ssh.end();
+    }
+  } catch (err) {
+    console.error('Error during cleanup:', err);
+  }
 }
 
 module.exports = {
-  setupSshTunnel,
-  updateProgress,
-  prodDbConfig,
-  localDbConfig,
-  BATCH_SIZE,
-  PROGRESS_INTERVAL,
-  outputProgress,
-  formatElapsedTime
+  setupConnections,
+  closeConnections
 };