Optimize database import queries with improved index selection

inventory-server/scripts/import/orders.js

@@ -52,7 +52,7 @@ async function importOrders(prodConnection, localConnection) {
     let processed = 0;
 
     // Process in batches
-    const batchSize = 20000; // Increased from 1000 since order records are small
+    const batchSize = 10000; // Increased from 1000 since order records are small
     let offset = 0;
 
     while (offset < total) {

inventory-server/scripts/import/purchase-orders.js

@@ -32,7 +32,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
       FROM (
         SELECT DISTINCT pop.po_id, pop.pid
         FROM po p 
-        FORCE INDEX (idx_date_created)
+        USE INDEX (idx_date_created)
         JOIN po_products pop ON p.po_id = pop.po_id
         JOIN suppliers s ON p.supplier_id = s.supplierid
         WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
@@ -42,6 +42,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
         UNION
         SELECT DISTINCT r.receiving_id as po_id, rp.pid
         FROM receivings_products rp
+        USE INDEX (received_date)
         LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
         WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
           AND (rp.received_date > ?
@@ -64,6 +65,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
         COALESCE(p.notes, '') as long_note
       FROM (
         SELECT po_id FROM po 
+        USE INDEX (idx_date_created)
         WHERE date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
           AND (date_ordered > ?
           OR stamp > ?
@@ -71,7 +73,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
         UNION
         SELECT DISTINCT r.receiving_id as po_id
         FROM receivings r
-        JOIN receivings_products rp ON r.receiving_id = rp.receiving_id
+        JOIN receivings_products rp USE INDEX (received_date) ON r.receiving_id = rp.receiving_id
         WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
           AND (rp.received_date > ?
           OR rp.stamp > ?)
@@ -110,7 +112,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
           pop.cost_each as cost_price,
           pop.qty_each as ordered
         FROM po_products pop
-        FORCE INDEX (PRIMARY)
+        USE INDEX (PRIMARY)
         JOIN products pr ON pop.pid = pr.pid
         WHERE pop.po_id IN (?)
       `, [poIds]);
@@ -138,6 +140,7 @@ async function importPurchaseOrders(prodConnection, localConnection) {
               ELSE 1                       -- Different PO
             END as is_alt_po
           FROM receivings_products rp
+          USE INDEX (received_date)
           LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
           WHERE rp.pid IN (?)
             AND rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)

inventory-server/scripts/scripts.js

@@ -0,0 +1,180 @@
+const readline = require('readline');
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout
+});
+
+const question = (query) => new Promise((resolve) => rl.question(query, resolve));
+
+async function loadScript(name) {
+  try {
+    return await require(name);
+  } catch (error) {
+    console.error(`Failed to load script ${name}:`, error);
+    return null;
+  }
+}
+
+async function runWithTimeout(fn) {
+  return new Promise((resolve, reject) => {
+    // Create a child process for the script
+    const child = require('child_process').fork(fn.toString(), [], {
+      stdio: 'inherit'
+    });
+
+    child.on('exit', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`Script exited with code ${code}`));
+      }
+    });
+
+    child.on('error', (err) => {
+      reject(err);
+    });
+  });
+}
+
+function clearScreen() {
+  process.stdout.write('\x1Bc');
+}
+
+const scripts = {
+  'Import Scripts': {
+    '1': { name: 'Full Import From Production', path: './import-from-prod' },
+    '2': { name: 'Individual Import Scripts ▸', submenu: {
+      '1': { name: 'Import Orders', path: './import/orders', key: 'importOrders' },
+      '2': { name: 'Import Products', path: './import/products', key: 'importProducts' },
+      '3': { name: 'Import Purchase Orders', path: './import/purchase-orders' },
+      '4': { name: 'Import Categories', path: './import/categories' },
+      'b': { name: 'Back to Main Menu' }
+    }}
+  },
+  'Metrics': {
+    '3': { name: 'Calculate All Metrics', path: './calculate-metrics' },
+    '4': { name: 'Individual Metric Scripts ▸', submenu: {
+      '1': { name: 'Brand Metrics', path: './metrics/brand-metrics' },
+      '2': { name: 'Category Metrics', path: './metrics/category-metrics' },
+      '3': { name: 'Financial Metrics', path: './metrics/financial-metrics' },
+      '4': { name: 'Product Metrics', path: './metrics/product-metrics' },
+      '5': { name: 'Sales Forecasts', path: './metrics/sales-forecasts' },
+      '6': { name: 'Time Aggregates', path: './metrics/time-aggregates' },
+      '7': { name: 'Vendor Metrics', path: './metrics/vendor-metrics' },
+      'b': { name: 'Back to Main Menu' }
+    }}
+  },
+  'Database Management': {
+    '5': { name: 'Test Production Connection', path: './test-prod-connection' }
+  },
+  'Reset Scripts': {
+    '6': { name: 'Reset Database', path: './reset-db' },
+    '7': { name: 'Reset Metrics', path: './reset-metrics' }
+  }
+};
+
+let lastRun = null;
+
+async function displayMenu(menuItems, title = 'Inventory Management Script Runner') {
+  clearScreen();
+  console.log(`\n${title}\n`);
+  
+  for (const [category, items] of Object.entries(menuItems)) {
+    console.log(`\n${category}:`);
+    Object.entries(items).forEach(([key, script]) => {
+      console.log(`${key}. ${script.name}`);
+    });
+  }
+
+  if (lastRun) {
+    console.log('\nQuick Access:');
+    console.log(`r. Repeat Last Script (${lastRun.name})`);
+  }
+  
+  console.log('\nq. Quit\n');
+}
+
+async function handleSubmenu(submenu, title) {
+  while (true) {
+    await displayMenu({"Individual Scripts": submenu}, title);
+    const choice = await question('Select an option (or b to go back): ');
+    
+    if (choice.toLowerCase() === 'b') {
+      return null;
+    }
+
+    if (submenu[choice]) {
+      return submenu[choice];
+    }
+
+    console.log('Invalid selection. Please try again.');
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+}
+
+async function runScript(script) {
+  console.log(`\nRunning: ${script.name}`);
+  try {
+    const scriptPath = require.resolve(script.path);
+    await runWithTimeout(scriptPath);
+    console.log('\nScript completed successfully');
+    lastRun = script;
+  } catch (error) {
+    console.error('\nError running script:', error);
+  }
+  await question('\nPress Enter to continue...');
+}
+
+async function main() {
+  while (true) {
+    await displayMenu(scripts);
+    
+    const choice = await question('Select an option: ');
+    
+    if (choice.toLowerCase() === 'q') {
+      break;
+    }
+
+    if (choice.toLowerCase() === 'r' && lastRun) {
+      await runScript(lastRun);
+      continue;
+    }
+
+    let selectedScript = null;
+    for (const category of Object.values(scripts)) {
+      if (category[choice]) {
+        selectedScript = category[choice];
+        break;
+      }
+    }
+
+    if (!selectedScript) {
+      console.log('Invalid selection. Please try again.');
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      continue;
+    }
+
+    if (selectedScript.submenu) {
+      const submenuChoice = await handleSubmenu(
+        selectedScript.submenu,
+        selectedScript.name
+      );
+      if (submenuChoice && submenuChoice.path) {
+        await runScript(submenuChoice);
+      }
+    } else if (selectedScript.path) {
+      await runScript(selectedScript);
+    }
+  }
+
+  rl.close();
+  process.exit(0);
+}
+
+if (require.main === module) {
+  main().catch(error => {
+    console.error('Fatal error:', error);
+    process.exit(1);
+  });
+}