diff --git a/inventory-server/scripts/import-csv.js b/inventory-server/scripts/import-csv.js index 5c3b2ae..68d3674 100644 --- a/inventory-server/scripts/import-csv.js +++ b/inventory-server/scripts/import-csv.js @@ -124,7 +124,14 @@ async function importProducts(pool, filePath) { }); function convertDate(dateStr) { - if (!dateStr) return null; + if (!dateStr) { + // Default to current date for missing dates + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; + } const [day, month, year] = dateStr.split('-'); return `${year}-${month}-${day}`; } @@ -268,8 +275,16 @@ async function importProducts(pool, filePath) { // Update stats if (result.affectedRows > 0) { - updated += result.affectedRows - result.insertId; - added += result.insertId; + // For INSERT ... ON DUPLICATE KEY UPDATE: + // - If a row is inserted, affectedRows = 1 + // - If a row is updated, affectedRows = 2 + // So we can calculate: + // - Number of inserts = number of rows where affectedRows = 1 + // - Number of updates = number of rows where affectedRows = 2 + const insertCount = result.affectedRows - result.changedRows; + const updateCount = result.changedRows; + added += insertCount; + updated += updateCount; } // Process categories within the same transaction @@ -304,7 +319,14 @@ async function importOrders(pool, filePath) { }); function convertDate(dateStr) { - if (!dateStr) return null; + if (!dateStr) { + // Default to current date for missing dates + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; + } const [day, month, year] = dateStr.split('-'); return `${year}-${month}-${day}`; } @@ -430,8 +452,16 @@ async function importOrders(pool, filePath) { // Update stats if (result.affectedRows > 0) { - updated += result.affectedRows - result.insertId; - added += result.insertId; + // For INSERT ... ON DUPLICATE KEY UPDATE: + // - If a row is inserted, affectedRows = 1 + // - If a row is updated, affectedRows = 2 + // So we can calculate: + // - Number of inserts = number of rows where affectedRows = 1 + // - Number of updates = number of rows where affectedRows = 2 + const insertCount = result.affectedRows - result.changedRows; + const updateCount = result.changedRows; + added += insertCount; + updated += updateCount; } } catch (error) { console.error(`\nError processing batch:`, error.message); @@ -456,7 +486,14 @@ async function importPurchaseOrders(pool, filePath) { }); function convertDate(dateStr) { - if (!dateStr) return null; + if (!dateStr) { + // Default to current date for missing dates + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; + } const [day, month, year] = dateStr.split('-'); return `${year}-${month}-${day}`; } @@ -583,8 +620,16 @@ async function importPurchaseOrders(pool, filePath) { // Update stats if (result.affectedRows > 0) { - updated += result.affectedRows - result.insertId; - added += result.insertId; + // For INSERT ... ON DUPLICATE KEY UPDATE: + // - If a row is inserted, affectedRows = 1 + // - If a row is updated, affectedRows = 2 + // So we can calculate: + // - Number of inserts = number of rows where affectedRows = 1 + // - Number of updates = number of rows where affectedRows = 2 + const insertCount = result.affectedRows - result.changedRows; + const updateCount = result.changedRows; + added += insertCount; + updated += updateCount; } } catch (error) { console.error(`\nError processing batch:`, error.message); @@ -617,8 +662,17 @@ async function main() { // Import products first since they're referenced by other tables await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv')); - await importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')); - await importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv')); + + // Process orders and purchase orders in parallel + outputProgress({ + operation: 'Starting parallel import', + message: 'Processing orders and purchase orders simultaneously...' + }); + + await Promise.all([ + importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')), + importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv')) + ]); outputProgress({ status: 'complete', diff --git a/inventory-server/scripts/reset-db.js b/inventory-server/scripts/reset-db.js index ef958ec..1246afe 100644 --- a/inventory-server/scripts/reset-db.js +++ b/inventory-server/scripts/reset-db.js @@ -1,7 +1,7 @@ const mysql = require('mysql2/promise'); const path = require('path'); const dotenv = require('dotenv'); -const { spawn } = require('child_process'); +const fs = require('fs'); dotenv.config({ path: path.join(__dirname, '../.env') }); @@ -33,77 +33,48 @@ async function resetDatabase() { const connection = await mysql.createConnection(dbConfig); try { - // Get list of all tables + // Get list of all tables efficiently outputProgress({ operation: 'Getting table list', message: 'Retrieving all table names...' }); - const [tables] = await connection.query( - 'SELECT table_name FROM information_schema.tables WHERE table_schema = ?', - [dbConfig.database] + // More efficient query to get table names + const [tables] = await connection.query(` + SELECT GROUP_CONCAT(table_name) as tables + FROM information_schema.tables + WHERE table_schema = DATABASE()` ); - if (tables.length === 0) { + if (!tables[0].tables) { outputProgress({ operation: 'No tables found', message: 'Database is already empty' }); } else { - // Disable foreign key checks to allow dropping tables with dependencies + // Disable foreign key checks and drop all tables in one query + outputProgress({ + operation: 'Dropping tables', + message: 'Dropping all tables...' + }); + await connection.query('SET FOREIGN_KEY_CHECKS = 0'); - - // Drop each table - for (let i = 0; i < tables.length; i++) { - const tableName = tables[i].TABLE_NAME; - outputProgress({ - operation: 'Dropping tables', - message: `Dropping table: ${tableName}`, - current: i + 1, - total: tables.length, - percentage: (((i + 1) / tables.length) * 100).toFixed(1) - }); - await connection.query(`DROP TABLE IF EXISTS \`${tableName}\``); - } - - // Re-enable foreign key checks + + // Create DROP TABLE statements for all tables at once + const dropQuery = `DROP TABLE IF EXISTS ${tables[0].tables.split(',').map(table => '`' + table + '`').join(', ')}`; + await connection.query(dropQuery); + await connection.query('SET FOREIGN_KEY_CHECKS = 1'); } - // Run setup-db.js + // Read and execute schema directly instead of spawning a process outputProgress({ operation: 'Running database setup', message: 'Creating new tables...' }); - const setupScript = path.join(__dirname, 'setup-db.js'); - const setupProcess = spawn('node', [setupScript]); - - setupProcess.stdout.on('data', (data) => { - const output = data.toString().trim(); - outputProgress({ - operation: 'Database setup', - message: output - }); - }); - - setupProcess.stderr.on('data', (data) => { - const error = data.toString().trim(); - outputProgress({ - status: 'error', - error - }); - }); - - await new Promise((resolve, reject) => { - setupProcess.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`Setup process exited with code ${code}`)); - } - }); - }); + const schemaSQL = fs.readFileSync(path.join(__dirname, '../db/schema.sql'), 'utf8'); + await connection.query(schemaSQL); outputProgress({ status: 'complete', diff --git a/inventory/src/pages/Settings.tsx b/inventory/src/pages/Settings.tsx index bd0bf96..0a2b851 100644 --- a/inventory/src/pages/Settings.tsx +++ b/inventory/src/pages/Settings.tsx @@ -50,6 +50,7 @@ export function Settings() { const [isResetting, setIsResetting] = useState(false); const [updateProgress, setUpdateProgress] = useState(null); const [importProgress, setImportProgress] = useState(null); + const [purchaseOrdersProgress, setPurchaseOrdersProgress] = useState(null); const [resetProgress, setResetProgress] = useState(null); const [eventSource, setEventSource] = useState(null); const [limits, setLimits] = useState({ @@ -58,49 +59,48 @@ export function Settings() { purchaseOrders: 0 }); + // Helper function to update progress state + const updateProgressState = (progressData: any) => { + const operation = progressData.operation?.toLowerCase() || ''; + const progressUpdate = { + status: progressData.status || 'running', + operation: progressData.operation, + current: progressData.current !== undefined ? Number(progressData.current) : undefined, + total: progressData.total !== undefined ? Number(progressData.total) : undefined, + rate: progressData.rate !== undefined ? Number(progressData.rate) : undefined, + percentage: progressData.percentage, + elapsed: progressData.elapsed, + remaining: progressData.remaining, + error: progressData.error, + message: progressData.message, + added: progressData.added, + updated: progressData.updated, + skipped: progressData.skipped, + duration: progressData.duration + }; + + if (operation.includes('products import completed')) { + setImportProgress(null); + } else if (operation.includes('products import')) { + setImportProgress(prev => ({ ...prev, ...progressUpdate })); + } else if (operation.includes('orders import') && !operation.includes('purchase')) { + setImportProgress(prev => ({ ...prev, ...progressUpdate })); + } else if (operation.includes('purchase orders import')) { + setPurchaseOrdersProgress(prev => ({ ...prev, ...progressUpdate })); + } + }; + // Helper to connect to event source const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset') => { if (eventSource) { eventSource.close(); } - console.log('Connecting to event source:', type); // Debug log const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, { withCredentials: true }); setEventSource(source); - source.onopen = () => { - console.log('Event source connected:', type); // Debug log - }; - - source.onerror = () => { - console.log('Event source error:', type, source.readyState); // Debug log - if (source.readyState === EventSource.CLOSED) { - source.close(); - setEventSource(null); - - // Only reset states if we're not in a completed state - const progress = type === 'update' ? updateProgress : - type === 'import' ? importProgress : - resetProgress; - - if (progress?.status !== 'complete') { - // Reset the appropriate state based on type - if (type === 'update') { - setIsUpdating(false); - setUpdateProgress(null); - } else if (type === 'import') { - setIsImporting(false); - setImportProgress(null); - } else if (type === 'reset') { - setIsResetting(false); - setResetProgress(null); - } - } - } - }; - source.onmessage = (event) => { try { const data = JSON.parse(event.data); @@ -108,44 +108,18 @@ export function Settings() { (typeof data.progress === 'string' ? JSON.parse(data.progress) : data.progress) : data; - // Update the appropriate progress state based on type - const setProgress = type === 'update' ? setUpdateProgress : - type === 'import' ? setImportProgress : - setResetProgress; - - // Also set the operation state if we're getting progress - if (progressData.status === 'running') { - if (type === 'update') setIsUpdating(true); - else if (type === 'import') setIsImporting(true); - else if (type === 'reset') setIsResetting(true); - } - - setProgress(prev => { - // If we're getting a new operation, clear out old messages - if (progressData.operation && progressData.operation !== prev?.operation) { - return { - status: progressData.status || 'running', - operation: progressData.operation, - current: progressData.current !== undefined ? Number(progressData.current) : undefined, - total: progressData.total !== undefined ? Number(progressData.total) : undefined, - rate: progressData.rate !== undefined ? Number(progressData.rate) : undefined, - percentage: progressData.percentage, - elapsed: progressData.elapsed, - remaining: progressData.remaining, - message: progressData.message, - error: progressData.error, - added: progressData.added, - updated: progressData.updated, - skipped: progressData.skipped, - duration: progressData.duration - }; - } - - // Otherwise update existing state - return { + // Handle different types of progress + if (type === 'import') { + updateProgressState(progressData); + } else { + // For non-import operations, use the existing logic + const setProgress = type === 'update' ? setUpdateProgress : + type === 'reset' ? setResetProgress : + setImportProgress; + setProgress(prev => ({ ...prev, - status: progressData.status || prev?.status || 'running', - operation: progressData.operation || prev?.operation, + status: progressData.status || 'running', + operation: progressData.operation, current: progressData.current !== undefined ? Number(progressData.current) : prev?.current, total: progressData.total !== undefined ? Number(progressData.total) : prev?.total, rate: progressData.rate !== undefined ? Number(progressData.rate) : prev?.rate, @@ -158,8 +132,15 @@ export function Settings() { updated: progressData.updated !== undefined ? progressData.updated : prev?.updated, skipped: progressData.skipped !== undefined ? progressData.skipped : prev?.skipped, duration: progressData.duration || prev?.duration - }; - }); + })); + } + + // Set operation state if we're getting progress + if (progressData.status === 'running') { + if (type === 'update') setIsUpdating(true); + else if (type === 'import') setIsImporting(true); + else if (type === 'reset') setIsResetting(true); + } if (progressData.status === 'complete') { source.close(); @@ -172,6 +153,7 @@ export function Settings() { } else if (type === 'import') { setIsImporting(false); setImportProgress(null); + setPurchaseOrdersProgress(null); } else if (type === 'reset') { setIsResetting(false); setResetProgress(null); @@ -196,7 +178,7 @@ export function Settings() { handleError(`${type.charAt(0).toUpperCase() + type.slice(1)}`, progressData.error || 'Unknown error'); } } catch (error) { - console.error('Error parsing event data:', error); // Debug log + console.error('Error parsing event data:', error); } }; }, []); // Remove dependencies that might prevent initial connection @@ -339,7 +321,7 @@ export function Settings() { setImportProgress({ status: 'running', operation: 'Starting import process' }); try { - // Connect to SSE for progress updates + // Connect to SSE for progress updates first connectToEventSource('import'); // Make the import request @@ -354,20 +336,23 @@ export function Settings() { if (!response.ok) { const data = await response.json().catch(() => ({})); - if (data.error === 'Import already in progress') { - // If there's already an import, just let the SSE connection handle showing progress - return; + // Only handle error if we don't have any progress yet + if (!importProgress?.current && !purchaseOrdersProgress?.current) { + throw new Error(data.error || 'Failed to start CSV import'); } - throw new Error(data.error || 'Failed to start CSV import'); } } catch (error) { - if (eventSource) { - eventSource.close(); - setEventSource(null); + // Only clean up if we don't have any progress + if (!importProgress?.current && !purchaseOrdersProgress?.current) { + if (eventSource) { + eventSource.close(); + setEventSource(null); + } + setIsImporting(false); + setImportProgress(null); + setPurchaseOrdersProgress(null); + handleError('Data import', error instanceof Error ? error.message : 'Unknown error'); } - setIsImporting(false); - setImportProgress(null); - handleError('Data import', error instanceof Error ? error.message : 'Unknown error'); } }; @@ -479,7 +464,11 @@ export function Settings() { if (error.includes('cancelled') || error.includes('Process exited with code 143') || error.includes('Operation cancelled') || - error.includes('500 Internal Server Error')) { + error.includes('500 Internal Server Error') || + // Skip "Failed to start" errors if we have active progress + (error.includes('Failed to start CSV import') && (importProgress || purchaseOrdersProgress)) || + // Skip connection errors if we have active progress + (error.includes('Failed to fetch') && (importProgress || purchaseOrdersProgress))) { return; } toast.error(`${operation} failed: ${error}`); @@ -615,7 +604,29 @@ export function Settings() { )} - {isImporting && renderProgress(importProgress)} + {isImporting && ( +
+ {/* Show products progress */} + {importProgress?.operation?.toLowerCase().includes('products') && ( +
+ {renderProgress(importProgress)} +
+ )} + {/* Show orders progress */} + {importProgress?.operation?.toLowerCase().includes('orders import') && + !importProgress.operation.toLowerCase().includes('purchase') && ( +
+ {renderProgress(importProgress)} +
+ )} + {/* Show purchase orders progress */} + {purchaseOrdersProgress && ( +
+ {renderProgress(purchaseOrdersProgress)} +
+ )} +
+ )}