Fix incorrect error during import, better logging during import

This commit is contained in:
2025-01-12 13:42:36 -05:00
parent 6c524aa3a9
commit 3e855eeb2b
2 changed files with 57 additions and 36 deletions

View File

@@ -46,12 +46,13 @@ function logError(error, context = '') {
} }
// Helper function to log import progress // Helper function to log import progress
function logImport(message) { function logImport(message, isSignificant = false) {
const timestamp = new Date().toISOString(); // Only write to disk if it's a significant event
const logMessage = `[${timestamp}] ${message}\n`; if (isSignificant) {
const timestamp = new Date().toISOString();
// Log to import file const logMessage = `[${timestamp}] ${message}\n`;
fs.appendFileSync(IMPORT_LOG, logMessage); fs.appendFileSync(IMPORT_LOG, logMessage);
}
} }
// Helper function to format duration // Helper function to format duration
@@ -70,7 +71,26 @@ function formatDuration(seconds) {
// Helper function to output progress // Helper function to output progress
function outputProgress(data) { function outputProgress(data) {
// Always send to stdout for frontend
process.stdout.write(JSON.stringify(data) + '\n'); process.stdout.write(JSON.stringify(data) + '\n');
// Log significant events to disk
const isSignificant =
// Operation starts
(data.operation && !data.current) ||
// Operation completions and errors
data.status === 'complete' ||
data.status === 'error' ||
// Test limits reached
data.message?.includes('test limit') ||
// Schema changes
data.operation?.includes('Creating database schema') ||
// Parallel import starts
data.message?.includes('Processing orders and purchase orders simultaneously');
if (isSignificant) {
logImport(`${data.operation || 'Operation'}${data.message ? ': ' + data.message : ''}${data.error ? ' Error: ' + data.error : ''}${data.status ? ' Status: ' + data.status : ''}`, true);
}
} }
// Helper function to count total rows in a CSV file // Helper function to count total rows in a CSV file
@@ -930,15 +950,16 @@ async function importPurchaseOrders(pool, filePath) {
} }
async function main() { async function main() {
outputProgress({
operation: 'Starting import process',
message: 'Creating connection pool...'
});
const startTime = Date.now(); const startTime = Date.now();
let pool; let pool;
let importInProgress = false;
try { try {
outputProgress({
operation: 'Starting import process',
message: 'Creating connection pool...'
});
pool = mysql.createPool(dbConfig); pool = mysql.createPool(dbConfig);
// Check if tables exist, if not create them // Check if tables exist, if not create them
@@ -970,6 +991,8 @@ async function main() {
// Import all data // Import all data
try { try {
importInProgress = true;
// Import products first since they're referenced by other tables // Import products first since they're referenced by other tables
await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv')); await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv'));
@@ -984,20 +1007,31 @@ async function main() {
importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv')) importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'))
]); ]);
outputProgress({ // Only output completion if we haven't encountered an error
status: 'complete', if (importInProgress) {
operation: 'Import process completed', outputProgress({
duration: formatDuration((Date.now() - startTime) / 1000) status: 'complete',
}); operation: 'Import process completed',
duration: formatDuration((Date.now() - startTime) / 1000)
});
}
} catch (error) { } catch (error) {
importInProgress = false;
logError(error, 'Error during import'); logError(error, 'Error during import');
outputProgress({
status: 'error',
operation: 'Import process',
error: error.message
});
throw error; throw error;
} }
} catch (error) { } catch (error) {
importInProgress = false;
logError(error, 'Fatal error during import process'); logError(error, 'Fatal error during import process');
outputProgress({ outputProgress({
status: 'error', status: 'error',
operation: 'Import process',
error: error.message error: error.message
}); });
process.exit(1); process.exit(1);

View File

@@ -528,31 +528,18 @@ export function Settings() {
return; return;
} }
if (!response.ok) { // Only check response status if we don't have any progress
const data = await response.json().catch(() => ({})); if (!importProgress?.current && !purchaseOrdersProgress?.current) {
// Only handle error if we don't have any progress yet if (!response.ok && response.status !== 200) {
if (!importProgress?.current && !purchaseOrdersProgress?.current) { const data = await response.json().catch(() => ({}));
throw new Error(data.error || 'Failed to start CSV import'); throw new Error(data.error || 'Failed to start CSV import');
} }
} else {
console.log('Response not ok but import is in progress, continuing...');
} }
} catch (error) { } catch (error) {
console.log('Import error:', error); handleError('CSV import', error instanceof Error ? error.message : 'Unknown error');
// If we have any progress, assume the import is still running
if (importProgress?.current || purchaseOrdersProgress?.current) {
console.log('Error occurred but import appears to be running');
return;
}
// Only clean up if we don't have any progress
if (eventSource) {
eventSource.close();
setEventSource(null);
}
setIsImporting(false); setIsImporting(false);
setImportProgress(null);
setPurchaseOrdersProgress(null);
handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
} }
}; };