Fix time out error on data import and fix regression on progress display + sort out core and metrics schemas better
This commit is contained in:
@@ -94,119 +94,102 @@ export function Settings() {
|
||||
setEventSource(null);
|
||||
}
|
||||
|
||||
try {
|
||||
console.log(`Creating new EventSource for ${config.apiUrl}/csv/${type}/progress`);
|
||||
const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
|
||||
withCredentials: true
|
||||
});
|
||||
|
||||
// Set up handlers before setting state
|
||||
source.onopen = () => {
|
||||
console.log('EventSource connected successfully');
|
||||
// Set event source state only after successful connection
|
||||
setEventSource(source);
|
||||
};
|
||||
let retryCount = 0;
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY = 2000; // 2 seconds
|
||||
|
||||
source.onerror = (event) => {
|
||||
console.error('EventSource failed:', event);
|
||||
source.close();
|
||||
const setupConnection = () => {
|
||||
try {
|
||||
console.log(`Creating new EventSource for ${config.apiUrl}/csv/${type}/progress`);
|
||||
const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
|
||||
withCredentials: true
|
||||
});
|
||||
|
||||
// Reset states based on type
|
||||
switch (type) {
|
||||
case 'update':
|
||||
setIsUpdating(false);
|
||||
setUpdateProgress(null);
|
||||
break;
|
||||
case 'import':
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
break;
|
||||
case 'reset':
|
||||
setIsResetting(false);
|
||||
setResetProgress(null);
|
||||
break;
|
||||
case 'reset-metrics':
|
||||
setIsResettingMetrics(false);
|
||||
setResetMetricsProgress(null);
|
||||
break;
|
||||
case 'calculate-metrics':
|
||||
setIsCalculatingMetrics(false);
|
||||
setMetricsProgress(null);
|
||||
break;
|
||||
}
|
||||
|
||||
setEventSource(null);
|
||||
handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Lost connection to server');
|
||||
};
|
||||
source.onopen = () => {
|
||||
console.log('EventSource connected successfully');
|
||||
retryCount = 0; // Reset retry count on successful connection
|
||||
setEventSource(source);
|
||||
};
|
||||
|
||||
source.onmessage = (event) => {
|
||||
try {
|
||||
console.log(`Received message for ${type}:`, event.data);
|
||||
source.onerror = async (event) => {
|
||||
console.error('EventSource error:', event);
|
||||
source.close();
|
||||
|
||||
// First parse the outer message
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
// If we have a progress field that's a string containing multiple JSON objects
|
||||
if (data.progress && typeof data.progress === 'string') {
|
||||
// Split the progress string into separate JSON objects
|
||||
const progressMessages = data.progress
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((message: string) => {
|
||||
try {
|
||||
return JSON.parse(message);
|
||||
} catch (err) {
|
||||
console.warn('Failed to parse progress message:', message, err);
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((msg: unknown): msg is ImportProgress => msg !== null);
|
||||
// Only retry if we haven't exceeded max retries and we're still in the active state
|
||||
const isActive = type === 'import' ? isImporting :
|
||||
type === 'update' ? isUpdating :
|
||||
type === 'reset' ? isResetting :
|
||||
type === 'reset-metrics' ? isResettingMetrics :
|
||||
type === 'calculate-metrics' ? isCalculatingMetrics : false;
|
||||
|
||||
// Process each progress message
|
||||
progressMessages.forEach((progressData: ImportProgress) => {
|
||||
handleProgressUpdate(type, progressData, source);
|
||||
});
|
||||
} else {
|
||||
// Handle single message case
|
||||
const progressData = data.progress || data;
|
||||
handleProgressUpdate(type, progressData, source);
|
||||
if (retryCount < MAX_RETRIES && isActive) {
|
||||
console.log(`Retrying connection (${retryCount + 1}/${MAX_RETRIES})...`);
|
||||
retryCount++;
|
||||
// Wait before retrying
|
||||
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
|
||||
setupConnection();
|
||||
} else if (retryCount >= MAX_RETRIES) {
|
||||
console.log('Max retries exceeded, but operation may still be running...');
|
||||
// Don't reset states or show error, just log it
|
||||
console.warn(`Lost connection to ${type} progress stream after ${MAX_RETRIES} retries`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing event data:', error, event.data);
|
||||
handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Failed to parse server response');
|
||||
}
|
||||
};
|
||||
|
||||
setEventSource(null);
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to set up EventSource:', error);
|
||||
|
||||
// Reset operation state
|
||||
switch (type) {
|
||||
case 'update':
|
||||
setIsUpdating(false);
|
||||
setUpdateProgress(null);
|
||||
break;
|
||||
case 'import':
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
break;
|
||||
case 'reset':
|
||||
setIsResetting(false);
|
||||
setResetProgress(null);
|
||||
break;
|
||||
case 'reset-metrics':
|
||||
setIsResettingMetrics(false);
|
||||
setResetMetricsProgress(null);
|
||||
break;
|
||||
case 'calculate-metrics':
|
||||
setIsCalculatingMetrics(false);
|
||||
setMetricsProgress(null);
|
||||
break;
|
||||
source.onmessage = (event) => {
|
||||
try {
|
||||
console.log(`Received message for ${type}:`, event.data);
|
||||
|
||||
// First parse the outer message
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
// If we have a progress field that's a string containing multiple JSON objects
|
||||
if (data.progress && typeof data.progress === 'string') {
|
||||
// Split the progress string into separate JSON objects
|
||||
const progressMessages = data.progress
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((message: string) => {
|
||||
try {
|
||||
return JSON.parse(message);
|
||||
} catch (err) {
|
||||
console.warn('Failed to parse progress message:', message, err);
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((msg: unknown): msg is ImportProgress => msg !== null);
|
||||
|
||||
// Process each progress message
|
||||
progressMessages.forEach((progressData: ImportProgress) => {
|
||||
handleProgressUpdate(type, progressData, source);
|
||||
});
|
||||
} else {
|
||||
// Handle single message case
|
||||
const progressData = data.progress || data;
|
||||
handleProgressUpdate(type, progressData, source);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing event data:', error, event.data);
|
||||
// Don't show error to user for parsing issues, just log them
|
||||
console.warn('Failed to parse server response:', error);
|
||||
}
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to set up EventSource:', error);
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
console.log(`Retrying connection (${retryCount + 1}/${MAX_RETRIES})...`);
|
||||
retryCount++;
|
||||
setTimeout(setupConnection, RETRY_DELAY);
|
||||
} else {
|
||||
console.log('Max retries exceeded, but operation may still be running...');
|
||||
}
|
||||
}
|
||||
|
||||
handleError(type.charAt(0).toUpperCase() + type.slice(1), 'Failed to connect to server');
|
||||
}
|
||||
}, [eventSource, handleComplete, handleError]);
|
||||
};
|
||||
|
||||
setupConnection();
|
||||
}, [eventSource, handleComplete, handleError, isImporting, isUpdating, isResetting, isResettingMetrics, isCalculatingMetrics]);
|
||||
|
||||
// Helper function to process a single progress update
|
||||
const handleProgressUpdate = (
|
||||
@@ -225,6 +208,18 @@ export function Settings() {
|
||||
status: progressData.status || 'running'
|
||||
};
|
||||
|
||||
// For import type, handle orders and purchase orders separately
|
||||
if (type === 'import' && progressData.operation) {
|
||||
const operation = progressData.operation.toLowerCase();
|
||||
if (operation.includes('purchase orders')) {
|
||||
setPurchaseOrdersProgress(prev => ({
|
||||
...prev,
|
||||
...processedData
|
||||
}));
|
||||
return; // Don't update main import progress for PO updates
|
||||
}
|
||||
}
|
||||
|
||||
// Update progress state based on type
|
||||
switch (type) {
|
||||
case 'update':
|
||||
@@ -261,6 +256,29 @@ export function Settings() {
|
||||
// Handle completion
|
||||
if (progressData.status === 'complete') {
|
||||
console.log(`Operation ${type} completed`);
|
||||
|
||||
// For import, only close connection when both operations are complete
|
||||
if (type === 'import') {
|
||||
const operation = progressData.operation?.toLowerCase() || '';
|
||||
if (operation.includes('purchase orders')) {
|
||||
setPurchaseOrdersProgress(null);
|
||||
} else {
|
||||
setImportProgress(null);
|
||||
}
|
||||
|
||||
// Only fully complete if both are done
|
||||
if (!importProgress || !purchaseOrdersProgress) {
|
||||
source.close();
|
||||
setEventSource(null);
|
||||
setIsImporting(false);
|
||||
if (!progressData.operation?.includes('cancelled')) {
|
||||
handleComplete(`${type.charAt(0).toUpperCase() + type.slice(1)}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// For other operations, close immediately
|
||||
source.close();
|
||||
setEventSource(null);
|
||||
|
||||
@@ -270,11 +288,6 @@ export function Settings() {
|
||||
setIsUpdating(false);
|
||||
setUpdateProgress(null);
|
||||
break;
|
||||
case 'import':
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
setPurchaseOrdersProgress(null);
|
||||
break;
|
||||
case 'reset':
|
||||
setIsResetting(false);
|
||||
setResetProgress(null);
|
||||
@@ -306,6 +319,8 @@ export function Settings() {
|
||||
break;
|
||||
case 'import':
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
setPurchaseOrdersProgress(null);
|
||||
break;
|
||||
case 'reset':
|
||||
setIsResetting(false);
|
||||
@@ -489,7 +504,7 @@ export function Settings() {
|
||||
// Connect to SSE for progress updates first
|
||||
connectToEventSource('import');
|
||||
|
||||
// Make the import request
|
||||
// Make the import request - no timeout needed since this just starts the process
|
||||
const response = await fetch(`${config.apiUrl}/csv/import`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
@@ -497,8 +512,22 @@ export function Settings() {
|
||||
},
|
||||
credentials: 'include',
|
||||
body: JSON.stringify(limits)
|
||||
}).catch(error => {
|
||||
// If we already have progress, ignore network errors
|
||||
if ((importProgress?.current || purchaseOrdersProgress?.current) &&
|
||||
(error.name === 'TypeError')) {
|
||||
console.log('Request error but import is in progress:', error);
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
|
||||
// If response is null, it means we caught an error but have progress
|
||||
if (!response) {
|
||||
console.log('Continuing with existing progress...');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
// Only handle error if we don't have any progress yet
|
||||
@@ -507,17 +536,23 @@ export function Settings() {
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Only clean up if we don't have any progress
|
||||
if (!importProgress?.current && !purchaseOrdersProgress?.current) {
|
||||
if (eventSource) {
|
||||
eventSource.close();
|
||||
setEventSource(null);
|
||||
}
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
setPurchaseOrdersProgress(null);
|
||||
handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
|
||||
console.log('Import error:', error);
|
||||
|
||||
// If we have any progress, assume the import is still running
|
||||
if (importProgress?.current || purchaseOrdersProgress?.current) {
|
||||
console.log('Error occurred but import appears to be running');
|
||||
return;
|
||||
}
|
||||
|
||||
// Only clean up if we don't have any progress
|
||||
if (eventSource) {
|
||||
eventSource.close();
|
||||
setEventSource(null);
|
||||
}
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
setPurchaseOrdersProgress(null);
|
||||
handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -783,12 +818,12 @@ export function Settings() {
|
||||
<Button
|
||||
className="flex-1"
|
||||
onClick={handleImportCSV}
|
||||
disabled={isUpdating || isImporting}
|
||||
disabled={isImporting || isUpdating || isResetting}
|
||||
>
|
||||
{isImporting ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Importing...
|
||||
Importing Data...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
@@ -810,49 +845,8 @@ export function Settings() {
|
||||
|
||||
{isImporting && (
|
||||
<div className="space-y-4">
|
||||
{/* Show products progress */}
|
||||
{importProgress?.operation?.toLowerCase().includes('products') && (
|
||||
<div>
|
||||
{renderProgress(importProgress)}
|
||||
</div>
|
||||
)}
|
||||
{/* Show orders progress */}
|
||||
{importProgress?.operation?.toLowerCase().includes('orders import') &&
|
||||
!importProgress.operation.toLowerCase().includes('purchase') && (
|
||||
<div>
|
||||
{renderProgress(importProgress)}
|
||||
</div>
|
||||
)}
|
||||
{/* Show purchase orders progress */}
|
||||
{purchaseOrdersProgress && (
|
||||
<div>
|
||||
{renderProgress(purchaseOrdersProgress)}
|
||||
</div>
|
||||
)}
|
||||
{/* Show metrics calculation progress */}
|
||||
{importProgress?.operation?.toLowerCase().includes('metrics') && (
|
||||
<div>
|
||||
<Progress value={Number(importProgress.percentage)} className="mb-2" />
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{importProgress.message || importProgress.operation || 'Calculating metrics...'}
|
||||
{importProgress.current && importProgress.total && (
|
||||
<> ({importProgress.current} of {importProgress.total})</>
|
||||
)}
|
||||
</p>
|
||||
{importProgress.elapsed && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Elapsed: {importProgress.elapsed}
|
||||
{importProgress.remaining && <> • Remaining: {importProgress.remaining}</>}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{/* Show vendor metrics progress */}
|
||||
{importProgress?.operation?.toLowerCase().includes('vendor metrics') && (
|
||||
<div>
|
||||
{renderProgress(importProgress)}
|
||||
</div>
|
||||
)}
|
||||
{importProgress && renderProgress(importProgress)}
|
||||
{purchaseOrdersProgress && renderProgress(purchaseOrdersProgress)}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
|
||||
Reference in New Issue
Block a user