Fix csv update/import on settings page + lots of cors work
This commit is contained in:
@@ -14,15 +14,77 @@ let activeImport = null;
|
||||
let importProgress = null;
|
||||
|
||||
// SSE clients for progress updates
|
||||
const clients = new Set();
|
||||
const updateClients = new Set();
|
||||
const importClients = new Set();
|
||||
|
||||
// Helper to send progress to all connected clients
|
||||
function sendProgressToClients(progress) {
|
||||
// Helper to send progress to specific clients
|
||||
function sendProgressToClients(clients, progress) {
|
||||
const data = typeof progress === 'string' ? { progress } : progress;
|
||||
|
||||
// Ensure we have a status field
|
||||
if (!data.status) {
|
||||
data.status = 'running';
|
||||
}
|
||||
|
||||
const message = `data: ${JSON.stringify(data)}\n\n`;
|
||||
|
||||
clients.forEach(client => {
|
||||
client.write(`data: ${JSON.stringify(progress)}\n\n`);
|
||||
try {
|
||||
client.write(message);
|
||||
// Immediately flush the response
|
||||
if (typeof client.flush === 'function') {
|
||||
client.flush();
|
||||
}
|
||||
} catch (error) {
|
||||
// Silently remove failed client
|
||||
clients.delete(client);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Progress endpoints
|
||||
router.get('/update/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the update set
|
||||
updateClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
updateClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/import/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the import set
|
||||
importClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
importClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Debug endpoint to verify route registration
|
||||
router.get('/test', (req, res) => {
|
||||
console.log('CSV test endpoint hit');
|
||||
@@ -39,45 +101,72 @@ router.get('/status', (req, res) => {
|
||||
});
|
||||
|
||||
// Route to update CSV files
|
||||
router.post('/update', async (req, res) => {
|
||||
console.log('CSV update endpoint hit');
|
||||
|
||||
router.post('/update', async (req, res, next) => {
|
||||
if (activeImport) {
|
||||
console.log('Import already in progress');
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'update-csv.js');
|
||||
console.log('Running script:', scriptPath);
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
console.error('Script not found:', scriptPath);
|
||||
return res.status(500).json({ error: 'Update script not found' });
|
||||
}
|
||||
|
||||
activeImport = spawn('node', [scriptPath]);
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
console.log(`CSV Update: ${data}`);
|
||||
importProgress = data.toString();
|
||||
sendProgressToClients({ status: 'running', progress: importProgress });
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'running',
|
||||
...jsonData
|
||||
});
|
||||
} catch (e) {
|
||||
// If not JSON, send as plain progress
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'running',
|
||||
progress: output
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
console.error(`CSV Update Error: ${data}`);
|
||||
sendProgressToClients({ status: 'error', error: data.toString() });
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code) => {
|
||||
console.log(`CSV update process exited with code ${code}`);
|
||||
if (code === 0) {
|
||||
sendProgressToClients({ status: 'complete' });
|
||||
// Don't treat cancellation (code 143/SIGTERM) as an error
|
||||
if (code === 0 || code === 143) {
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'complete',
|
||||
operation: code === 143 ? 'Operation cancelled' : 'Update complete'
|
||||
});
|
||||
resolve();
|
||||
} else {
|
||||
sendProgressToClients({ status: 'error', error: `Process exited with code ${code}` });
|
||||
reject(new Error(`Update process exited with code ${code}`));
|
||||
const errorMsg = `Update process exited with code ${code}`;
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error: errorMsg
|
||||
});
|
||||
reject(new Error(errorMsg));
|
||||
}
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
@@ -89,7 +178,138 @@ router.post('/update', async (req, res) => {
|
||||
console.error('Error updating CSV files:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
res.status(500).json({ error: 'Failed to update CSV files', details: error.message });
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
// Route to import CSV files
|
||||
router.post('/import', async (req, res) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'import-csv.js');
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
return res.status(500).json({ error: 'Import script not found' });
|
||||
}
|
||||
|
||||
// Get test limits from request body
|
||||
const { products = 0, orders = 10000, purchaseOrders = 10000 } = req.body;
|
||||
|
||||
// Create environment variables for the script
|
||||
const env = {
|
||||
...process.env,
|
||||
PRODUCTS_TEST_LIMIT: products.toString(),
|
||||
ORDERS_TEST_LIMIT: orders.toString(),
|
||||
PURCHASE_ORDERS_TEST_LIMIT: purchaseOrders.toString()
|
||||
};
|
||||
|
||||
activeImport = spawn('node', [scriptPath], { env });
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'running',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
// If not JSON, send as plain progress
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'running',
|
||||
progress: output
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code) => {
|
||||
// Don't treat cancellation (code 143/SIGTERM) as an error
|
||||
if (code === 0 || code === 143) {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'complete',
|
||||
operation: code === 143 ? 'Operation cancelled' : 'Import complete'
|
||||
});
|
||||
resolve();
|
||||
} else {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error: `Process exited with code ${code}`
|
||||
});
|
||||
reject(new Error(`Import process exited with code ${code}`));
|
||||
}
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
});
|
||||
});
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error importing CSV files:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to import CSV files', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Route to cancel active process
|
||||
router.post('/cancel', (req, res) => {
|
||||
if (!activeImport) {
|
||||
return res.status(404).json({ error: 'No active process to cancel' });
|
||||
}
|
||||
|
||||
try {
|
||||
// Kill the process
|
||||
activeImport.kill();
|
||||
|
||||
// Clean up
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
|
||||
// Notify all clients
|
||||
const cancelMessage = {
|
||||
status: 'complete',
|
||||
operation: 'Operation cancelled'
|
||||
};
|
||||
sendProgressToClients(updateClients, cancelMessage);
|
||||
sendProgressToClients(importClients, cancelMessage);
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
// Even if there's an error, try to clean up
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
res.status(500).json({ error: 'Failed to cancel process' });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user