Update frontend to launch relevant scripts and show script history + output
This commit is contained in:
@@ -2,6 +2,7 @@ const express = require('express');
|
||||
const router = express.Router();
|
||||
const { spawn } = require('child_process');
|
||||
const path = require('path');
|
||||
const db = require('../utils/db');
|
||||
|
||||
// Debug middleware MUST be first
|
||||
router.use((req, res, next) => {
|
||||
@@ -9,9 +10,11 @@ router.use((req, res, next) => {
|
||||
next();
|
||||
});
|
||||
|
||||
// Store active import process and its progress
|
||||
// Store active processes and their progress
|
||||
let activeImport = null;
|
||||
let importProgress = null;
|
||||
let activeFullUpdate = null;
|
||||
let activeFullReset = null;
|
||||
|
||||
// SSE clients for progress updates
|
||||
const updateClients = new Set();
|
||||
@@ -19,17 +22,16 @@ const importClients = new Set();
|
||||
const resetClients = new Set();
|
||||
const resetMetricsClients = new Set();
|
||||
const calculateMetricsClients = new Set();
|
||||
const fullUpdateClients = new Set();
|
||||
const fullResetClients = new Set();
|
||||
|
||||
// Helper to send progress to specific clients
|
||||
function sendProgressToClients(clients, progress) {
|
||||
const data = typeof progress === 'string' ? { progress } : progress;
|
||||
|
||||
// Ensure we have a status field
|
||||
if (!data.status) {
|
||||
data.status = 'running';
|
||||
}
|
||||
|
||||
const message = `data: ${JSON.stringify(data)}\n\n`;
|
||||
function sendProgressToClients(clients, data) {
|
||||
// If data is a string, send it directly
|
||||
// If it's an object, convert it to JSON
|
||||
const message = typeof data === 'string'
|
||||
? `data: ${data}\n\n`
|
||||
: `data: ${JSON.stringify(data)}\n\n`;
|
||||
|
||||
clients.forEach(client => {
|
||||
try {
|
||||
@@ -45,115 +47,118 @@ function sendProgressToClients(clients, progress) {
|
||||
});
|
||||
}
|
||||
|
||||
// Helper to run a script and stream progress
|
||||
function runScript(scriptPath, type, clients) {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Kill any existing process of this type
|
||||
let activeProcess;
|
||||
switch (type) {
|
||||
case 'update':
|
||||
if (activeFullUpdate) {
|
||||
try { activeFullUpdate.kill(); } catch (e) { }
|
||||
}
|
||||
activeProcess = activeFullUpdate;
|
||||
break;
|
||||
case 'reset':
|
||||
if (activeFullReset) {
|
||||
try { activeFullReset.kill(); } catch (e) { }
|
||||
}
|
||||
activeProcess = activeFullReset;
|
||||
break;
|
||||
}
|
||||
|
||||
const child = spawn('node', [scriptPath], {
|
||||
stdio: ['inherit', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
switch (type) {
|
||||
case 'update':
|
||||
activeFullUpdate = child;
|
||||
break;
|
||||
case 'reset':
|
||||
activeFullReset = child;
|
||||
break;
|
||||
}
|
||||
|
||||
let output = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
const text = data.toString();
|
||||
output += text;
|
||||
// Send raw output directly
|
||||
sendProgressToClients(clients, text);
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
const text = data.toString();
|
||||
console.error(text);
|
||||
// Send stderr output directly too
|
||||
sendProgressToClients(clients, text);
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
switch (type) {
|
||||
case 'update':
|
||||
activeFullUpdate = null;
|
||||
break;
|
||||
case 'reset':
|
||||
activeFullReset = null;
|
||||
break;
|
||||
}
|
||||
|
||||
if (code !== 0) {
|
||||
const error = `Script ${scriptPath} exited with code ${code}`;
|
||||
sendProgressToClients(clients, error);
|
||||
reject(new Error(error));
|
||||
} else {
|
||||
sendProgressToClients(clients, `${type} completed successfully`);
|
||||
resolve({ output });
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
switch (type) {
|
||||
case 'update':
|
||||
activeFullUpdate = null;
|
||||
break;
|
||||
case 'reset':
|
||||
activeFullReset = null;
|
||||
break;
|
||||
}
|
||||
sendProgressToClients(clients, err.message);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Progress endpoints
|
||||
router.get('/update/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the update set
|
||||
updateClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
updateClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/import/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the import set
|
||||
importClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
importClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/reset/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the reset set
|
||||
resetClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
resetClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Add reset-metrics progress endpoint
|
||||
router.get('/reset-metrics/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the reset-metrics set
|
||||
resetMetricsClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
resetMetricsClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Add calculate-metrics progress endpoint
|
||||
router.get('/calculate-metrics/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send current progress if it exists
|
||||
if (importProgress) {
|
||||
res.write(`data: ${JSON.stringify(importProgress)}\n\n`);
|
||||
} else {
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
router.get('/:type/progress', (req, res) => {
|
||||
const { type } = req.params;
|
||||
if (!['update', 'reset'].includes(type)) {
|
||||
return res.status(400).json({ error: 'Invalid operation type' });
|
||||
}
|
||||
|
||||
// Add this client to the calculate-metrics set
|
||||
calculateMetricsClients.add(res);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Remove client when connection closes
|
||||
// Add this client to the correct set
|
||||
const clients = type === 'update' ? fullUpdateClients : fullResetClients;
|
||||
clients.add(res);
|
||||
|
||||
// Send initial connection message
|
||||
sendProgressToClients(new Set([res]), JSON.stringify({
|
||||
status: 'running',
|
||||
operation: 'Initializing connection...'
|
||||
}));
|
||||
|
||||
// Handle client disconnect
|
||||
req.on('close', () => {
|
||||
calculateMetricsClients.delete(res);
|
||||
clients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,7 +179,6 @@ router.get('/status', (req, res) => {
|
||||
|
||||
// Add calculate-metrics status endpoint
|
||||
router.get('/calculate-metrics/status', (req, res) => {
|
||||
console.log('Calculate metrics status endpoint hit');
|
||||
const calculateMetrics = require('../../scripts/calculate-metrics');
|
||||
const progress = calculateMetrics.getProgress();
|
||||
|
||||
@@ -371,49 +375,35 @@ router.post('/import', async (req, res) => {
|
||||
|
||||
// Route to cancel active process
|
||||
router.post('/cancel', (req, res) => {
|
||||
if (!activeImport) {
|
||||
return res.status(404).json({ error: 'No active process to cancel' });
|
||||
let killed = false;
|
||||
|
||||
// Get the operation type from the request
|
||||
const { type } = req.query;
|
||||
const clients = type === 'update' ? fullUpdateClients : fullResetClients;
|
||||
const activeProcess = type === 'update' ? activeFullUpdate : activeFullReset;
|
||||
|
||||
if (activeProcess) {
|
||||
try {
|
||||
activeProcess.kill('SIGTERM');
|
||||
if (type === 'update') {
|
||||
activeFullUpdate = null;
|
||||
} else {
|
||||
activeFullReset = null;
|
||||
}
|
||||
killed = true;
|
||||
sendProgressToClients(clients, JSON.stringify({
|
||||
status: 'cancelled',
|
||||
operation: 'Operation cancelled'
|
||||
}));
|
||||
} catch (err) {
|
||||
console.error(`Error killing ${type} process:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// If it's the prod import module, call its cancel function
|
||||
if (typeof activeImport.cancelImport === 'function') {
|
||||
activeImport.cancelImport();
|
||||
} else {
|
||||
// Otherwise it's a child process
|
||||
activeImport.kill('SIGTERM');
|
||||
}
|
||||
|
||||
// Get the operation type from the request
|
||||
const { operation } = req.query;
|
||||
|
||||
// Send cancel message only to the appropriate client set
|
||||
const cancelMessage = {
|
||||
status: 'cancelled',
|
||||
operation: 'Operation cancelled'
|
||||
};
|
||||
|
||||
switch (operation) {
|
||||
case 'update':
|
||||
sendProgressToClients(updateClients, cancelMessage);
|
||||
break;
|
||||
case 'import':
|
||||
sendProgressToClients(importClients, cancelMessage);
|
||||
break;
|
||||
case 'reset':
|
||||
sendProgressToClients(resetClients, cancelMessage);
|
||||
break;
|
||||
case 'calculate-metrics':
|
||||
sendProgressToClients(calculateMetricsClients, cancelMessage);
|
||||
break;
|
||||
}
|
||||
|
||||
if (killed) {
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
// Even if there's an error, try to clean up
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
res.status(500).json({ error: 'Failed to cancel process' });
|
||||
} else {
|
||||
res.status(404).json({ error: 'No active process to cancel' });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -552,20 +542,6 @@ router.post('/reset-metrics', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Add calculate-metrics status endpoint
|
||||
router.get('/calculate-metrics/status', (req, res) => {
|
||||
const calculateMetrics = require('../../scripts/calculate-metrics');
|
||||
const progress = calculateMetrics.getProgress();
|
||||
|
||||
// Only consider it active if both the process is running and we have progress
|
||||
const isActive = !!activeImport && !!progress;
|
||||
|
||||
res.json({
|
||||
active: isActive,
|
||||
progress: isActive ? progress : null
|
||||
});
|
||||
});
|
||||
|
||||
// Add calculate-metrics endpoint
|
||||
router.post('/calculate-metrics', async (req, res) => {
|
||||
if (activeImport) {
|
||||
@@ -711,4 +687,96 @@ router.post('/import-from-prod', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// POST /csv/full-update - Run full update script
|
||||
router.post('/full-update', async (req, res) => {
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '../../scripts/full-update.js');
|
||||
runScript(scriptPath, 'update', fullUpdateClients)
|
||||
.catch(error => {
|
||||
console.error('Update failed:', error);
|
||||
});
|
||||
res.status(202).json({ message: 'Update started' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /csv/full-reset - Run full reset script
|
||||
router.post('/full-reset', async (req, res) => {
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '../../scripts/full-reset.js');
|
||||
runScript(scriptPath, 'reset', fullResetClients)
|
||||
.catch(error => {
|
||||
console.error('Reset failed:', error);
|
||||
});
|
||||
res.status(202).json({ message: 'Reset started' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /history/import - Get recent import history
|
||||
router.get('/history/import', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const [rows] = await pool.query(`
|
||||
SELECT * FROM import_history
|
||||
ORDER BY start_time DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
res.json(rows || []);
|
||||
} catch (error) {
|
||||
console.error('Error fetching import history:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /history/calculate - Get recent calculation history
|
||||
router.get('/history/calculate', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const [rows] = await pool.query(`
|
||||
SELECT * FROM calculate_history
|
||||
ORDER BY start_time DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
res.json(rows || []);
|
||||
} catch (error) {
|
||||
console.error('Error fetching calculate history:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /status/modules - Get module calculation status
|
||||
router.get('/status/modules', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const [rows] = await pool.query(`
|
||||
SELECT module_name, last_calculation_timestamp
|
||||
FROM calculate_status
|
||||
ORDER BY module_name
|
||||
`);
|
||||
res.json(rows || []);
|
||||
} catch (error) {
|
||||
console.error('Error fetching module status:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /status/tables - Get table sync status
|
||||
router.get('/status/tables', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const [rows] = await pool.query(`
|
||||
SELECT table_name, last_sync_timestamp
|
||||
FROM sync_status
|
||||
ORDER BY table_name
|
||||
`);
|
||||
res.json(rows || []);
|
||||
} catch (error) {
|
||||
console.error('Error fetching table status:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Reference in New Issue
Block a user