Optimize imports, fix up tracking records and time overall
This commit is contained in:
@@ -357,7 +357,7 @@ async function syncSettingsProductTable() {
|
||||
* @param {string} config.historyType - Type identifier for calculate_history.
|
||||
* @param {string} config.statusModule - Module name for calculate_status.
|
||||
* @param {object} progress - Progress utility functions.
|
||||
* @returns {Promise<{success: boolean, message: string, duration: number}>}
|
||||
* @returns {Promise<{success: boolean, message: string, duration: number, rowsAffected: number}>}
|
||||
*/
|
||||
async function executeSqlStep(config, progress) {
|
||||
if (isCancelled) throw new Error(`Calculation skipped step ${config.name} due to prior cancellation.`);
|
||||
@@ -366,6 +366,7 @@ async function executeSqlStep(config, progress) {
|
||||
console.log(`\n--- Starting Step: ${config.name} ---`);
|
||||
const stepStartTime = Date.now();
|
||||
let connection = null;
|
||||
let rowsAffected = 0; // Track rows affected by this step
|
||||
|
||||
// Set timeout for this specific step
|
||||
if (stepTimeoutHandle) clearTimeout(stepTimeoutHandle); // Clear previous step's timeout
|
||||
@@ -414,7 +415,10 @@ async function executeSqlStep(config, progress) {
|
||||
current: 0, total: 100,
|
||||
elapsed: progress.formatElapsedTime(stepStartTime),
|
||||
remaining: 'Calculating...', rate: 0, percentage: '0',
|
||||
timing: { start_time: new Date(stepStartTime).toISOString() }
|
||||
timing: {
|
||||
start_time: new Date(stepStartTime).toISOString(),
|
||||
step_start_ms: stepStartTime
|
||||
}
|
||||
});
|
||||
|
||||
// 5. Execute the Main SQL Query
|
||||
@@ -423,15 +427,36 @@ async function executeSqlStep(config, progress) {
|
||||
operation: `Executing SQL: ${config.name}`,
|
||||
current: 25, total: 100,
|
||||
elapsed: progress.formatElapsedTime(stepStartTime),
|
||||
remaining: 'Executing...', rate: 0, percentage: '25',
|
||||
timing: { start_time: new Date(stepStartTime).toISOString() }
|
||||
remaining: 'Executing query...', rate: 0, percentage: '25',
|
||||
timing: {
|
||||
start_time: new Date(stepStartTime).toISOString(),
|
||||
step_start_ms: stepStartTime
|
||||
}
|
||||
});
|
||||
console.log(`Executing SQL for ${config.name}...`);
|
||||
|
||||
try {
|
||||
// Try executing exactly as individual scripts do
|
||||
console.log('Executing SQL with simple query method...');
|
||||
await connection.query(sqlQuery);
|
||||
const result = await connection.query(sqlQuery);
|
||||
|
||||
// Try to extract row count from result
|
||||
if (result && result.rowCount !== undefined) {
|
||||
rowsAffected = result.rowCount;
|
||||
} else if (Array.isArray(result) && result[0] && result[0].rowCount !== undefined) {
|
||||
rowsAffected = result[0].rowCount;
|
||||
}
|
||||
|
||||
// Check if the query returned a result set with row count info
|
||||
if (result && result.rows && result.rows.length > 0 && result.rows[0].rows_processed) {
|
||||
rowsAffected = parseInt(result.rows[0].rows_processed) || rowsAffected;
|
||||
console.log(`SQL returned metrics: ${JSON.stringify(result.rows[0])}`);
|
||||
} else if (Array.isArray(result) && result[0] && result[0].rows && result[0].rows[0] && result[0].rows[0].rows_processed) {
|
||||
rowsAffected = parseInt(result[0].rows[0].rows_processed) || rowsAffected;
|
||||
console.log(`SQL returned metrics: ${JSON.stringify(result[0].rows[0])}`);
|
||||
}
|
||||
|
||||
console.log(`SQL affected ${rowsAffected} rows`);
|
||||
} catch (sqlError) {
|
||||
if (sqlError.message.includes('could not determine data type of parameter')) {
|
||||
console.log('Simple query failed with parameter type error, trying alternative method...');
|
||||
@@ -492,7 +517,8 @@ async function executeSqlStep(config, progress) {
|
||||
return {
|
||||
success: true,
|
||||
message: `${config.name} completed successfully`,
|
||||
duration: stepDuration
|
||||
duration: stepDuration,
|
||||
rowsAffected: rowsAffected
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
@@ -664,6 +690,17 @@ async function runAllCalculations() {
|
||||
combinedHistoryId = historyResult.rows[0].id;
|
||||
console.log(`Created combined history record ID: ${combinedHistoryId}`);
|
||||
|
||||
// Get initial counts for tracking
|
||||
const productCount = await connection.query('SELECT COUNT(*) as count FROM products');
|
||||
const totalProducts = parseInt(productCount.rows[0].count);
|
||||
|
||||
// Update history with initial counts
|
||||
await connection.query(`
|
||||
UPDATE calculate_history
|
||||
SET additional_info = additional_info || jsonb_build_object('total_products', $1::integer)
|
||||
WHERE id = $2
|
||||
`, [totalProducts, combinedHistoryId]);
|
||||
|
||||
connection.release();
|
||||
} catch (historyError) {
|
||||
console.error('Error creating combined history record:', historyError);
|
||||
@@ -692,28 +729,49 @@ async function runAllCalculations() {
|
||||
|
||||
// Track completed steps
|
||||
const completedSteps = [];
|
||||
const stepTimings = {};
|
||||
const stepRowCounts = {};
|
||||
let currentStepIndex = 0;
|
||||
|
||||
// Now run the calculation steps
|
||||
for (const step of steps) {
|
||||
if (step.run) {
|
||||
if (isCancelled) {
|
||||
console.log(`Skipping step "${step.name}" due to cancellation.`);
|
||||
overallSuccess = false; // Mark as not fully successful if steps are skipped due to cancel
|
||||
continue; // Skip to next step
|
||||
for (const step of stepsToRun) {
|
||||
if (isCancelled) {
|
||||
console.log(`Skipping step "${step.name}" due to cancellation.`);
|
||||
overallSuccess = false; // Mark as not fully successful if steps are skipped due to cancel
|
||||
continue; // Skip to next step
|
||||
}
|
||||
|
||||
currentStepIndex++;
|
||||
|
||||
// Update overall progress
|
||||
progressUtils.outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Running calculations',
|
||||
message: `Step ${currentStepIndex} of ${stepsToRun.length}: ${step.name}`,
|
||||
current: currentStepIndex - 1,
|
||||
total: stepsToRun.length,
|
||||
elapsed: progressUtils.formatElapsedTime(overallStartTime),
|
||||
remaining: progressUtils.estimateRemaining(overallStartTime, currentStepIndex - 1, stepsToRun.length),
|
||||
percentage: Math.round(((currentStepIndex - 1) / stepsToRun.length) * 100).toString(),
|
||||
timing: {
|
||||
overall_start_time: new Date(overallStartTime).toISOString(),
|
||||
current_step: step.name,
|
||||
completed_steps: completedSteps.length
|
||||
}
|
||||
|
||||
// Pass the progress utilities to the step executor
|
||||
const result = await executeSqlStep(step, progressUtils);
|
||||
|
||||
if (result.success) {
|
||||
completedSteps.push({
|
||||
name: step.name,
|
||||
duration: result.duration,
|
||||
status: 'completed'
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.log(`Skipping step "${step.name}" (disabled by configuration).`);
|
||||
});
|
||||
|
||||
// Pass the progress utilities to the step executor
|
||||
const result = await executeSqlStep(step, progressUtils);
|
||||
|
||||
if (result.success) {
|
||||
completedSteps.push({
|
||||
name: step.name,
|
||||
duration: result.duration,
|
||||
status: 'completed',
|
||||
rowsAffected: result.rowsAffected
|
||||
});
|
||||
stepTimings[step.name] = result.duration;
|
||||
stepRowCounts[step.name] = result.rowsAffected;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -726,18 +784,32 @@ async function runAllCalculations() {
|
||||
connection = await getConnection();
|
||||
const totalDuration = Math.round((Date.now() - overallStartTime) / 1000);
|
||||
|
||||
// Get final processed counts
|
||||
const processedCounts = await connection.query(`
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM product_metrics WHERE last_calculated >= $1) as processed_products
|
||||
`, [new Date(overallStartTime)]);
|
||||
|
||||
await connection.query(`
|
||||
UPDATE calculate_history
|
||||
SET
|
||||
end_time = NOW(),
|
||||
duration_seconds = $1::integer,
|
||||
status = $2::calculation_status,
|
||||
additional_info = additional_info || jsonb_build_object('completed_steps', $3::jsonb)
|
||||
WHERE id = $4::integer;
|
||||
additional_info = additional_info || jsonb_build_object(
|
||||
'processed_products', $3::integer,
|
||||
'completed_steps', $4::jsonb,
|
||||
'step_timings', $5::jsonb,
|
||||
'step_row_counts', $6::jsonb
|
||||
)
|
||||
WHERE id = $7::integer;
|
||||
`, [
|
||||
totalDuration,
|
||||
isCancelled ? 'cancelled' : 'completed',
|
||||
JSON.stringify(completedSteps),
|
||||
isCancelled ? 'cancelled' : 'completed',
|
||||
processedCounts.rows[0].processed_products,
|
||||
JSON.stringify(completedSteps),
|
||||
JSON.stringify(stepTimings),
|
||||
JSON.stringify(stepRowCounts),
|
||||
combinedHistoryId
|
||||
]);
|
||||
|
||||
@@ -753,6 +825,26 @@ async function runAllCalculations() {
|
||||
overallSuccess = false;
|
||||
} else {
|
||||
console.log("\n--- All enabled calculations finished successfully ---");
|
||||
|
||||
// Send final completion progress
|
||||
progressUtils.outputProgress({
|
||||
status: 'complete',
|
||||
operation: 'All calculations completed',
|
||||
message: `Successfully completed ${completedSteps.length} of ${stepsToRun.length} steps`,
|
||||
current: stepsToRun.length,
|
||||
total: stepsToRun.length,
|
||||
elapsed: progressUtils.formatElapsedTime(overallStartTime),
|
||||
remaining: '0s',
|
||||
percentage: '100',
|
||||
timing: {
|
||||
overall_start_time: new Date(overallStartTime).toISOString(),
|
||||
overall_end_time: new Date().toISOString(),
|
||||
total_duration_seconds: Math.round((Date.now() - overallStartTime) / 1000),
|
||||
step_timings: stepTimings,
|
||||
completed_steps: completedSteps.length
|
||||
}
|
||||
});
|
||||
|
||||
progressUtils.clearProgress(); // Clear progress only on full success
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user