Dashboard tweaks, add bulk /check-upcs endpoint for import validation

This commit is contained in:
2026-07-27 13:47:06 -04:00
parent 6c973f0a9d
commit 4e4a09ee3f
5 changed files with 345 additions and 179 deletions
@@ -38,26 +38,54 @@ const getImageUrls = (pid, iid = 1) => {
};
};
// Main stats endpoint - replaces /api/klaviyo/events/stats
router.get('/stats', async (req, res) => {
const startTime = Date.now();
console.log(`[STATS] Starting request for timeRange: ${req.query.timeRange}`);
// Set a timeout for the entire operation
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => reject(new Error('Request timeout after 15 seconds')), 15000);
});
try {
const mainOperation = async () => {
const { timeRange, startDate, endDate, excludeCherryBox } = req.query;
const excludeCB = parseBoolParam(excludeCherryBox);
console.log(`[STATS] Getting DB connection... (excludeCherryBox: ${excludeCB})`);
const { connection, release } = await getDbConnection();
console.log(`[STATS] DB connection obtained in ${Date.now() - startTime}ms`);
try {
// The DB sits across an SSH tunnel (~190ms per round trip; the queries
// themselves are ~5ms on the server), so /stats response time is dominated by
// round-trip count, not query cost. Two mitigations:
// 1. computeStats runs its queries over several pooled connections (mysql2
// serializes per connection), capped so concurrent refreshes can't starve
// the 20-slot pool.
// 2. Responses are cached per param-set and served stale-while-revalidate:
// the dashboards poll every 30-60s, so they get the cached payload
// instantly while a single-flight background refresh keeps it current.
const STATS_QUERY_WIDTH = 4;
const STATS_CACHE_TTL = 60 * 1000; // younger than this: serve as-is
const STATS_CACHE_MAX_STALE = 15 * 60 * 1000; // older than this: block on refresh
const statsCache = new Map(); // cacheKey -> { data, timestamp }
const statsInflight = new Map(); // cacheKey -> Promise<response>
const withTimeout = (promise, ms, label) => {
let timer;
const timeout = new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`${label} timeout after ${ms / 1000} seconds`)), ms);
});
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
};
// Drain [name, fn] tasks with up to `width` workers, each holding its own
// pooled connection, so wall time is ~ceil(tasks / width) round trips. If a
// task rejects, remaining queued tasks are abandoned and every worker still
// releases its connection (mirrors the old single-connection failure mode).
async function runStatsTasks(tasks, width) {
const results = {};
const queue = [...tasks];
const worker = async () => {
const { connection, release } = await getDbConnection();
try {
let task;
while ((task = queue.shift()) !== undefined) {
results[task[0]] = await task[1](connection);
}
} finally {
release();
}
};
await Promise.all(Array.from({ length: Math.min(width, tasks.length) }, worker));
return results;
}
async function computeStats(timeRange, startDate, endDate, excludeCB) {
const { whereClause, params, dateRange } = getTimeRangeConditions(timeRange, startDate, endDate);
const isSingleDay = ['today', 'yesterday'].includes(timeRange);
// Main order stats query (optionally excludes Cherry Box orders)
// Note: order_status > 15 excludes cancelled (15), so cancelled stats are queried separately
@@ -75,9 +103,6 @@ router.get('/stats', async (req, res) => {
WHERE order_status > 15 AND ${getCherryBoxClause(excludeCB)} AND ${whereClause}
`;
const [mainStats] = await connection.execute(mainStatsQuery, params);
const stats = mainStats[0];
// Cancelled orders query - uses date_cancelled instead of date_placed
// Shows orders cancelled during the selected period, regardless of when they were placed
const cancelledQuery = `
@@ -90,9 +115,6 @@ router.get('/stats', async (req, res) => {
AND ${whereClause.replace(/date_placed/g, 'date_cancelled')}
`;
const [cancelledResult] = await connection.execute(cancelledQuery, params);
const cancelledStats = cancelledResult[0] || { cancelledCount: 0, cancelledTotal: 0 };
// Refunds query (optionally excludes Cherry Box orders)
const refundsQuery = `
SELECT
@@ -103,8 +125,6 @@ router.get('/stats', async (req, res) => {
WHERE payment_amount < 0 AND o.order_status > 15 AND ${getCherryBoxClauseAliased('o', excludeCB)} AND ${whereClause.replace(/date_placed/g, 'o.date_placed')}
`;
const [refundStats] = await connection.execute(refundsQuery, params);
// Shipped orders query - uses date_shipped instead of date_placed
// This counts orders that were SHIPPED during the selected period, regardless of when they were placed
const shippedQuery = `
@@ -115,9 +135,6 @@ router.get('/stats', async (req, res) => {
AND ${whereClause.replace(/date_placed/g, 'date_shipped')}
`;
const [shippedResult] = await connection.execute(shippedQuery, params);
const shippedCount = parseInt(shippedResult[0]?.shippedCount || 0);
// Best revenue day query (optionally excludes Cherry Box orders)
const bestDayQuery = `
SELECT
@@ -131,78 +148,38 @@ router.get('/stats', async (req, res) => {
LIMIT 1
`;
const [bestDayResult] = await connection.execute(bestDayQuery, params);
// Peak hour query - uses selected time range for the card value.
// date_placed stores Central wall-clock; the office reads the axis in
// Eastern, and ET = CT + 1h year-round, so shift before taking HOUR().
let peakHour = null;
if (['today', 'yesterday'].includes(timeRange)) {
const peakHourQuery = `
SELECT
HOUR(ADDTIME(date_placed, '01:00:00')) as hour,
COUNT(*) as count
FROM _order
WHERE order_status > 15 AND ${getCherryBoxClause(excludeCB)} AND ${whereClause}
GROUP BY HOUR(ADDTIME(date_placed, '01:00:00'))
ORDER BY count DESC
LIMIT 1
`;
const [peakHourResult] = await connection.execute(peakHourQuery, params);
if (peakHourResult.length > 0) {
const hour = parseInt(peakHourResult[0].hour);
peakHour = {
hour,
count: parseInt(peakHourResult[0].count),
displayHour: DateTime.fromObject({ hour }).toFormat('h a')
};
}
}
// Only run for single-day ranges (see tasks below).
const peakHourQuery = `
SELECT
HOUR(ADDTIME(date_placed, '01:00:00')) as hour,
COUNT(*) as count
FROM _order
WHERE order_status > 15 AND ${getCherryBoxClause(excludeCB)} AND ${whereClause}
GROUP BY HOUR(ADDTIME(date_placed, '01:00:00'))
ORDER BY count DESC
LIMIT 1
`;
// Hourly breakdown for detail chart - always rolling 24 hours (like revenue/orders use 30 days)
// Returns data ordered chronologically: [24hrs ago, 23hrs ago, ..., 1hr ago, current hour]
let hourlyOrders = null;
if (['today', 'yesterday'].includes(timeRange)) {
// Hourly counts in EASTERN display hours (column stores Central
// wall-clock; ET = CT + 1h year-round). currentHour comes from the same
// expression so the rolling window stays aligned.
const hourlyQuery = `
SELECT
HOUR(ADDTIME(date_placed, '01:00:00')) as hour,
COUNT(*) as count,
HOUR(ADDTIME(NOW(), '01:00:00')) as currentHour
FROM _order
WHERE order_status > 15
AND ${getCherryBoxClause(excludeCB)}
AND date_placed >= NOW() - INTERVAL 24 HOUR
GROUP BY HOUR(ADDTIME(date_placed, '01:00:00'))
`;
// Hourly counts in EASTERN display hours (column stores Central
// wall-clock; ET = CT + 1h year-round). currentHour comes from the same
// expression so the rolling window stays aligned.
const hourlyQuery = `
SELECT
HOUR(ADDTIME(date_placed, '01:00:00')) as hour,
COUNT(*) as count,
HOUR(ADDTIME(NOW(), '01:00:00')) as currentHour
FROM _order
WHERE order_status > 15
AND ${getCherryBoxClause(excludeCB)}
AND date_placed >= NOW() - INTERVAL 24 HOUR
GROUP BY HOUR(ADDTIME(date_placed, '01:00:00'))
`;
const [hourlyResult] = await connection.execute(hourlyQuery);
// Current hour in Eastern (fallback computes it directly)
const currentHour = hourlyResult.length > 0
? parseInt(hourlyResult[0].currentHour)
: DateTime.now().setZone(TIMEZONE).hour;
// Build map of hour -> count
const hourCounts = {};
hourlyResult.forEach(row => {
hourCounts[parseInt(row.hour)] = parseInt(row.count);
});
// Build array in chronological order starting from (currentHour + 1) which is 24 hours ago
hourlyOrders = [];
for (let i = 0; i < 24; i++) {
const hour = (currentHour + 1 + i) % 24; // Start from 24hrs ago, end at current hour
hourlyOrders.push({
hour,
count: hourCounts[hour] || 0
});
}
}
// Brands query - products.company links to product_categories.cat_id for brand name
// Only include products that have a brand assigned (INNER JOIN)
const brandsQuery = `
@@ -223,8 +200,6 @@ router.get('/stats', async (req, res) => {
LIMIT 100
`;
const [brandsResult] = await connection.execute(brandsQuery, params);
// Categories query - uses product_category_index to get category assignments
// Only include categories with valid types (no NULL/uncategorized)
const categoriesQuery = `
@@ -249,8 +224,6 @@ router.get('/stats', async (req, res) => {
LIMIT 100
`;
const [categoriesResult] = await connection.execute(categoriesQuery, params);
// Shipping locations query - uses date_shipped to match shippedCount
const shippingQuery = `
SELECT
@@ -265,11 +238,6 @@ router.get('/stats', async (req, res) => {
GROUP BY ship_country, ship_state, ship_method_selected
`;
const [shippingResult] = await connection.execute(shippingQuery, params);
// Process shipping data
const shippingStats = processShippingData(shippingResult, shippedCount);
// Order value range query (optionally excludes Cherry Box orders)
// Excludes $0 orders from min calculation
const orderRangeQuery = `
@@ -280,17 +248,78 @@ router.get('/stats', async (req, res) => {
WHERE order_status > 15 AND ${getCherryBoxClause(excludeCB)} AND ${whereClause}
`;
const [orderRangeResult] = await connection.execute(orderRangeQuery, params);
// Run everything in parallel; heavier joins first so they don't tail-end
// the worker queue.
const rows = (sql, p = params) => (conn) => conn.execute(sql, p).then(([result]) => result);
const tasks = [
['brands', rows(brandsQuery)],
['categories', rows(categoriesQuery)],
['mainStats', rows(mainStatsQuery)],
['prevPeriod', (conn) => getPreviousPeriodData(conn, timeRange, startDate, endDate, excludeCB)],
['cancelled', rows(cancelledQuery)],
['refunds', rows(refundsQuery)],
['shipped', rows(shippedQuery)],
['bestDay', rows(bestDayQuery)],
['shipping', rows(shippingQuery)],
['orderRange', rows(orderRangeQuery)],
];
if (isSingleDay) {
tasks.push(['peakHour', rows(peakHourQuery)]);
tasks.push(['hourly', rows(hourlyQuery, [])]);
}
const r = await runStatsTasks(tasks, STATS_QUERY_WIDTH);
const stats = r.mainStats[0];
const cancelledStats = r.cancelled[0] || { cancelledCount: 0, cancelledTotal: 0 };
const refundStats = r.refunds;
const shippedCount = parseInt(r.shipped[0]?.shippedCount || 0);
const bestDayResult = r.bestDay;
const brandsResult = r.brands;
const categoriesResult = r.categories;
const orderRangeResult = r.orderRange;
const prevPeriodData = r.prevPeriod;
const shippingStats = processShippingData(r.shipping, shippedCount);
let peakHour = null;
if (isSingleDay && r.peakHour.length > 0) {
const hour = parseInt(r.peakHour[0].hour);
peakHour = {
hour,
count: parseInt(r.peakHour[0].count),
displayHour: DateTime.fromObject({ hour }).toFormat('h a')
};
}
// Returns data ordered chronologically: [24hrs ago, ..., 1hr ago, current hour]
let hourlyOrders = null;
if (isSingleDay) {
// Current hour in Eastern (fallback computes it directly)
const currentHour = r.hourly.length > 0
? parseInt(r.hourly[0].currentHour)
: DateTime.now().setZone(TIMEZONE).hour;
const hourCounts = {};
r.hourly.forEach(row => {
hourCounts[parseInt(row.hour)] = parseInt(row.count);
});
hourlyOrders = [];
for (let i = 0; i < 24; i++) {
const hour = (currentHour + 1 + i) % 24; // Start from 24hrs ago, end at current hour
hourlyOrders.push({
hour,
count: hourCounts[hour] || 0
});
}
}
// Calculate period progress for incomplete periods
let periodProgress = 100;
if (['today', 'thisWeek', 'thisMonth'].includes(timeRange)) {
periodProgress = calculatePeriodProgress(timeRange);
}
// Previous period comparison data
const prevPeriodData = await getPreviousPeriodData(connection, timeRange, startDate, endDate, excludeCB);
const response = {
timeRange: dateRange,
stats: {
@@ -384,27 +413,57 @@ router.get('/stats', async (req, res) => {
};
return response;
} finally {
// Always release the connection regardless of whether the outer Promise.race
// used our result. If the timeout wins, this IIFE keeps running in the
// background until MySQL responds, then this finally releases. Without it,
// every timed-out request permanently leaks one pool slot.
release();
}
};
}
const response = await Promise.race([mainOperation(), timeoutPromise]);
// Main stats endpoint - replaces /api/klaviyo/events/stats
router.get('/stats', async (req, res) => {
const startTime = Date.now();
const { timeRange, startDate, endDate, excludeCherryBox } = req.query;
const excludeCB = parseBoolParam(excludeCherryBox);
const cacheKey = `${timeRange || ''}|${startDate || ''}|${endDate || ''}|${excludeCB}`;
console.log(`[STATS] Request completed in ${Date.now() - startTime}ms`);
res.json(response);
// Single-flight refresh: concurrent misses for the same key share one
// computation instead of racing multi-connection batches against the pool.
const refresh = () => {
let inflight = statsInflight.get(cacheKey);
if (inflight) return inflight;
inflight = withTimeout(computeStats(timeRange, startDate, endDate, excludeCB), 15000, '[STATS]')
.then((data) => {
statsCache.set(cacheKey, { data, timestamp: Date.now() });
// Arbitrary custom date ranges would otherwise grow the map forever
if (statsCache.size > 100) {
const oldest = [...statsCache.entries()]
.reduce((a, b) => (a[1].timestamp <= b[1].timestamp ? a : b));
statsCache.delete(oldest[0]);
}
return data;
})
.finally(() => statsInflight.delete(cacheKey));
statsInflight.set(cacheKey, inflight);
return inflight;
};
try {
const cached = statsCache.get(cacheKey);
const age = cached ? Date.now() - cached.timestamp : Infinity;
if (cached && age >= STATS_CACHE_TTL && age < STATS_CACHE_MAX_STALE) {
refresh().catch((err) => console.error('[STATS] Background refresh failed:', err.message));
}
if (cached && age < STATS_CACHE_MAX_STALE) {
console.log(`[STATS] Cache hit (${Math.round(age / 1000)}s old) for ${cacheKey}`);
return res.json(cached.data);
}
const data = await refresh();
console.log(`[STATS] Computed ${cacheKey} in ${Date.now() - startTime}ms`);
res.json(data);
} catch (error) {
if (error.message.includes('timeout')) {
console.log(`[STATS] Request timed out in ${Date.now() - startTime}ms`);
} else {
console.error('Error in /stats:', error);
}
console.log(`[STATS] Request failed in ${Date.now() - startTime}ms`);
res.status(500).json({ error: error.message });
}
});
@@ -812,23 +871,15 @@ router.get('/products', async (req, res) => {
}
});
// Projection endpoint - replaces /api/klaviyo/events/projection
router.get('/projection', async (req, res) => {
// Same stale-while-revalidate treatment as /stats: several tunnel round trips
// per computation (~800ms), polled every 60s by every dashboard.
const projectionCache = new Map(); // cacheKey -> { data, timestamp }
const projectionInflight = new Map(); // cacheKey -> Promise<data>
async function computeProjection(timeRange, startDate, endDate, excludeCB) {
const startTime = Date.now();
let release;
const { connection, release } = await getDbConnection();
try {
const { timeRange, startDate, endDate, excludeCherryBox } = req.query;
const excludeCB = parseBoolParam(excludeCherryBox);
console.log(`[PROJECTION] Starting request for timeRange: ${timeRange}`);
// Only provide projections for incomplete periods
if (!['today', 'thisWeek', 'thisMonth'].includes(timeRange)) {
return res.json({ projectedRevenue: 0, confidence: 0, method: 'none' });
}
const { connection, release: releaseConn } = await getDbConnection();
release = releaseConn;
console.log(`[PROJECTION] DB connection obtained in ${Date.now() - startTime}ms`);
const now = DateTime.now().setZone(TIMEZONE);
@@ -877,15 +928,52 @@ router.get('/projection', async (req, res) => {
projection.currentRevenue = parseFloat(current.currentRevenue || 0);
projection.currentOrders = parseInt(current.currentOrders || 0);
console.log(`[PROJECTION] Request completed in ${Date.now() - startTime}ms - method: ${projection.method}, projected: $${projection.projectedRevenue?.toFixed(2)}`);
res.json(projection);
} catch (error) {
console.error(`[PROJECTION] Error after ${Date.now() - startTime}ms:`, error);
res.status(500).json({ error: error.message });
console.log(`[PROJECTION] Computed in ${Date.now() - startTime}ms - method: ${projection.method}, projected: $${projection.projectedRevenue?.toFixed(2)}`);
return projection;
} finally {
// Release connection back to pool
if (release) release();
release();
}
}
// Projection endpoint - replaces /api/klaviyo/events/projection
router.get('/projection', async (req, res) => {
const { timeRange, startDate, endDate, excludeCherryBox } = req.query;
const excludeCB = parseBoolParam(excludeCherryBox);
// Only provide projections for incomplete periods
if (!['today', 'thisWeek', 'thisMonth'].includes(timeRange)) {
return res.json({ projectedRevenue: 0, confidence: 0, method: 'none' });
}
const cacheKey = `${timeRange}|${excludeCB}`;
const refresh = () => {
let inflight = projectionInflight.get(cacheKey);
if (inflight) return inflight;
inflight = computeProjection(timeRange, startDate, endDate, excludeCB)
.then((data) => {
projectionCache.set(cacheKey, { data, timestamp: Date.now() });
return data;
})
.finally(() => projectionInflight.delete(cacheKey));
projectionInflight.set(cacheKey, inflight);
return inflight;
};
try {
const cached = projectionCache.get(cacheKey);
const age = cached ? Date.now() - cached.timestamp : Infinity;
if (cached && age >= STATS_CACHE_TTL && age < STATS_CACHE_MAX_STALE) {
refresh().catch((err) => console.error('[PROJECTION] Background refresh failed:', err.message));
}
if (cached && age < STATS_CACHE_MAX_STALE) {
return res.json(cached.data);
}
res.json(await refresh());
} catch (error) {
console.error('[PROJECTION] Error:', error);
res.status(500).json({ error: error.message });
}
});