Fix identified issues with server consolidation

This commit is contained in:
2026-05-24 16:17:27 -04:00
parent e83d975bd6
commit cfe3b29c98
19 changed files with 2390 additions and 193 deletions
+5 -1
View File
@@ -2,6 +2,7 @@
"name": "chat-server",
"version": "1.0.0",
"description": "Chat archive server for Rocket.Chat data",
"type": "module",
"main": "server.js",
"scripts": {
"start": "node server.js",
@@ -12,7 +13,10 @@
"cors": "^2.8.5",
"pg": "^8.11.0",
"dotenv": "^16.0.3",
"morgan": "^1.10.0"
"morgan": "^1.10.0",
"jsonwebtoken": "^9.0.2",
"pino": "^9.5.0",
"pino-http": "^10.3.0"
},
"devDependencies": {
"nodemon": "^2.0.22"
+10 -3
View File
@@ -1,5 +1,12 @@
const express = require('express');
const path = require('path');
import express from 'express';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
// ESM polyfill — Phase 9 §9.1. Handlers below use __dirname to resolve the
// db-convert/db/files/{uploads,avatars} static asset paths.
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const router = express.Router();
// Serve uploaded files with proper mapping from database paths to actual file locations
@@ -646,4 +653,4 @@ router.get('/users/:userId/search', async (req, res) => {
}
});
module.exports = router;
export default router;
+96 -47
View File
@@ -1,23 +1,62 @@
require('dotenv').config({ path: '../.env' });
const express = require('express');
const cors = require('cors');
const { Pool } = require('pg');
const morgan = require('morgan');
const chatRoutes = require('./routes');
// chat-server — Phase 9 §9.1 of CONSOLIDATION_PLAN.md.
//
// ESM conversion + in-process authenticate() defense-in-depth. Previously this
// service relied on the Caddy `forward_auth` gate alone — `localhost:3014`
// was reachable unauthenticated. Now:
// 1. Bound to 127.0.0.1 (was 0.0.0.0) so direct-port access is impossible.
// 2. authenticate() runs against an in-process `inventory_db` pool before
// any route handler sees the request.
//
// Two pools intentionally:
// - `inventoryPool`: used by authenticate() for users/permissions lookups
// against the main inventory_db (matches DB_* env vars).
// - `pool` (set as global.pool for routes.js): the existing
// `rocketchat_converted` pool driven by CHAT_DB_* env vars. routes.js
// reads global.pool throughout — no handler-body changes needed.
import { config as loadEnv } from 'dotenv';
import express from 'express';
import cors from 'cors';
import morgan from 'morgan';
import pg from 'pg';
import path from 'node:path';
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { authenticate } from '../shared/auth/middleware.js';
import { corsOptions } from '../shared/cors/policy.js';
import { errorHandler } from '../shared/errors/handler.js';
import { requestLog } from '../shared/logging/request-log.js';
import chatRoutes from './routes.js';
const { Pool } = pg;
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Env layering matches dashboard-server (Deviation #18): shared .env wins on
// collisions for security-critical vars, local .env supplies CHAT_DB_*.
const sharedEnvPath = '/var/www/inventory/.env';
const localEnvPath = path.resolve(__dirname, '.env');
if (fs.existsSync(sharedEnvPath)) loadEnv({ path: sharedEnvPath });
if (fs.existsSync(localEnvPath)) loadEnv({ path: localEnvPath });
if (!process.env.JWT_SECRET) {
console.error('JWT_SECRET is not set; refusing to start (per Phase 6.4)');
process.exit(1);
}
const app = express();
const port = Number(process.env.CHAT_PORT) || 3014;
// Log startup configuration
console.log('Starting chat server with config:', {
host: process.env.CHAT_DB_HOST,
user: process.env.CHAT_DB_USER,
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
port: process.env.CHAT_DB_PORT,
chat_port: process.env.CHAT_PORT || 3014
chat_port: port,
});
const app = express();
const port = process.env.CHAT_PORT || 3014;
// Database configuration for rocketchat_converted database
// Rocket.Chat archive pool — routes.js reads it via global.pool.
const pool = new Pool({
host: process.env.CHAT_DB_HOST,
user: process.env.CHAT_DB_USER,
@@ -25,59 +64,69 @@ const pool = new Pool({
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
port: process.env.CHAT_DB_PORT,
});
// Make pool available globally
global.pool = pool;
// Middleware
// inventory_db pool — used by authenticate() for user/permission lookups.
const inventoryPool = new Pool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: Number(process.env.DB_PORT) || 5432,
ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
});
app.use(requestLog());
app.use(express.json());
app.use(morgan('combined'));
app.use(cors({
origin: ['http://localhost:5175', 'http://localhost:5174', 'https://inventory.kent.pw', 'https://acot.site', 'https://tools.acherryontop.com'],
credentials: true
}));
app.use(cors(corsOptions));
// Test database connection endpoint
app.get('/test-db', async (req, res) => {
// /health stays unauthenticated for out-of-band probes — mounted BEFORE
// authenticate() so monitoring tools on the host can poll without a JWT.
// Only reachable via localhost:3014 directly (Caddy routes /health to
// inventory-server:3010, not here).
app.get('/health', (req, res) => res.json({ status: 'healthy' }));
// Phase 9 §9.1 — per-server auth re-verification. Every chat route must pass
// authenticate() in addition to the Caddy forward_auth gate.
app.use(authenticate({ pool: inventoryPool, secret: process.env.JWT_SECRET }));
app.get('/test-db', async (req, res, next) => {
try {
const result = await pool.query('SELECT COUNT(*) as user_count FROM users WHERE active = true');
const messageResult = await pool.query('SELECT COUNT(*) as message_count FROM message');
const roomResult = await pool.query('SELECT COUNT(*) as room_count FROM room');
res.json({
status: 'success',
database: 'rocketchat_converted',
stats: {
active_users: parseInt(result.rows[0].user_count),
total_messages: parseInt(messageResult.rows[0].message_count),
total_rooms: parseInt(roomResult.rows[0].room_count)
}
});
} catch (error) {
console.error('Database test error:', error);
res.status(500).json({
status: 'error',
error: 'Database connection failed',
details: error.message
active_users: parseInt(result.rows[0].user_count, 10),
total_messages: parseInt(messageResult.rows[0].message_count, 10),
total_rooms: parseInt(roomResult.rows[0].room_count, 10),
},
});
} catch (err) {
next(err);
}
});
// Mount all routes from routes.js
app.use('/', chatRoutes);
// Health check endpoint
app.get('/health', (req, res) => {
res.json({ status: 'healthy' });
app.use(errorHandler);
// Phase 9 §9.1 — bind to 127.0.0.1. Caddy reverse_proxy targets localhost:3014
// already; this closes the gap where unauthenticated direct-port access from
// any host on the network was possible.
const server = app.listen(port, '127.0.0.1', () => {
console.log(`Chat server running on 127.0.0.1:${port}`);
});
// Error handling middleware
app.use((err, req, res, next) => {
console.error(err.stack);
res.status(500).json({ error: 'Something broke!' });
});
// Start server
app.listen(port, () => {
console.log(`Chat server running on port ${port}`);
});
const shutdown = async (signal) => {
console.log(`chat-server shutting down (${signal})`);
server.close();
try { await pool.end(); } catch { /* ignore */ }
try { await inventoryPool.end(); } catch { /* ignore */ }
process.exit(0);
};
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
+200
View File
@@ -0,0 +1,200 @@
# Caddyfile — Phase 9 §9.2 proposed form.
#
# Three changes vs. /etc/caddy/Caddyfile (2026-05-24):
# 1. @static matcher now explicitly excludes /uploads/* — without this, an
# uploaded *.jpg matched @static before @gated and slipped past the
# forward_auth gate, hitting the SPA build root and returning a public 404.
# 2. The security_headers snippet no longer sets Access-Control-Allow-* —
# the upstreams' shared/cors/policy.js is the single source of truth for
# CORS responses (Phase 6.6).
# 3. New @cors_preflight handler punts OPTIONS preflights past forward_auth
# so the upstream's CORS middleware can answer them (preflights have no
# Authorization header, so they 401'd at the gate previously).
#
# Apply via the staged-cutover convention in Deviation #8:
# scp this file to netcup:/home/matt/Caddyfile.new
# curl --silent -X POST -H "Content-Type: text/caddyfile" \
# --data-binary @/home/matt/Caddyfile.new http://localhost:2020/load
# # ...smoke-test, then persist:
# sudo cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.bak.YYYY-MM-DD
# sudo cp /home/matt/Caddyfile.new /etc/caddy/Caddyfile
{
admin :2020
}
(security_headers) {
header {
X-Content-Type-Options "nosniff"
X-Frame-Options "SAMEORIGIN"
X-XSS-Protection "1; mode=block"
Strict-Transport-Security "max-age=31536000; includeSubDomains"
Referrer-Policy "strict-origin-when-cross-origin"
# Phase 9 §9.2: CORS headers removed. Upstreams set ACAO conditionally
# via shared/cors/policy.js; Caddy stamping `*` here was overriding it.
}
}
files.acot.site {
reverse_proxy localhost:8060
}
pbx.acot.site {
@ws path /ws
handle @ws {
reverse_proxy 127.0.0.1:8088
}
handle {
reverse_proxy 127.0.0.1:8080 {
header_up Host {host}
header_down Location http://127.0.0.1:8080 https://pbx.acot.site
header_down Location http://pbx.acot.site:8080 https://pbx.acot.site
}
}
}
turn.acot.site {
respond 404
}
freescout.acot.site {
root * /var/www/freescout/public
encode gzip
php_fastcgi unix//run/php/php8.3-fpm.sock
file_server
# Deny access to dotfiles
@dotfiles path */.*
respond @dotfiles 403
}
phone.acot.site {
reverse_proxy 127.0.0.1:3020
encode gzip
}
crafty.acot.site {
reverse_proxy localhost:8443 {
header_up X-Forwarded-Proto https
header_up X-Forwarded-Port 443
header_up Host {host}
transport http {
tls_insecure_skip_verify
}
}
}
cronicle.acot.site {
reverse_proxy localhost:3100 {
header_up X-Forwarded-Proto https
}
}
inventory.acot.site, acot.site {
redir https://tools.acherryontop.com{uri} permanent
}
tools.acherryontop.com {
import security_headers
# Public: login endpoint
handle /auth-inv/* {
uri strip_prefix /auth-inv
reverse_proxy localhost:3011
}
# Phase 9 §9.2 — CORS preflight bypass.
# Browsers send OPTIONS preflights without Authorization, so forward_auth
# 401s them. Route preflights straight to the upstream which runs
# shared/cors/policy.js and answers correctly. Must come before @static
# and @gated so OPTIONS to *.jpg paths under /uploads/* also work if any
# frontend ever XHRs an upload URL.
@cors_preflight {
method OPTIONS
header Access-Control-Request-Method *
}
handle @cors_preflight {
handle /api/klaviyo/* {
reverse_proxy localhost:3015
}
handle /api/meta/* {
reverse_proxy localhost:3015
}
handle /api/dashboard-analytics/* {
reverse_proxy localhost:3015
}
handle /api/typeform/* {
reverse_proxy localhost:3015
}
handle /api/acot/* {
reverse_proxy localhost:3012
}
handle /chat-api/* {
uri strip_prefix /chat-api
reverse_proxy localhost:3014
}
handle /api/* {
reverse_proxy localhost:3010
}
}
# Public: static frontend assets (long-cache).
# Phase 9 §9.2: `not path /uploads/*` ensures uploaded images never get
# served from the SPA build root — they must go through @gated below.
@static {
path *.js *.css *.png *.jpg *.jpeg *.gif *.ico *.svg *.woff *.woff2
not path /uploads/*
}
handle @static {
header Cache-Control "public, max-age=2592000"
root * /var/www/inventory/frontend/build
file_server
}
# ----- Authenticated zone -----
# Phase 6.1: forward_auth subrequest to auth-server:/verify. 2xx → proceeds.
# 401/403 → Caddy returns auth-server response to client; backend never sees it.
@gated path /api/* /chat-api/* /uploads/*
handle @gated {
forward_auth localhost:3011 {
uri /verify
copy_headers Authorization
}
# Phase 6.7: /uploads/* now behind the gate (was a public file_server before).
# Phase 9 §9.2 closes the static-matcher bypass that made this ineffective.
handle /uploads/* {
root * /var/www/inventory
file_server
}
# Phase 4: merged dashboard-server (klaviyo + meta + google + typeform).
handle /api/klaviyo/* {
reverse_proxy localhost:3015
}
handle /api/meta/* {
reverse_proxy localhost:3015
}
handle /api/dashboard-analytics/* {
reverse_proxy localhost:3015
}
handle /api/typeform/* {
reverse_proxy localhost:3015
}
# ACOT
handle /api/acot/* {
reverse_proxy localhost:3012
}
# Chat (Phase 9 §9.1 — chat-server now has its own authenticate() too)
handle /chat-api/* {
uri strip_prefix /chat-api
reverse_proxy localhost:3014
}
# Catch-all: inventory-server
handle /api/* {
reverse_proxy localhost:3010
}
}
# Out-of-band probes (unauthenticated)
handle /health {
reverse_proxy localhost:3010
}
# SPA fallback (public assets)
handle {
root * /var/www/inventory/frontend/build
try_files {path} /index.html
file_server
encode gzip
}
handle_errors {
respond "{err.status_code} {err.status_text}"
}
}
+1439 -1
View File
File diff suppressed because it is too large Load Diff
+4 -2
View File
@@ -13,7 +13,8 @@
"prod:logs": "pm2 logs inventory-server",
"prod:status": "pm2 status inventory-server",
"setup": "mkdir -p logs uploads",
"test": "echo \"Error: no test specified\" && exit 1"
"test": "vitest run",
"test:watch": "vitest"
},
"keywords": [],
"author": "",
@@ -43,6 +44,7 @@
"uuid": "^9.0.1"
},
"devDependencies": {
"nodemon": "^3.0.2"
"nodemon": "^3.0.2",
"vitest": "^2.1.9"
}
}
@@ -0,0 +1,163 @@
// Phase 9 §9.4 — vitest scaffold + auth-boundary tests.
//
// Covers shared/auth/middleware.js. Mocks the Postgres pool with a thin
// in-memory fake — no real DB required.
import { describe, it, expect, vi, beforeEach } from 'vitest';
import jwt from 'jsonwebtoken';
import { authenticate, requirePermission } from './middleware.js';
const SECRET = 'test-secret-please-do-not-use-in-prod';
function makeFakePool(users, permissions = {}) {
const calls = { count: 0 };
return {
calls,
query: vi.fn(async (sql, params) => {
calls.count += 1;
if (sql.includes('FROM users WHERE id')) {
const user = users[params[0]];
return { rows: user ? [user] : [] };
}
if (sql.includes('FROM permissions')) {
return { rows: (permissions[params[0]] || []).map((code) => ({ code })) };
}
return { rows: [] };
}),
};
}
function makeReq(authHeader) {
return { headers: authHeader ? { authorization: authHeader } : {} };
}
function makeRes() {
const res = {};
res.status = vi.fn(() => res);
res.json = vi.fn(() => res);
return res;
}
describe('authenticate middleware', () => {
let activeUser;
let inactiveUser;
let validToken;
beforeEach(() => {
activeUser = { id: 1, username: 'alice', email: 'a@x', is_admin: false, is_active: true };
inactiveUser = { id: 2, username: 'bob', email: 'b@x', is_admin: false, is_active: false };
validToken = jwt.sign({ userId: 1 }, SECRET, { expiresIn: '1h' });
});
it('returns 401 when no Authorization header is present', async () => {
const pool = makeFakePool({ 1: activeUser });
const mw = authenticate({ pool, secret: SECRET });
const res = makeRes();
const next = vi.fn();
await mw(makeReq(), res, next);
expect(res.status).toHaveBeenCalledWith(401);
expect(next).not.toHaveBeenCalled();
});
it('returns 401 when Authorization is not Bearer', async () => {
const pool = makeFakePool({ 1: activeUser });
const mw = authenticate({ pool, secret: SECRET });
const res = makeRes();
const next = vi.fn();
await mw(makeReq('Basic abc'), res, next);
expect(res.status).toHaveBeenCalledWith(401);
expect(next).not.toHaveBeenCalled();
});
it('returns 401 when token is malformed', async () => {
const pool = makeFakePool({ 1: activeUser });
const mw = authenticate({ pool, secret: SECRET });
const res = makeRes();
const next = vi.fn();
await mw(makeReq('Bearer not-a-jwt'), res, next);
expect(res.status).toHaveBeenCalledWith(401);
expect(next).not.toHaveBeenCalled();
});
it('returns 403 when the user is inactive', async () => {
const inactiveToken = jwt.sign({ userId: 2 }, SECRET, { expiresIn: '1h' });
const pool = makeFakePool({ 2: inactiveUser });
const mw = authenticate({ pool, secret: SECRET });
const res = makeRes();
const next = vi.fn();
await mw(makeReq(`Bearer ${inactiveToken}`), res, next);
expect(res.status).toHaveBeenCalledWith(403);
expect(next).not.toHaveBeenCalled();
});
it('calls next() and populates req.user for a valid token + active user', async () => {
const pool = makeFakePool({ 1: activeUser }, { 1: ['products:read'] });
const mw = authenticate({ pool, secret: SECRET });
const req = makeReq(`Bearer ${validToken}`);
const res = makeRes();
const next = vi.fn();
await mw(req, res, next);
expect(next).toHaveBeenCalledOnce();
expect(req.user).toBeDefined();
expect(req.user.id).toBe(1);
expect(req.user.permissions).toEqual(['products:read']);
});
it('caches the user lookup — same token within TTL → one DB hit', async () => {
const pool = makeFakePool({ 1: activeUser }, { 1: [] });
const mw = authenticate({ pool, secret: SECRET });
const next = vi.fn();
await mw(makeReq(`Bearer ${validToken}`), makeRes(), next);
await mw(makeReq(`Bearer ${validToken}`), makeRes(), next);
// Two queries on first hit (user + permissions), zero on the second
expect(pool.calls.count).toBe(2);
expect(next).toHaveBeenCalledTimes(2);
});
it('refetches after TTL expiry', async () => {
vi.useFakeTimers();
const pool = makeFakePool({ 1: activeUser }, { 1: [] });
const mw = authenticate({ pool, secret: SECRET });
const next = vi.fn();
await mw(makeReq(`Bearer ${validToken}`), makeRes(), next);
expect(pool.calls.count).toBe(2);
vi.advanceTimersByTime(61_000);
await mw(makeReq(`Bearer ${validToken}`), makeRes(), next);
expect(pool.calls.count).toBe(4);
vi.useRealTimers();
});
});
describe('requirePermission middleware', () => {
it('returns 401 when req.user is missing', () => {
const mw = requirePermission('products:write');
const res = makeRes();
const next = vi.fn();
mw({}, res, next);
expect(res.status).toHaveBeenCalledWith(401);
expect(next).not.toHaveBeenCalled();
});
it('calls next() for admin users regardless of code', () => {
const mw = requirePermission('products:write');
const next = vi.fn();
mw({ user: { is_admin: true, permissions: [] } }, makeRes(), next);
expect(next).toHaveBeenCalledOnce();
});
it('calls next() when user has the required permission', () => {
const mw = requirePermission('products:write');
const next = vi.fn();
mw({ user: { is_admin: false, permissions: ['products:write'] } }, makeRes(), next);
expect(next).toHaveBeenCalledOnce();
});
it('returns 403 when user lacks the required permission', () => {
const mw = requirePermission('products:write');
const res = makeRes();
const next = vi.fn();
mw({ user: { is_admin: false, permissions: ['products:read'] } }, res, next);
expect(res.status).toHaveBeenCalledWith(403);
expect(next).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,75 @@
// Phase 9 §9.4 — vitest scaffold + auth-boundary tests.
//
// Covers the security-critical surface in shared/auth/verify.js. Five cases
// per the original Phase 2 testing-scaffold spec.
import { describe, it, expect, beforeAll } from 'vitest';
import jwt from 'jsonwebtoken';
import { extractBearerToken, verifyToken, TokenError } from './verify.js';
const SECRET = 'test-secret-please-do-not-use-in-prod';
const WRONG_SECRET = 'a-different-secret';
let validToken;
let expiredToken;
let wrongSigToken;
beforeAll(() => {
validToken = jwt.sign({ userId: 42, username: 'alice' }, SECRET, { expiresIn: '1h' });
expiredToken = jwt.sign({ userId: 42, username: 'alice' }, SECRET, { expiresIn: '-1s' });
wrongSigToken = jwt.sign({ userId: 42, username: 'alice' }, WRONG_SECRET, { expiresIn: '1h' });
});
describe('extractBearerToken', () => {
it('returns token from a well-formed Bearer header', () => {
expect(extractBearerToken('Bearer abc.def.ghi')).toBe('abc.def.ghi');
});
it('throws TokenError(missing) when no header is provided', () => {
expect(() => extractBearerToken(undefined)).toThrow(TokenError);
try { extractBearerToken(undefined); } catch (err) { expect(err.code).toBe('missing'); }
});
it('throws TokenError(malformed) when header is not Bearer-prefixed', () => {
expect(() => extractBearerToken('Basic abc')).toThrow(TokenError);
try { extractBearerToken('Basic abc'); } catch (err) { expect(err.code).toBe('malformed'); }
});
it('throws TokenError(malformed) when Bearer header has empty token', () => {
expect(() => extractBearerToken('Bearer ')).toThrow(TokenError);
try { extractBearerToken('Bearer '); } catch (err) { expect(err.code).toBe('malformed'); }
});
it('throws TokenError(missing) when header is not a string', () => {
expect(() => extractBearerToken(null)).toThrow(TokenError);
expect(() => extractBearerToken(42)).toThrow(TokenError);
});
});
describe('verifyToken', () => {
it('returns decoded payload for a valid token', () => {
const decoded = verifyToken(validToken, SECRET);
expect(decoded.userId).toBe(42);
expect(decoded.username).toBe('alice');
});
it('throws TokenError(expired) for an expired token', () => {
expect(() => verifyToken(expiredToken, SECRET)).toThrow(TokenError);
try { verifyToken(expiredToken, SECRET); } catch (err) { expect(err.code).toBe('expired'); }
});
it('throws TokenError(invalid) for a wrong-signature token', () => {
expect(() => verifyToken(wrongSigToken, SECRET)).toThrow(TokenError);
try { verifyToken(wrongSigToken, SECRET); } catch (err) { expect(err.code).toBe('invalid'); }
});
it('throws TokenError(invalid) for malformed JWT', () => {
expect(() => verifyToken('not-a-jwt', SECRET)).toThrow(TokenError);
try { verifyToken('not-a-jwt', SECRET); } catch (err) { expect(err.code).toBe('invalid'); }
});
it('throws TokenError(misconfigured) when secret is missing', () => {
expect(() => verifyToken(validToken, undefined)).toThrow(TokenError);
try { verifyToken(validToken, undefined); } catch (err) { expect(err.code).toBe('misconfigured'); }
});
});
+124 -114
View File
@@ -32,13 +32,81 @@ router.get('/brands', async (req, res) => {
});
// Get all products with pagination, filtering, and sorting
// Whitelist of allowed sort keys → SQL column expressions. Used to gate
// `?sort=` against direct interpolation into ORDER BY (the previous code
// dropped req.query.sort straight into the query string — SQL injection sink).
// Keys are the camelCase identifiers the frontend ProductMetricColumnKey union
// emits. Anything not in the map falls back to `p.title`.
const SORT_COLUMN_MAP = {
pid: 'p.pid',
title: 'p.title',
sku: 'p.sku',
barcode: 'p.barcode',
brand: 'p.brand',
line: 'p.line',
subline: 'p.subline',
artist: 'p.artist',
vendor: 'p.vendor',
vendorReference: 'p.vendor_reference',
notionsReference: 'p.notions_reference',
harmonizedTariffCode: 'p.harmonized_tariff_code',
countryOfOrigin: 'p.country_of_origin',
location: 'p.location',
moq: 'p.moq',
weight: 'p.weight',
rating: 'p.rating',
reviews: 'p.reviews',
baskets: 'p.baskets',
notifies: 'p.notifies',
preorderCount: 'p.preorder_count',
notionsInvCount: 'p.notions_inv_count',
dateCreated: 'p.created_at',
dateLastSold: 'p.date_last_sold',
stock: 'p.stock_quantity',
stockQuantity: 'p.stock_quantity',
price: 'p.price',
costPrice: 'p.cost_price',
totalSold: 'p.total_sold',
// product_metrics columns (current schema names; camelCase aliases the
// frontend uses are mapped to the canonical SQL column)
dailySalesAvg: 'pm.avg_sales_per_day_30d',
weeklySalesAvg: 'pm.sales_7d',
monthlySalesAvg: 'pm.avg_sales_per_month_30d',
margin: 'pm.margin_30d',
gmroi: 'pm.gmroi_30d',
gmroi30d: 'pm.gmroi_30d',
inventoryValue: 'pm.current_stock_cost',
costOfGoodsSold: 'pm.cogs_30d',
grossProfit: 'pm.profit_30d',
turnoverRate: 'pm.stockturn_30d',
stockturn30d: 'pm.stockturn_30d',
leadTime: 'pm.config_lead_time',
currentLeadTime: 'pm.config_lead_time',
targetLeadTime: 'pm.config_lead_time',
stockCoverage: 'pm.stock_cover_in_days',
daysOfStock: 'pm.stock_cover_in_days',
reorderPoint: 'pm.replenishment_units',
safetyStock: 'pm.config_safety_stock',
abcClass: 'pm.abc_class',
status: 'pm.status',
ageDays: 'pm.age_days',
sales7d: 'pm.sales_7d',
sales30d: 'pm.sales_30d',
sales365d: 'pm.sales_365d',
revenue7d: 'pm.revenue_7d',
revenue30d: 'pm.revenue_30d',
revenue365d: 'pm.revenue_365d',
sellThrough30d: 'pm.sell_through_30d',
salesGrowth30dVsPrev: 'pm.sales_growth_30d_vs_prev',
};
router.get('/', async (req, res) => {
const pool = req.app.locals.pool;
try {
const page = parseInt(req.query.page) || 1;
const limit = parseInt(req.query.limit) || 50;
const offset = (page - 1) * limit;
const sortColumn = req.query.sort || 'title';
const sortColumn = SORT_COLUMN_MAP[req.query.sort] || 'p.title';
const sortDirection = req.query.order === 'desc' ? 'DESC' : 'ASC';
const conditions = ['p.visible = true'];
@@ -120,30 +188,28 @@ router.get('/', async (req, res) => {
paramCounter++;
}
// Handle numeric filters with operators
// Handle numeric filters with operators. Mapped to current product_metrics
// column names; frontend keys (camelCase) preserved for compatibility.
const numericFields = {
stock: 'p.stock_quantity',
price: 'p.price',
costPrice: 'p.cost_price',
dailySalesAvg: 'pm.daily_sales_avg',
weeklySalesAvg: 'pm.weekly_sales_avg',
monthlySalesAvg: 'pm.monthly_sales_avg',
avgQuantityPerOrder: 'pm.avg_quantity_per_order',
numberOfOrders: 'pm.number_of_orders',
margin: 'pm.avg_margin_percent',
gmroi: 'pm.gmroi',
inventoryValue: 'pm.inventory_value',
costOfGoodsSold: 'pm.cost_of_goods_sold',
grossProfit: 'pm.gross_profit',
turnoverRate: 'pm.turnover_rate',
leadTime: 'pm.current_lead_time',
currentLeadTime: 'pm.current_lead_time',
targetLeadTime: 'pm.target_lead_time',
stockCoverage: 'pm.days_of_inventory',
daysOfStock: 'pm.days_of_inventory',
weeksOfStock: 'pm.weeks_of_inventory',
reorderPoint: 'pm.reorder_point',
safetyStock: 'pm.safety_stock',
dailySalesAvg: 'pm.avg_sales_per_day_30d',
weeklySalesAvg: 'pm.sales_7d',
monthlySalesAvg: 'pm.avg_sales_per_month_30d',
margin: 'pm.margin_30d',
gmroi: 'pm.gmroi_30d',
inventoryValue: 'pm.current_stock_cost',
costOfGoodsSold: 'pm.cogs_30d',
grossProfit: 'pm.profit_30d',
turnoverRate: 'pm.stockturn_30d',
leadTime: 'pm.config_lead_time',
currentLeadTime: 'pm.config_lead_time',
targetLeadTime: 'pm.config_lead_time',
stockCoverage: 'pm.stock_cover_in_days',
daysOfStock: 'pm.stock_cover_in_days',
reorderPoint: 'pm.replenishment_units',
safetyStock: 'pm.config_safety_stock',
// Add new numeric fields
preorderCount: 'p.preorder_count',
notionsInvCount: 'p.notions_inv_count',
@@ -267,88 +333,33 @@ router.get('/', async (req, res) => {
'SELECT DISTINCT COALESCE(brand, \'Unbranded\') as brand FROM products WHERE visible = true ORDER BY brand'
);
// Main query with all fields
// Main query with all fields. Aliases new product_metrics column names back to
// the legacy names the frontend ProductRow type still uses — same pattern as the
// /:id detail route below.
const query = `
WITH RECURSIVE
category_path AS (
SELECT
c.cat_id,
c.name,
c.parent_id,
c.name::text as path
FROM categories c
WHERE c.parent_id IS NULL
UNION ALL
SELECT
c.cat_id,
c.name,
c.parent_id,
(cp.path || ' > ' || c.name)::text
FROM categories c
JOIN category_path cp ON c.parent_id = cp.cat_id
),
product_thresholds AS (
SELECT
p.pid,
COALESCE(
(SELECT overstock_days FROM stock_thresholds st
WHERE st.category_id IN (
SELECT pc.cat_id
FROM product_categories pc
WHERE pc.pid = p.pid
)
AND (st.vendor = p.vendor OR st.vendor IS NULL)
ORDER BY st.vendor IS NULL
LIMIT 1),
(SELECT overstock_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND (st.vendor = p.vendor OR st.vendor IS NULL)
ORDER BY st.vendor IS NULL
LIMIT 1),
90
) as target_days
FROM products p
),
product_leaf_categories AS (
SELECT DISTINCT pc.cat_id
FROM product_categories pc
WHERE NOT EXISTS (
SELECT 1
FROM categories child
JOIN product_categories child_pc ON child.cat_id = child_pc.cat_id
WHERE child.parent_id = pc.cat_id
AND child_pc.pid = pc.pid
)
)
SELECT
SELECT
p.*,
COALESCE(p.brand, 'Unbranded') as brand,
string_agg(DISTINCT (c.cat_id || ':' || c.name), ',') as categories,
pm.daily_sales_avg,
pm.weekly_sales_avg,
pm.monthly_sales_avg,
pm.avg_quantity_per_order,
pm.number_of_orders,
pm.first_sale_date,
pm.last_sale_date,
pm.days_of_inventory,
pm.weeks_of_inventory,
pm.reorder_point,
pm.safety_stock,
pm.avg_margin_percent,
CAST(pm.total_revenue AS DECIMAL(15,3)) as total_revenue,
CAST(pm.inventory_value AS DECIMAL(15,3)) as inventory_value,
CAST(pm.cost_of_goods_sold AS DECIMAL(15,3)) as cost_of_goods_sold,
CAST(pm.gross_profit AS DECIMAL(15,3)) as gross_profit,
pm.gmroi,
pm.avg_sales_per_day_30d AS daily_sales_avg,
pm.sales_7d AS weekly_sales_avg,
pm.avg_sales_per_month_30d AS monthly_sales_avg,
pm.date_first_sold AS first_sale_date,
pm.date_last_sold AS last_sale_date,
pm.stock_cover_in_days AS days_of_inventory,
pm.replenishment_units AS reorder_point,
pm.config_safety_stock AS safety_stock,
pm.margin_30d AS avg_margin_percent,
CAST(pm.lifetime_revenue AS DECIMAL(15,3)) as total_revenue,
CAST(pm.current_stock_cost AS DECIMAL(15,3)) as inventory_value,
CAST(pm.cogs_30d AS DECIMAL(15,3)) as cost_of_goods_sold,
CAST(pm.profit_30d AS DECIMAL(15,3)) as gross_profit,
pm.gmroi_30d AS gmroi,
pm.avg_lead_time_days,
pm.last_purchase_date,
pm.last_received_date,
pm.date_last_received AS last_received_date,
pm.abc_class,
pm.stock_status,
pm.turnover_rate,
pm.status AS stock_status,
pm.stockturn_30d AS turnover_rate,
p.date_last_sold
FROM products p
LEFT JOIN product_metrics pm ON p.pid = pm.pid
@@ -389,12 +400,12 @@ router.get('/trending', async (req, res) => {
try {
// First check if we have any data
const { rows } = await pool.query(`
SELECT COUNT(*) as count,
MAX(total_revenue) as max_revenue,
MAX(daily_sales_avg) as max_daily_sales,
SELECT COUNT(*) as count,
MAX(lifetime_revenue) as max_revenue,
MAX(avg_sales_per_day_30d) as max_daily_sales,
COUNT(DISTINCT pid) as products_with_metrics
FROM product_metrics
WHERE total_revenue > 0 OR daily_sales_avg > 0
FROM product_metrics
WHERE lifetime_revenue > 0 OR avg_sales_per_day_30d > 0
`);
console.log('Product metrics stats:', rows[0]);
@@ -403,25 +414,24 @@ router.get('/trending', async (req, res) => {
return res.json([]);
}
// Get trending products
// Get trending products. growth_rate uses sales_growth_30d_vs_prev — a
// pre-computed % delta of the last 30d window vs the prior 30d window.
// (The old formula compared a per-day rate against a 7-day total, which
// mixed units and produced nonsense after the metrics-schema rename.)
const { rows: trendingProducts } = await pool.query(`
SELECT
SELECT
p.pid,
p.sku,
p.title,
COALESCE(pm.daily_sales_avg, 0) as daily_sales_avg,
COALESCE(pm.weekly_sales_avg, 0) as weekly_sales_avg,
CASE
WHEN pm.weekly_sales_avg > 0 AND pm.daily_sales_avg > 0
THEN ((pm.daily_sales_avg - pm.weekly_sales_avg) / pm.weekly_sales_avg) * 100
ELSE 0
END as growth_rate,
COALESCE(pm.total_revenue, 0) as total_revenue
COALESCE(pm.avg_sales_per_day_30d, 0) as daily_sales_avg,
COALESCE(pm.sales_7d, 0) as weekly_sales_avg,
COALESCE(pm.sales_growth_30d_vs_prev, 0) as growth_rate,
COALESCE(pm.lifetime_revenue, 0) as total_revenue
FROM products p
INNER JOIN product_metrics pm ON p.pid = pm.pid
WHERE (pm.total_revenue > 0 OR pm.daily_sales_avg > 0)
WHERE (pm.lifetime_revenue > 0 OR pm.avg_sales_per_day_30d > 0)
AND p.visible = true
ORDER BY growth_rate DESC
ORDER BY growth_rate DESC NULLS LAST
LIMIT 50
`);
+17
View File
@@ -0,0 +1,17 @@
import { defineConfig } from 'vitest/config';
// Exclude macOS AppleDouble sidecar files (`._*.js`) that get created when
// editing through the NFS mount from macOS. See Deviation #15 in
// CONSOLIDATION_PLAN.md — these aren't real tests, but vitest's default file
// glob picks them up and fails the suite when rollup tries to parse them.
export default defineConfig({
test: {
include: ['**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts}'],
exclude: [
'**/node_modules/**',
'**/dist/**',
'**/build/**',
'**/._*',
],
},
});