// LIVE PM2 config for netcup. Deployed copy: /var/www/ecosystem.config.cjs // // This repo file is the source of truth; the deployed copy is what PM2 actually // evaluates. Keep them in sync — edit here, then copy across (see "To apply"). // Note /var/www itself is root-owned, so you cannot CREATE files there without // sudo, but /var/www/ecosystem.config.cjs is matt:matt and edits in place fine. // // All apps run under matt's single PM2 daemon (no sudo, no root daemon). // Log paths follow the per-server `logs/pm2/` convention (each service directory // already has the folder created); `pm2-logrotate` rotates them in place. // // ─── READ THIS BEFORE CHANGING ANYTHING ────────────────────────────────────── // `pm2 save` IS NOT OPTIONAL. systemd's pm2-matt.service runs // `ExecStart=pm2 resurrect`, which rebuilds every app's env from // ~/.pm2/dump.pm2 — NOT from this file and NOT from .env. Only `pm2 save` // writes that dump. A reload you didn't save is a time bomb, not a fix. // // That unit gets bounced without any reboot: on 2026-07-28 06:57 an // unattended-upgrade of libc6 triggered needrestart, which restarted // pm2-matt.service, which resurrected a MAY-23 dump. Two consequences: // * The Phase 6.4 JWT_SECRET unification (applied + verified 2026-07-20 but // never saved) silently reverted. new-auth-server went back to an old // 128-char secret while inventory-server kept the correct .env one, so // login/`/me` returned 200 while every /api/* call 401'd "Invalid token" // and bounced users to the login screen. // * dashboard-server was absent from that old dump entirely, so nothing came // up on :3015 and all five Caddy vendor paths 502'd for ~9 hours. // Both fixed 2026-07-28 and `pm2 save`d. Audit the dump periodically — it drifts // in both directions (it still listed four apps deleted months earlier). // // Also note `--update-env` MERGES; it cannot delete a key. A var inherited from // an old dump survives every reload. Only `pm2 delete` + `start` clears it. // ───────────────────────────────────────────────────────────────────────────── // // Design decisions worth preserving (CONSOLIDATION_PLAN.md §4, §6.4, §6.10): // 1. NO `JWT_SECRET: process.env.JWT_SECRET` override in new-auth-server's env // block. That override shadowed the .env value with whatever shell var was // exported when pm2 was last started. With it gone, .env is the single // source. Do not reintroduce it. // 2. Log paths live in per-service `logs/pm2/...` (matt:matt), not // `/var/log/pm2/...` where matt has no write perms. // 3. Phase 4: four per-vendor apps (klaviyo-server, meta-server, google-server, // typeform-server) collapsed into a single `dashboard-server` on :3015. // Those four are long gone — deleted, and purged from the dump 2026-07-28. // 4. Phase 6.10 (2026-05-24): `ADD_WORD_TOKEN` is NOT inline on lt-wordlist-api; // `node_args: ['--env-file=/opt/lt-wordlist-api/.env']` lets Node ≥20.6 read // it natively at startup. To rotate: edit that .env, then // `pm2 restart lt-wordlist-api --update-env`. // 5. Script paths corrected during apply (both were wrong in the original plan): // lt-wordlist-api → /opt/lt-wordlist-api/index.js (was server.js) // acot-phone-server → /var/www/acot-phone/dist/server.js // (was ./inventory/acot-phone/server.js) // // To apply a change: // cp /var/www/ecosystem.config.cjs ~/backups/ecosystem.config.cjs.bak.$(date +%F) // cp /var/www/inventory/deploy/ecosystem.config.cjs /var/www/ecosystem.config.cjs // node --check /var/www/ecosystem.config.cjs # syntax gate before reload // pm2 reload /var/www/ecosystem.config.cjs --update-env [--only ] // pm2 save # ← REQUIRED, see above // // Verify after reload: // pm2 list // pm2 env new-auth-server | grep -i jwt # JWT_SECRET from .env only // pm2 env lt-wordlist-api | grep ADD_WORD # empty (loaded from /opt/.env) // # confirm the dump actually took, not just the running processes: // python3 -c "import json;print([a['name'] for a in json.load(open('/home/matt/.pm2/dump.pm2'))])" const inventoryEnv = require('dotenv').config({ path: '/var/www/inventory/.env' }).parsed; module.exports = { apps: [ { name: 'new-auth-server', // Phase 8 may rename to 'auth-server' — cosmetic script: './inventory/auth/server.js', cwd: '/var/www', env: { ...inventoryEnv, NODE_ENV: 'production', TZ: 'America/Chicago', // business time (see inventory/docs/TIME.md) AUTH_PORT: 3011, // PHASE 6.4 FIX: no JWT_SECRET override here. .env wins. }, max_memory_restart: '500M', error_file: './inventory/auth/logs/pm2/new-auth-server-error.log', out_file: './inventory/auth/logs/pm2/new-auth-server-out.log', }, { name: 'inventory-server', script: './inventory/src/server.js', cwd: '/var/www', env: { ...inventoryEnv, NODE_ENV: 'production', TZ: 'America/Chicago', // business time (see inventory/docs/TIME.md) PORT: 3010, UPLOADS_DIR: '/var/www/inventory/uploads', }, max_memory_restart: '1G', error_file: './inventory/logs/pm2/inventory-server-error.log', out_file: './inventory/logs/pm2/inventory-server-out.log', }, { name: 'chat-server', script: './inventory/chat/server.js', cwd: '/var/www', env: { ...inventoryEnv, NODE_ENV: 'production', TZ: 'America/Chicago', PORT: 3014 }, max_memory_restart: '500M', error_file: './inventory/chat/logs/pm2/chat-server-error.log', out_file: './inventory/chat/logs/pm2/chat-server-out.log', }, { name: 'acot-server', script: './inventory/dashboard/acot-server/server.js', cwd: '/var/www', env: { ...inventoryEnv, NODE_ENV: 'production', TZ: 'America/Chicago', ACOT_PORT: 3012 }, max_memory_restart: '1G', error_file: './inventory/dashboard/acot-server/logs/pm2/acot-server-error.log', out_file: './inventory/dashboard/acot-server/logs/pm2/acot-server-out.log', }, // Phase 4: merged ESM dashboard-server. Replaced klaviyo-server (3004), // meta-server (3005), google-server (3007) and typeform-server (3008); those // four are deleted and purged from the dump. Cutover is COMPLETE — Caddy has // pointed the vendor paths at :3015 since 2026-07-21. // // Now serves FIVE routers, not four (see dashboard/server.js): // /api/klaviyo /api/meta /api/dashboard-analytics /api/typeform // /api/freescout ← added 2026-07-18 for the CS dashboard // All five are proxied here by Caddy. Memory cap = sum of the four old caps // minus dedup'd Redis client + Pool overhead, rounded up. // // Adding a vendor needs NO change here — it's another router inside this same // process. Vendor credentials deliberately do NOT live in this file: server.js // layers /var/www/inventory/.env first, then dashboard/.env, with dotenv // override:false so shared security vars win and vendor keys stay per-service. // FREESCOUT_DB_* / ACOT_PHONE_DB_* therefore live in dashboard/.env only. // // Fail-soft gotcha: without FREESCOUT_DB_HOST the route is simply not mounted // (404s, no startup error). The only signal is the log line // "FREESCOUT_DB_* not set — /api/freescout not mounted". // // Health check: curl -fsS http://localhost:3015/health { name: 'dashboard-server', script: './inventory/dashboard/server.js', cwd: '/var/www', env: { ...inventoryEnv, NODE_ENV: 'production', TZ: 'America/Chicago', // business time (see inventory/docs/TIME.md) DASHBOARD_PORT: 3015, }, max_memory_restart: '1G', error_file: './inventory/dashboard/logs/pm2/dashboard-server-error.log', out_file: './inventory/dashboard/logs/pm2/dashboard-server-out.log', }, // Script entry is index.js (NOT server.js — earlier proposed value was wrong). // PORT MUST be set explicitly to 3030 (Caddy's `/lt-wordlist/*` block proxies // there) — otherwise pm2 inherits PORT=3010 from the parent shell / inventory // .env, the script's `process.env.PORT || 3030` picks up 3010, lt-wordlist // squats on inventory-server's port, and inventory-server crashes with // EADDRINUSE. Caught during apply 2026-05-24. // // Phase 6.10 (applied 2026-05-24, Deviation #25): ADD_WORD_TOKEN is loaded via // Node's native `--env-file` (supported on Node ≥20.6 — netcup runs v22). The // token lives in /opt/lt-wordlist-api/.env (matt:matt 0600), NEVER in this // file. The script reads `process.env.ADD_WORD_TOKEN` directly; the prior // `'tokenhere'` insecure fallback no longer applies because the env var is // always set from the file. To rotate: edit /opt/lt-wordlist-api/.env, then // `pm2 restart lt-wordlist-api --update-env`. { name: 'lt-wordlist-api', script: '/opt/lt-wordlist-api/index.js', cwd: '/opt/lt-wordlist-api', node_args: ['--env-file=/opt/lt-wordlist-api/.env'], env: { NODE_ENV: 'production', PORT: 3030, }, max_memory_restart: '200M', error_file: '/opt/lt-wordlist-api/logs/pm2/lt-wordlist-api-error.log', out_file: '/opt/lt-wordlist-api/logs/pm2/lt-wordlist-api-out.log', }, // Lives in a separate repo at /var/www/acot-phone/ (matt:matt). The compiled // entrypoint is dist/server.js. Loads its own /var/www/acot-phone/.env at boot // (PORT=3020 there) but dotenv defaults to override:false, so any PORT already // set in the pm2 env (e.g. inherited from inventory's .env=3010) WINS — same // EADDRINUSE-on-3010 footgun as lt-wordlist-api. Set PORT explicitly here. // Caddy proxies phone.acot.site → :3020. { name: 'acot-phone-server', script: '/var/www/acot-phone/dist/server.js', cwd: '/var/www/acot-phone', env: { NODE_ENV: 'production', PORT: 3020, }, max_memory_restart: '300M', error_file: '/var/www/acot-phone/logs/pm2/acot-phone-server-error.log', out_file: '/var/www/acot-phone/logs/pm2/acot-phone-server-out.log', }, ], };