matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 3 + 6

Browse Source
This commit is contained in:
Matt
2026-05-23 19:38:12 -04:00
parent 1ab14ba45f
commit 82e568d455
60 changed files with 1983 additions and 2720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inventory-server/src/middleware/cors.js
+8 -35
View File
@@ -1,41 +1,14 @@
const cors = require('cors');
import cors from 'cors';
import { corsOptions } from '../../shared/cors/policy.js';
// Single CORS middleware for all endpoints
const corsMiddleware = cors({
origin: [
'https://inventory.kent.pw',
'http://localhost:5175',
'https://acot.site',
'https://tools.acherryontop.com',
/^http:\/\/192\.168\.\d+\.\d+(:\d+)?$/,
/^http:\/\/10\.\d+\.\d+\.\d+(:\d+)?$/
],
methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
allowedHeaders: ['Content-Type', 'Authorization'],
exposedHeaders: ['Content-Type'],
credentials: true
});
export const corsMiddleware = cors(corsOptions);
// Error handler for CORS
const corsErrorHandler = (err, req, res, next) => {
if (err.message === 'CORS not allowed') {
console.error('CORS Error:', {
origin: req.get('Origin'),
method: req.method,
path: req.path,
headers: req.headers
});
res.status(403).json({
export function corsErrorHandler(err, req, res, next) {
if (err && err.message === 'CORS not allowed') {
return res.status(403).json({
error: 'CORS not allowed',
origin: req.get('Origin'),
message: 'Origin not in allowed list: https://inventory.kent.pw, https://acot.site, https://tools.acherryontop.com, localhost:5175, 192.168.x.x, or 10.x.x.x'
});
} else {
next(err);
}
};
module.exports = {
corsMiddleware,
corsErrorHandler
};
next(err);
}
inventory-server/src/routes/ai-prompts.js
+10 -2
View File
@@ -1,6 +1,14 @@
const express = require('express');
import express from 'express';
import { requirePermission } from '../../shared/auth/middleware.js';
const router = express.Router();
// Phase 6.2: prompt edits require ai_admin. Reads remain authenticated-only.
router.use((req, res, next) => {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('ai_admin')(req, res, next);
});
// Get all AI prompts
router.get('/', async (req, res) => {
try {
@@ -307,4 +315,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/ai-validation.js
+22 -9
View File
@@ -1,12 +1,25 @@
const express = require("express");
import express from "express";
import OpenAI from "openai";
import { promises as fs } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import dotenv from "dotenv";
import mysql from 'mysql2/promise';
import { Client } from 'ssh2';
import { getDbConnection, closeAllConnections } from '../utils/dbConnection.js'; // Import the optimized connection function
import { requirePermission } from '../../shared/auth/middleware.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const router = express.Router();
const OpenAI = require("openai");
const fs = require("fs").promises;
const path = require("path");
const dotenv = require("dotenv");
const mysql = require('mysql2/promise');
const { Client } = require('ssh2');
const { getDbConnection, closeAllConnections } = require('../utils/dbConnection'); // Import the optimized connection function
// Phase 6.2: AI validation runs (which trigger OpenAI calls + DB writes) require ai_admin.
// Status/health reads stay authenticated-only.
router.use((req, res, next) => {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('ai_admin')(req, res, next);
});
// Ensure environment variables are loaded
dotenv.config({ path: path.join(__dirname, "../../.env") });
@@ -1401,7 +1414,7 @@ router.get("/test-taxonomy", async (req, res) => {
}
});
module.exports = router;
export default router;
function extractResponseText(response) {
if (!response) return "";
inventory-server/src/routes/ai.js
+6 -6
View File
@@ -5,10 +5,11 @@
* Provides embedding generation and similarity-based suggestions.
*/
const express = require('express');
import express from 'express';
import aiService from '../services/ai/index.js';
import { getDbConnection, closeAllConnections } from '../utils/dbConnection.js';
const router = express.Router();
const aiService = require('../services/ai');
const { getDbConnection, closeAllConnections } = require('../utils/dbConnection');
// Track initialization state
let initializationPromise = null;
@@ -440,11 +441,10 @@ router.post('/validate/sanity-check', async (req, res) => {
* Call once from server startup so the taxonomy embeddings are ready before
* the first user request hits a taxonomy dropdown.
*/
function initInBackground() {
export function initInBackground() {
ensureInitialized().catch(err =>
console.error('[AI Routes] Background initialization failed:', err)
);
}
module.exports = router;
module.exports.initInBackground = initInBackground;
export default router;
inventory-server/src/routes/analytics.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Forecasting: summarize sales for products received in a period by brand
@@ -980,4 +980,4 @@ router.get('/seasonal', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/brandsAggregate.js
+3 -3
View File
@@ -1,6 +1,6 @@
const express = require('express');
import express from 'express';
import { parseValue } from '../utils/apiHelpers.js'; // Adjust path if needed
const router = express.Router();
const { parseValue } = require('../utils/apiHelpers'); // Adjust path if needed
// --- Configuration & Helpers ---
const DEFAULT_PAGE_LIMIT = 50;
@@ -281,4 +281,4 @@ router.get('/', async (req, res) => {
// GET /brands-aggregate/:name (Get single brand metric)
// Implement if needed, remember to URL-decode the name parameter
module.exports = router;
export default router;
inventory-server/src/routes/categoriesAggregate.js
+3 -3
View File
@@ -1,6 +1,6 @@
const express = require('express');
import express from 'express';
import { parseValue } from '../utils/apiHelpers.js'; // Adjust path if needed
const router = express.Router();
const { parseValue } = require('../utils/apiHelpers'); // Adjust path if needed
// --- Configuration & Helpers ---
const DEFAULT_PAGE_LIMIT = 50;
@@ -360,4 +360,4 @@ router.get('/', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/config.js
+8 -5
View File
@@ -1,10 +1,13 @@
const express = require('express');
import express from 'express';
import { requireAdmin } from '../../shared/auth/middleware.js';
const router = express.Router();
// Debug middleware
// Phase 6.2: global settings are admin-only on write. Reads pass through to any
// authenticated user (the server-level authenticate() already gates that).
router.use((req, res, next) => {
console.log(`[Config Route] ${req.method} ${req.path}`);
next();
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requireAdmin(req, res, next);
});
// ===== GLOBAL SETTINGS =====
@@ -322,4 +325,4 @@ router.post('/vendors/:vendor/reset', async (req, res) => {
});
// Export the router
module.exports = router;
export default router;
inventory-server/src/routes/dashboard.js
+3 -3
View File
@@ -1,6 +1,6 @@
const express = require('express');
import express from 'express';
import db from '../utils/db.js';
const router = express.Router();
const db = require('../utils/db');
// Helper function to execute queries using the connection pool
async function executeQuery(sql, params = []) {
@@ -1288,4 +1288,4 @@ router.get('/replenish/products', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/data-management.js
+16 -9
View File
@@ -1,13 +1,20 @@
const express = require('express');
const router = express.Router();
const { spawn } = require('child_process');
const path = require('path');
const db = require('../utils/db');
import express from 'express';
import { spawn } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import db from '../utils/db.js';
import { requirePermission } from '../../shared/auth/middleware.js';
// Debug middleware MUST be first
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const router = express.Router();
// Phase 6.2: CSV / full-update / full-reset / metrics-recalc are destructive.
// Writes require data_management; reads (status polls, SSE streams) pass through.
router.use((req, res, next) => {
console.log(`[CSV Route Debug] ${req.method} ${req.path}`);
next();
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('data_management')(req, res, next);
});
// Store active processes and their progress
@@ -437,4 +444,4 @@ router.get('/status/table-counts', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/hts-lookup.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// GET /api/hts-lookup?search=term
@@ -167,4 +167,4 @@ router.get('/', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/import-audit-log.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Create a new audit log entry
@@ -190,4 +190,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/import-sessions.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Get all import sessions for a user (named + unnamed)
@@ -334,4 +334,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/import.js
+37 -19
View File
@@ -1,14 +1,23 @@
const express = require('express');
import express from 'express';
import { Client } from 'ssh2';
import mysql from 'mysql2/promise';
import multer from 'multer';
import path from 'node:path';
import fs from 'node:fs';
import sharp from 'sharp';
import axios from 'axios';
import net from 'node:net';
import { requirePermission } from '../../shared/auth/middleware.js';
const router = express.Router();
const { Client } = require('ssh2');
const mysql = require('mysql2/promise');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const fsp = fs.promises;
const sharp = require('sharp');
const axios = require('axios');
const net = require('net');
// Phase 6.2: imports, uploads, generate-upc and deletions all require product_import.
// Reads (list-uploads, status checks) remain authenticated-only.
router.use((req, res, next) => {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('product_import')(req, res, next);
});
// Create uploads directory if it doesn't exist
const uploadsDir = path.join('/var/www/inventory/uploads/products');
@@ -515,21 +524,30 @@ const storage = multer.diskStorage({
const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
// Phase 6.7: exact-match MIME + extension allowlist. Substring-based regex
// matchers (the previous /jpeg|png|.../ approach) accepted MIMEs like
// `application/jpeg-payload` because of partial matches; this rejects them.
const ALLOWED_MIME_TYPES = new Set([
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'image/tiff',
]);
const ALLOWED_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.tif', '.tiff']);
const upload = multer({
storage: storage,
limits: {
fileSize: MAX_UPLOAD_BYTES,
files: 1,
},
fileFilter: function (req, file, cb) {
// Accept only image files
const filetypes = /jpeg|jpg|png|gif|webp|tiff?/;
const mimetype = filetypes.test(file.mimetype);
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
if (mimetype && extname) {
return cb(null, true);
const ext = path.extname(file.originalname).toLowerCase();
if (!ALLOWED_MIME_TYPES.has(file.mimetype) || !ALLOWED_EXTENSIONS.has(ext)) {
return cb(new Error('Only image files are allowed (jpg, png, gif, webp, tiff)'));
}
cb(new Error('Only image files are allowed'));
cb(null, true);
}
});
@@ -647,7 +665,7 @@ async function setupSshTunnel() {
port: process.env.PROD_SSH_PORT || 22,
username: process.env.PROD_SSH_USER,
privateKey: process.env.PROD_SSH_KEY_PATH
? require('fs').readFileSync(process.env.PROD_SSH_KEY_PATH)
? fs.readFileSync(process.env.PROD_SSH_KEY_PATH)
: undefined,
compress: true
};
@@ -2872,4 +2890,4 @@ router.get('/query-products', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/linesAggregate.js
+3 -3
View File
@@ -1,6 +1,6 @@
const express = require('express');
import express from 'express';
import { parseValue } from '../utils/apiHelpers.js';
const router = express.Router();
const { parseValue } = require('../utils/apiHelpers');
// --- Configuration & Helpers ---
const DEFAULT_PAGE_LIMIT = 50;
@@ -378,4 +378,4 @@ router.get('/:brand/:line/products', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/metrics.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// --- Configuration & Helpers ---
@@ -645,4 +645,4 @@ function parseValue(value, type) {
}
}
module.exports = router;
export default router;
inventory-server/src/routes/newsletter.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Shared CTE fragment for the reference date.
@@ -721,4 +721,4 @@ router.get('/campaigns/links', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/orders.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Get all orders with pagination, filtering, and sorting
@@ -258,4 +258,4 @@ router.get('/:orderNumber', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/product-editor-audit-log.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Create a new audit log entry
@@ -192,4 +192,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/products.js
+4 -24
View File
@@ -1,27 +1,7 @@
const express = require('express');
import express from 'express';
import { PurchaseOrderStatus, ReceivingStatus } from '../types/status-codes.js';
const router = express.Router();
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const { importProductsFromCSV } = require('../utils/csvImporter');
const { PurchaseOrderStatus, ReceivingStatus } = require('../types/status-codes');
// Configure multer for file uploads without silent fallbacks
const configuredUploadsDir = process.env.UPLOADS_DIR;
const uploadsDir = configuredUploadsDir
? (path.isAbsolute(configuredUploadsDir)
? configuredUploadsDir
: path.resolve(__dirname, '../../', configuredUploadsDir))
: path.resolve(__dirname, '../../uploads');
try {
fs.mkdirSync(uploadsDir, { recursive: true });
} catch (error) {
console.error(`Failed to initialize uploads directory at ${uploadsDir}:`, error);
throw error;
}
const upload = multer({ dest: uploadsDir });
// Get unique brands
router.get('/brands', async (req, res) => {
@@ -983,4 +963,4 @@ router.get('/:id/forecast', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/purchase-orders.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Status code constants
@@ -1277,4 +1277,4 @@ router.get('/pipeline', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/repeat-orders.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
// Stale PO statuses to exclude from our counting — these are the "still active"
@@ -388,4 +388,4 @@ router.get('/:pid/history', async (req, res) => {
}
});
module.exports = router;
export default router;
inventory-server/src/routes/reusable-images.js
+26 -15
View File
@@ -1,8 +1,16 @@
const express = require('express');
import express from 'express';
import multer from 'multer';
import path from 'node:path';
import fs from 'node:fs';
import { requirePermission } from '../../shared/auth/middleware.js';
const router = express.Router();
const multer = require('multer');
const path = require('path');
const fs = require('fs');
// Phase 6.2: uploads + deletions of reusable images require image_admin.
router.use((req, res, next) => {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('image_admin')(req, res, next);
});
// Create reusable uploads directory if it doesn't exist
const uploadsDir = path.join('/var/www/inventory/uploads/reusable');
@@ -38,21 +46,24 @@ const storage = multer.diskStorage({
}
});
const upload = multer({
// Phase 6.7: exact-match MIME + extension allowlist.
const ALLOWED_MIME_TYPES = new Set([
'image/jpeg', 'image/png', 'image/gif', 'image/webp',
]);
const ALLOWED_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp']);
const upload = multer({
storage: storage,
limits: {
fileSize: 5 * 1024 * 1024, // 5MB max file size
fileSize: 5 * 1024 * 1024,
files: 1,
},
fileFilter: function (req, file, cb) {
// Accept only image files
const filetypes = /jpeg|jpg|png|gif|webp/;
const mimetype = filetypes.test(file.mimetype);
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
if (mimetype && extname) {
return cb(null, true);
const ext = path.extname(file.originalname).toLowerCase();
if (!ALLOWED_MIME_TYPES.has(file.mimetype) || !ALLOWED_EXTENSIONS.has(ext)) {
return cb(new Error('Only image files are allowed (jpg, png, gif, webp)'));
}
cb(new Error('Only image files are allowed'));
cb(null, true);
}
});
@@ -393,4 +404,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/spec-lookup.js
+2 -2
View File
@@ -1,4 +1,4 @@
const express = require('express');
import express from 'express';
const router = express.Router();
const MAX_MATCHES = 500;
@@ -267,4 +267,4 @@ function descriptionAggregate(products) {
return { duplicates, samples };
}
module.exports = router;
export default router;
inventory-server/src/routes/templates.js
+16 -5
View File
@@ -1,12 +1,23 @@
const express = require('express');
const { getPool } = require('../utils/db');
const dotenv = require('dotenv');
const path = require('path');
import express from 'express';
import dotenv from 'dotenv';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { getPool } from '../utils/db.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
dotenv.config({ path: path.join(__dirname, "../../.env") });
const router = express.Router();
// Phase 6.2: template edits require templates_write. Reads pass through.
import { requirePermission } from '../../shared/auth/middleware.js';
router.use((req, res, next) => {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') return next();
return requirePermission('templates_write')(req, res, next);
});
// Get all templates
router.get('/', async (req, res) => {
try {
@@ -278,4 +289,4 @@ router.use((err, req, res, next) => {
});
});
module.exports = router;
export default router;
inventory-server/src/routes/vendorsAggregate.js
+3 -3
View File
@@ -1,6 +1,6 @@
const express = require('express');
import express from 'express';
import { parseValue } from '../utils/apiHelpers.js'; // Adjust path if needed
const router = express.Router();
const { parseValue } = require('../utils/apiHelpers'); // Adjust path if needed
// --- Configuration & Helpers ---
const DEFAULT_PAGE_LIMIT = 50;
@@ -320,4 +320,4 @@ router.get('/', async (req, res) => {
// GET /vendors-aggregate/:name (Get single vendor metric)
// Implement if needed, remember to URL-decode the name parameter
module.exports = router;
export default router;
inventory-server/src/server.js
+79 -153
View File
@@ -1,59 +1,64 @@
const express = require('express');
const cors = require('cors');
const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');
const { corsMiddleware, corsErrorHandler } = require('./middleware/cors');
const { initPool } = require('./utils/db');
const productsRouter = require('./routes/products');
const dashboardRouter = require('./routes/dashboard');
const ordersRouter = require('./routes/orders');
const csvRouter = require('./routes/data-management');
const analyticsRouter = require('./routes/analytics');
const purchaseOrdersRouter = require('./routes/purchase-orders');
const configRouter = require('./routes/config');
const metricsRouter = require('./routes/metrics');
const importRouter = require('./routes/import');
const aiValidationRouter = require('./routes/ai-validation');
const aiRouter = require('./routes/ai');
const templatesRouter = require('./routes/templates');
const aiPromptsRouter = require('./routes/ai-prompts');
const reusableImagesRouter = require('./routes/reusable-images');
const categoriesAggregateRouter = require('./routes/categoriesAggregate');
const vendorsAggregateRouter = require('./routes/vendorsAggregate');
const brandsAggregateRouter = require('./routes/brandsAggregate');
const htsLookupRouter = require('./routes/hts-lookup');
const specLookupRouter = require('./routes/spec-lookup');
const importSessionsRouter = require('./routes/import-sessions');
const importAuditLogRouter = require('./routes/import-audit-log');
const productEditorAuditLogRouter = require('./routes/product-editor-audit-log');
const newsletterRouter = require('./routes/newsletter');
const linesAggregateRouter = require('./routes/linesAggregate');
const repeatOrdersRouter = require('./routes/repeat-orders');
import { config as loadEnv } from 'dotenv';
import express from 'express';
import path from 'node:path';
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { corsMiddleware, corsErrorHandler } from './middleware/cors.js';
import { initPool } from './utils/db.js';
import { authenticate } from '../shared/auth/middleware.js';
import { requestLog } from '../shared/logging/request-log.js';
import { logger } from '../shared/logging/logger.js';
import { errorHandler } from '../shared/errors/handler.js';
import productsRouter from './routes/products.js';
import dashboardRouter from './routes/dashboard.js';
import ordersRouter from './routes/orders.js';
import csvRouter from './routes/data-management.js';
import analyticsRouter from './routes/analytics.js';
import purchaseOrdersRouter from './routes/purchase-orders.js';
import configRouter from './routes/config.js';
import metricsRouter from './routes/metrics.js';
import importRouter from './routes/import.js';
import aiValidationRouter from './routes/ai-validation.js';
import aiRouter, { initInBackground as initAiInBackground } from './routes/ai.js';
import templatesRouter from './routes/templates.js';
import aiPromptsRouter from './routes/ai-prompts.js';
import reusableImagesRouter from './routes/reusable-images.js';
import categoriesAggregateRouter from './routes/categoriesAggregate.js';
import vendorsAggregateRouter from './routes/vendorsAggregate.js';
import brandsAggregateRouter from './routes/brandsAggregate.js';
import htsLookupRouter from './routes/hts-lookup.js';
import specLookupRouter from './routes/spec-lookup.js';
import importSessionsRouter from './routes/import-sessions.js';
import importAuditLogRouter from './routes/import-audit-log.js';
import productEditorAuditLogRouter from './routes/product-editor-audit-log.js';
import newsletterRouter from './routes/newsletter.js';
import linesAggregateRouter from './routes/linesAggregate.js';
import repeatOrdersRouter from './routes/repeat-orders.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Get the absolute path to the .env file
const envPath = '/var/www/inventory/.env';
console.log('Looking for .env file at:', envPath);
console.log('.env file exists:', fs.existsSync(envPath));
loadEnv({ path: envPath });
logger.info({
envPath,
envExists: fs.existsSync(envPath),
NODE_ENV: process.env.NODE_ENV || 'not set',
PORT: process.env.PORT || 'not set',
DB_HOST: process.env.DB_HOST || 'not set',
DB_NAME: process.env.DB_NAME || 'not set',
DB_PASSWORD: process.env.DB_PASSWORD ? '[set]' : 'not set',
DB_SSL: process.env.DB_SSL || 'not set',
}, 'inventory-server starting');
try {
require('dotenv').config({ path: envPath });
console.log('.env file loaded successfully');
console.log('Environment check:', {
NODE_ENV: process.env.NODE_ENV || 'not set',
PORT: process.env.PORT || 'not set',
DB_HOST: process.env.DB_HOST || 'not set',
DB_USER: process.env.DB_USER || 'not set',
DB_NAME: process.env.DB_NAME || 'not set',
DB_PASSWORD: process.env.DB_PASSWORD ? '[password set]' : 'not set',
DB_PORT: process.env.DB_PORT || 'not set',
DB_SSL: process.env.DB_SSL || 'not set'
});
} catch (error) {
console.error('Error loading .env file:', error);
if (!process.env.JWT_SECRET) {
logger.error('JWT_SECRET is not set; refusing to start (per Phase 6.4)');
process.exit(1);
}
// Resolve important directories relative to the project root
const serverRoot = path.resolve(__dirname, '..');
const configuredUploadsDir = process.env.UPLOADS_DIR;
const uploadsDir = configuredUploadsDir
@@ -62,12 +67,10 @@ const uploadsDir = configuredUploadsDir
: path.resolve(serverRoot, configuredUploadsDir))
: path.resolve(serverRoot, 'uploads');
// Persist the resolved uploads directory so downstream modules share the same path
process.env.UPLOADS_DIR = uploadsDir;
const requiredDirs = [path.resolve(serverRoot, 'logs'), uploadsDir];
requiredDirs.forEach(dir => {
requiredDirs.forEach((dir) => {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
@@ -75,28 +78,18 @@ requiredDirs.forEach(dir => {
const app = express();
// Debug middleware to log request details
app.use((req, res, next) => {
console.log('Request details:', {
method: req.method,
url: req.url,
origin: req.get('Origin'),
headers: req.headers
});
next();
});
// Phase 6.5/6.9: structured access log (replaces the previous header-dumping debug
// middleware that wrote raw Authorization values to stdout). Pino redaction strips
// `authorization` and `cookie` automatically — see shared/logging/logger.js.
app.use(requestLog());
// Apply CORS middleware first, before any other middleware
app.use(corsMiddleware);
// Body parser middleware
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
// Initialize database pool and start server
async function startServer() {
try {
// Initialize database pool
const pool = await initPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
@@ -104,17 +97,18 @@ async function startServer() {
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432,
max: process.env.NODE_ENV === 'production' ? 20 : 10,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
ssl: process.env.DB_SSL === 'true' ? {
rejectUnauthorized: false
} : false
idleTimeoutMillis: 30_000,
connectionTimeoutMillis: 2_000,
ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
});
// Make pool available to routes
app.locals.pool = pool;
// Set up routes after pool is initialized
// Phase 6.1/6.2: every /api request requires a valid JWT. Defense in depth — Caddy
// forward_auth (when enabled) is the first reject; this is the second. Frontend
// service files MUST include `Authorization: Bearer <token>` on every fetch.
app.use('/api', authenticate({ pool, secret: process.env.JWT_SECRET }));
app.use('/api/products', productsRouter);
app.use('/api/dashboard', dashboardRouter);
app.use('/api/orders', ordersRouter);
@@ -123,10 +117,8 @@ async function startServer() {
app.use('/api/purchase-orders', purchaseOrdersRouter);
app.use('/api/config', configRouter);
app.use('/api/metrics', metricsRouter);
// Use only the aggregate routes for vendors and categories
app.use('/api/vendors', vendorsAggregateRouter);
app.use('/api/vendors', vendorsAggregateRouter);
app.use('/api/categories', categoriesAggregateRouter);
// Keep the aggregate-specific endpoints for backward compatibility
app.use('/api/categories-aggregate', categoriesAggregateRouter);
app.use('/api/vendors-aggregate', vendorsAggregateRouter);
app.use('/api/brands-aggregate', brandsAggregateRouter);
@@ -145,101 +137,35 @@ async function startServer() {
app.use('/api/lines-aggregate', linesAggregateRouter);
app.use('/api/repeat-orders', repeatOrdersRouter);
// Basic health check route
app.get('/health', (req, res) => {
res.json({
res.json({
status: 'ok',
timestamp: new Date().toISOString(),
environment: process.env.NODE_ENV
environment: process.env.NODE_ENV,
});
});
// CORS error handler - must be before other error handlers
app.use(corsErrorHandler);
// Error handling middleware - MUST be after routes and CORS error handler
app.use((err, req, res, next) => {
console.error(`[${new Date().toISOString()}] Error:`, err);
// Send detailed error in development, generic in production
const error = process.env.NODE_ENV === 'production'
? 'An internal server error occurred'
: err.message || err;
res.status(err.status || 500).json({ error });
});
app.use(errorHandler);
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
console.log(`[Server] Running in ${process.env.NODE_ENV || 'development'} mode on port ${PORT}`);
// Pre-warm AI service so taxonomy embeddings are ready before first user request
aiRouter.initInBackground();
logger.info({ port: PORT, mode: process.env.NODE_ENV || 'development' }, 'inventory-server listening');
initAiInBackground();
});
} catch (error) {
console.error('Failed to start server:', error);
logger.error({ err: error }, 'Failed to start server');
process.exit(1);
}
}
// Handle uncaught exceptions
process.on('uncaughtException', (err) => {
console.error(`[${new Date().toISOString()}] Uncaught Exception:`, err);
logger.error({ err }, 'Uncaught Exception');
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
console.error(`[${new Date().toISOString()}] Unhandled Rejection at:`, promise, 'reason:', reason);
logger.error({ reason, promise }, 'Unhandled Rejection');
});
// Initialize client sets for SSE
const importClients = new Set();
const updateClients = new Set();
const resetClients = new Set();
const resetMetricsClients = new Set();
// Helper function to send progress to SSE clients
const sendProgressToClients = (clients, data) => {
clients.forEach(client => {
try {
client.write(`data: ${JSON.stringify(data)}\n\n`);
} catch (error) {
console.error('Error sending SSE update:', error);
}
});
};
// Setup SSE connection
const setupSSE = (req, res) => {
const { type } = req.params;
// Set headers for SSE
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': req.headers.origin || '*',
'Access-Control-Allow-Credentials': 'true'
});
// Send initial message
res.write('data: {"status":"connected"}\n\n');
// Add client to appropriate set
const clientSet = type === 'import' ? importClients :
type === 'update' ? updateClients :
type === 'reset' ? resetClients :
type === 'reset-metrics' ? resetMetricsClients :
null;
if (clientSet) {
clientSet.add(res);
// Remove client when connection closes
req.on('close', () => {
clientSet.delete(res);
});
}
};
// Start the server
startServer();
startServer();
inventory-server/src/services/ai/embeddings/similarity.js
+12 -58
View File
@@ -1,82 +1,36 @@
/**
* Vector similarity utilities
*/
/**
* Compute cosine similarity between two vectors
* @param {number[]} a
* @param {number[]} b
* @returns {number} Similarity score between -1 and 1
*/
function cosineSimilarity(a, b) {
if (!a || !b || a.length !== b.length) {
return 0;
}
export function cosineSimilarity(a, b) {
if (!a || !b || a.length !== b.length) return 0;
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
if (denominator === 0) return 0;
return dotProduct / denominator;
return denominator === 0 ? 0 : dotProduct / denominator;
}
/**
* Find top K most similar items from a collection
* @param {number[]} queryEmbedding - The embedding to search for
* @param {Array<{id: any, embedding: number[]}>} items - Items with embeddings
* @param {number} topK - Number of results to return
* @returns {Array<{id: any, similarity: number}>}
*/
function findTopMatches(queryEmbedding, items, topK = 10) {
if (!queryEmbedding || !items || items.length === 0) {
return [];
}
const scored = items.map(item => ({
export function findTopMatches(queryEmbedding, items, topK = 10) {
if (!queryEmbedding || !items || items.length === 0) return [];
const scored = items.map((item) => ({
id: item.id,
similarity: cosineSimilarity(queryEmbedding, item.embedding)
similarity: cosineSimilarity(queryEmbedding, item.embedding),
}));
scored.sort((a, b) => b.similarity - a.similarity);
return scored.slice(0, topK);
}
/**
* Find matches above a similarity threshold
* @param {number[]} queryEmbedding
* @param {Array<{id: any, embedding: number[]}>} items
* @param {number} threshold - Minimum similarity (0-1)
* @returns {Array<{id: any, similarity: number}>}
*/
function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
if (!queryEmbedding || !items || items.length === 0) {
return [];
}
export function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
if (!queryEmbedding || !items || items.length === 0) return [];
const scored = items
.map(item => ({
.map((item) => ({
id: item.id,
similarity: cosineSimilarity(queryEmbedding, item.embedding)
similarity: cosineSimilarity(queryEmbedding, item.embedding),
}))
.filter(item => item.similarity >= threshold);
.filter((item) => item.similarity >= threshold);
scored.sort((a, b) => b.similarity - a.similarity);
return scored;
}
module.exports = {
cosineSimilarity,
findTopMatches,
findMatchesAboveThreshold
};
inventory-server/src/services/ai/embeddings/taxonomyEmbeddings.js
+42 -141
View File
@@ -1,74 +1,52 @@
/**
* Taxonomy Embedding Service
*
* Generates and caches embeddings for categories, themes, and colors.
* Excludes "Black Friday", "Gifts", "Deals" categories and their children.
*
* Disk cache: embeddings are saved to data/taxonomy-embeddings.json and reused
* across server restarts. Cache is invalidated by content hash — if the taxonomy
* rows in MySQL change, the next check will detect it and regenerate automatically.
*
* Background check: after initialization, call startBackgroundCheck(getConnectionFn)
* to poll for taxonomy changes on a configurable interval (default 1h).
* Disk cache at data/taxonomy-embeddings.json; content-hash invalidated.
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { findTopMatches } = require('./similarity');
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { findTopMatches } from './similarity.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Categories to exclude (and all their children)
const EXCLUDED_CATEGORY_NAMES = ['black friday', 'gifts', 'deals'];
// Disk cache config
const CACHE_PATH = path.join(__dirname, '..', '..', '..', '..', 'data', 'taxonomy-embeddings.json');
class TaxonomyEmbeddings {
export class TaxonomyEmbeddings {
constructor({ provider, logger }) {
this.provider = provider;
this.logger = logger || console;
// Cached taxonomy with embeddings
this.categories = [];
this.themes = [];
this.colors = [];
// Raw data without embeddings (for lookup)
this.categoryMap = new Map();
this.themeMap = new Map();
this.colorMap = new Map();
// Content hash of the last successfully built taxonomy (from DB rows)
this.contentHash = null;
this.initialized = false;
this.initializing = false;
this._checkInterval = null;
this._regenerating = false;
}
/**
* Initialize embeddings — fetches raw taxonomy rows to compute a content hash,
* then either loads the matching disk cache or generates fresh embeddings.
*/
async initialize(connection) {
if (this.initialized) {
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
}
if (this.initializing) {
// Wait for existing initialization
while (this.initializing) {
await new Promise(resolve => setTimeout(resolve, 100));
await new Promise((resolve) => setTimeout(resolve, 100));
}
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
}
this.initializing = true;
try {
// Always fetch raw rows first — cheap (~10ms), no OpenAI calls.
// Used to compute a content hash for cache validation.
const rawRows = await this._fetchRawRows(connection);
const freshHash = this._computeContentHash(rawRows);
@@ -77,9 +55,9 @@ class TaxonomyEmbeddings {
this.categories = cached.categories;
this.themes = cached.themes;
this.colors = cached.colors;
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
this.categoryMap = new Map(this.categories.map((c) => [c.id, c]));
this.themeMap = new Map(this.themes.map((t) => [t.id, t]));
this.colorMap = new Map(this.colors.map((c) => [c.id, c]));
this.contentHash = freshHash;
this.initialized = true;
this.logger.info(`[TaxonomyEmbeddings] Loaded from cache: ${this.categories.length} categories, ${this.themes.length} themes, ${this.colors.length} colors`);
@@ -95,7 +73,6 @@ class TaxonomyEmbeddings {
await this._buildAndEmbed(rawRows, freshHash);
this.initialized = true;
this.logger.info('[TaxonomyEmbeddings] Initialization complete');
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
} catch (error) {
this.logger.error('[TaxonomyEmbeddings] Initialization failed:', error);
@@ -105,26 +82,16 @@ class TaxonomyEmbeddings {
}
}
/**
* Start a background interval that checks for taxonomy changes and regenerates
* embeddings automatically if the content hash differs.
*
* @param {Function} getConnectionFn - async function returning { connection }
* @param {number} intervalMs - check interval, default 1 hour
*/
startBackgroundCheck(getConnectionFn, intervalMs = 60 * 60 * 1000) {
if (this._checkInterval) return;
this.logger.info(`[TaxonomyEmbeddings] Background taxonomy check started (every ${intervalMs / 60000} min)`);
this._checkInterval = setInterval(async () => {
if (this._regenerating) return;
try {
const { connection } = await getConnectionFn();
const rawRows = await this._fetchRawRows(connection);
const freshHash = this._computeContentHash(rawRows);
if (freshHash === this.contentHash) return;
this.logger.info('[TaxonomyEmbeddings] Taxonomy changed, regenerating embeddings in background...');
@@ -146,123 +113,79 @@ class TaxonomyEmbeddings {
}
}
/**
* Find similar categories for a product embedding
*/
findSimilarCategories(productEmbedding, topK = 10) {
if (!this.initialized || !productEmbedding) {
return [];
}
if (!this.initialized || !productEmbedding) return [];
const matches = findTopMatches(productEmbedding, this.categories, topK);
return matches.map(match => {
return matches.map((match) => {
const cat = this.categoryMap.get(match.id);
return {
id: match.id,
name: cat?.name || '',
fullPath: cat?.fullPath || '',
similarity: match.similarity
similarity: match.similarity,
};
});
}
/**
* Find similar themes for a product embedding
*/
findSimilarThemes(productEmbedding, topK = 5) {
if (!this.initialized || !productEmbedding) {
return [];
}
if (!this.initialized || !productEmbedding) return [];
const matches = findTopMatches(productEmbedding, this.themes, topK);
return matches.map(match => {
return matches.map((match) => {
const theme = this.themeMap.get(match.id);
return {
id: match.id,
name: theme?.name || '',
fullPath: theme?.fullPath || '',
similarity: match.similarity
similarity: match.similarity,
};
});
}
/**
* Find similar colors for a product embedding
*/
findSimilarColors(productEmbedding, topK = 5) {
if (!this.initialized || !productEmbedding) {
return [];
}
if (!this.initialized || !productEmbedding) return [];
const matches = findTopMatches(productEmbedding, this.colors, topK);
return matches.map(match => {
return matches.map((match) => {
const color = this.colorMap.get(match.id);
return {
id: match.id,
name: color?.name || '',
similarity: match.similarity
similarity: match.similarity,
};
});
}
/**
* Get all taxonomy data (without embeddings) for frontend
*/
getTaxonomyData() {
return {
categories: this.categories.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
themes: this.themes.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
colors: this.colors.map(({ id, name }) => ({ id, name }))
colors: this.colors.map(({ id, name }) => ({ id, name })),
};
}
/**
* Check if service is ready
*/
isReady() {
return this.initialized;
}
// ============================================================================
// Private Methods
// ============================================================================
/**
* Fetch minimal raw rows from MySQL — used for content hash computation.
* This is the cheap path: no path-building, no embeddings, just the raw data.
*/
async _fetchRawRows(connection) {
const [[catRows], [themeRows], [colorRows]] = await Promise.all([
connection.query('SELECT cat_id, name, master_cat_id, type FROM product_categories WHERE type IN (10, 11, 12, 13) ORDER BY cat_id'),
connection.query('SELECT cat_id, name, master_cat_id, type FROM product_categories WHERE type IN (20, 21) ORDER BY cat_id'),
connection.query('SELECT color, name, hex_color FROM product_color_list ORDER BY `order`')
connection.query('SELECT color, name, hex_color FROM product_color_list ORDER BY `order`'),
]);
return { catRows, themeRows, colorRows };
}
/**
* Compute a stable SHA-256 hash of the taxonomy row content.
* Any change to IDs, names, or parent relationships will produce a different hash.
*/
_computeContentHash({ catRows, themeRows, colorRows }) {
const content = JSON.stringify({
cats: catRows.map(r => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
themes: themeRows.map(r => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
colors: colorRows.map(r => [r.color, r.name]).sort()
cats: catRows.map((r) => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
themes: themeRows.map((r) => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
colors: colorRows.map((r) => [r.color, r.name]).sort(),
});
return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
}
/**
* Build full taxonomy objects and generate embeddings, then atomically swap
* the in-memory state. Called on cache miss and on background change detection.
*/
async _buildAndEmbed(rawRows, contentHash) {
const { catRows, themeRows, colorRows } = rawRows;
const categories = this._buildCategories(catRows);
const themes = this._buildThemes(themeRows);
const colors = this._buildColors(colorRows);
@@ -272,23 +195,22 @@ class TaxonomyEmbeddings {
const [catEmbeddings, themeEmbeddings, colorEmbeddings] = await Promise.all([
this._generateEmbeddings(categories, 'categories'),
this._generateEmbeddings(themes, 'themes'),
this._generateEmbeddings(colors, 'colors')
this._generateEmbeddings(colors, 'colors'),
]);
// Atomic in-memory swap (single-threaded JS — readers always see a consistent state)
this.categories = catEmbeddings;
this.themes = themeEmbeddings;
this.colors = colorEmbeddings;
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
this.categoryMap = new Map(this.categories.map((c) => [c.id, c]));
this.themeMap = new Map(this.themes.map((t) => [t.id, t]));
this.colorMap = new Map(this.colors.map((c) => [c.id, c]));
this.contentHash = contentHash;
this._saveCache();
}
_buildCategories(rows) {
const byId = new Map(rows.map(r => [r.cat_id, r]));
const byId = new Map(rows.map((r) => [r.cat_id, r]));
const excludedIds = new Set();
for (const row of rows) {
@@ -297,7 +219,6 @@ class TaxonomyEmbeddings {
}
}
// Multiple passes to find all descendants of excluded categories
let foundNew = true;
while (foundNew) {
foundNew = false;
@@ -314,98 +235,80 @@ class TaxonomyEmbeddings {
const categories = [];
for (const row of rows) {
if (excludedIds.has(row.cat_id)) continue;
const pathParts = [];
let current = row;
while (current) {
pathParts.unshift(current.name);
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
}
categories.push({
id: row.cat_id,
name: row.name,
parentId: row.master_cat_id,
type: row.type,
fullPath: pathParts.join(' > '),
embeddingText: pathParts.join(' ')
embeddingText: pathParts.join(' '),
});
}
return categories;
}
_buildThemes(rows) {
const byId = new Map(rows.map(r => [r.cat_id, r]));
return rows.map(row => {
const byId = new Map(rows.map((r) => [r.cat_id, r]));
return rows.map((row) => {
const pathParts = [];
let current = row;
while (current) {
pathParts.unshift(current.name);
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
}
return {
id: row.cat_id,
name: row.name,
parentId: row.master_cat_id,
type: row.type,
fullPath: pathParts.join(' > '),
embeddingText: pathParts.join(' ')
embeddingText: pathParts.join(' '),
};
});
}
_buildColors(rows) {
return rows.map(row => ({
return rows.map((row) => ({
id: row.color,
name: row.name,
hexColor: row.hex_color,
embeddingText: row.name
embeddingText: row.name,
}));
}
async _generateEmbeddings(items, label) {
if (items.length === 0) {
return items;
}
if (items.length === 0) return items;
const startTime = Date.now();
const texts = items.map(item => item.embeddingText);
const texts = items.map((item) => item.embeddingText);
const results = [...items];
// Process in batches
for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) {
for (let i = 0; i < chunk.embeddings.length; i++) {
const globalIndex = chunk.startIndex + i;
results[globalIndex] = {
...results[globalIndex],
embedding: chunk.embeddings[i]
};
results[globalIndex] = { ...results[globalIndex], embedding: chunk.embeddings[i] };
}
}
const elapsed = Date.now() - startTime;
this.logger.info(`[TaxonomyEmbeddings] Generated ${items.length} ${label} embeddings in ${elapsed}ms`);
return results;
}
// ============================================================================
// Disk Cache Methods
// ============================================================================
_loadCache() {
try {
if (!fs.existsSync(CACHE_PATH)) return null;
const data = JSON.parse(fs.readFileSync(CACHE_PATH, 'utf8'));
if (!data.contentHash || !data.categories?.length || !data.themes?.length || !data.colors?.length) {
this.logger.warn('[TaxonomyEmbeddings] Disk cache malformed or missing content hash, will regenerate');
return null;
}
return data;
} catch (err) {
this.logger.warn('[TaxonomyEmbeddings] Failed to load disk cache:', err.message);
@@ -429,5 +332,3 @@ class TaxonomyEmbeddings {
}
}
}
module.exports = { TaxonomyEmbeddings };
inventory-server/src/services/ai/index.js
+55 -220
View File
@@ -1,17 +1,16 @@
/**
* AI Service
*
* Main entry point for AI functionality including:
* - Embeddings for taxonomy suggestions (OpenAI)
* - Chat completions for validation tasks (Groq)
* - Task registry for AI operations
* Main entry point for AI functionality (embeddings + chat completions + task registry).
*/
const { OpenAIProvider } = require('./providers/openaiProvider');
const { GroqProvider, MODELS: GROQ_MODELS } = require('./providers/groqProvider');
const { TaxonomyEmbeddings } = require('./embeddings/taxonomyEmbeddings');
const { cosineSimilarity, findTopMatches } = require('./embeddings/similarity');
const { getRegistry, TASK_IDS, registerAllTasks } = require('./tasks');
import { OpenAIProvider } from './providers/openaiProvider.js';
import { GroqProvider, MODELS as GROQ_MODELS } from './providers/groqProvider.js';
import { TaxonomyEmbeddings } from './embeddings/taxonomyEmbeddings.js';
import { cosineSimilarity, findTopMatches } from './embeddings/similarity.js';
import { getRegistry, TASK_IDS, registerAllTasks } from './tasks/index.js';
export { TASK_IDS, GROQ_MODELS, cosineSimilarity, findTopMatches };
let initialized = false;
let initializing = false;
@@ -19,54 +18,28 @@ let openaiProvider = null;
let groqProvider = null;
let taxonomyEmbeddings = null;
let logger = console;
// Store pool reference for task access
let appPool = null;
/**
* Initialize the AI service
* @param {Object} options
* @param {string} options.openaiApiKey - OpenAI API key (for embeddings)
* @param {string} [options.groqApiKey] - Groq API key (for chat completions)
* @param {Object} options.mysqlConnection - MySQL connection for taxonomy data
* @param {Object} [options.pool] - PostgreSQL pool for prompt loading
* @param {Object} [options.logger] - Logger instance
*/
async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, logger: customLogger }) {
if (initialized) {
return { success: true, message: 'Already initialized' };
}
export async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, logger: customLogger }) {
if (initialized) return { success: true, message: 'Already initialized' };
if (initializing) {
// Wait for existing initialization
while (initializing) {
await new Promise(resolve => setTimeout(resolve, 100));
await new Promise((resolve) => setTimeout(resolve, 100));
}
return { success: initialized, message: initialized ? 'Initialized' : 'Initialization failed' };
}
initializing = true;
try {
if (customLogger) {
logger = customLogger;
}
if (!openaiApiKey) {
throw new Error('OpenAI API key is required');
}
if (customLogger) logger = customLogger;
if (!openaiApiKey) throw new Error('OpenAI API key is required');
logger.info('[AI] Initializing AI service...');
if (pool) appPool = pool;
// Store pool reference for tasks
if (pool) {
appPool = pool;
}
// Create OpenAI provider (for embeddings)
openaiProvider = new OpenAIProvider({ apiKey: openaiApiKey });
// Create Groq provider (for chat completions) if API key provided
if (groqApiKey) {
groqProvider = new GroqProvider({ apiKey: groqApiKey });
logger.info('[AI] Groq provider initialized for chat completions');
@@ -74,32 +47,19 @@ async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, log
logger.warn('[AI] No Groq API key provided - chat completion tasks will not be available');
}
// Create and initialize taxonomy embeddings
taxonomyEmbeddings = new TaxonomyEmbeddings({
provider: openaiProvider,
logger
});
taxonomyEmbeddings = new TaxonomyEmbeddings({ provider: openaiProvider, logger });
const stats = await taxonomyEmbeddings.initialize(mysqlConnection);
// Register validation tasks if Groq is available
if (groqProvider) {
registerValidationTasks();
}
if (groqProvider) registerValidationTasks();
initialized = true;
logger.info('[AI] AI service initialized', {
...stats,
groqEnabled: !!groqProvider,
tasksRegistered: getRegistry().list()
tasksRegistered: getRegistry().list(),
});
return {
success: true,
message: 'Initialized',
stats,
groqEnabled: !!groqProvider
};
return { success: true, message: 'Initialized', stats, groqEnabled: !!groqProvider };
} catch (error) {
logger.error('[AI] Initialization failed:', error);
return { success: false, message: error.message };
@@ -108,41 +68,20 @@ async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, log
}
}
/**
* Register validation tasks with the task registry
* Called during initialization if Groq is available
*/
function registerValidationTasks() {
registerAllTasks(logger);
logger.info('[AI] Validation tasks registered');
}
/**
* Check if service is ready
*/
function isReady() {
export function isReady() {
return initialized && taxonomyEmbeddings?.isReady();
}
/**
* Start background taxonomy change detection.
* Call once after initialization, passing a function that returns { connection }.
* @param {Function} getConnectionFn
* @param {number} [intervalMs] - default 1 hour
*/
function startBackgroundCheck(getConnectionFn, intervalMs) {
export function startBackgroundCheck(getConnectionFn, intervalMs) {
if (!initialized || !taxonomyEmbeddings) return;
taxonomyEmbeddings.startBackgroundCheck(getConnectionFn, intervalMs);
}
/**
* Build weighted product text for embedding.
* Weights the product name heavily by repeating it, and truncates long descriptions
* to prevent verbose marketing copy from drowning out the product signal.
*
* @param {Object} product - Product with name, description, company, line
* @returns {string} - Combined text for embedding
*/
function buildProductText(product) {
const parts = [];
const name = product.name?.trim();
@@ -150,20 +89,9 @@ function buildProductText(product) {
const company = (product.company_name || product.company)?.trim();
const line = (product.line_name || product.line)?.trim();
// Name is most important - repeat 3x to weight it heavily in the embedding
if (name) {
parts.push(name, name, name);
}
// Company and line provide context
if (company) {
parts.push(company);
}
if (line) {
parts.push(line);
}
// Truncate description to prevent it from overwhelming the signal
if (name) parts.push(name, name, name);
if (company) parts.push(company);
if (line) parts.push(line);
if (description) {
const truncated = description.length > 500
? description.substring(0, 500) + '...'
@@ -174,74 +102,35 @@ function buildProductText(product) {
return parts.join(' ').trim();
}
/**
* Generate embedding for a product
* @param {Object} product - Product with name, description, company, line
* @returns {Promise<{embedding: number[], latencyMs: number}>}
*/
async function getProductEmbedding(product) {
if (!initialized || !openaiProvider) {
throw new Error('AI service not initialized');
}
export async function getProductEmbedding(product) {
if (!initialized || !openaiProvider) throw new Error('AI service not initialized');
const text = buildProductText(product);
if (!text) {
return { embedding: null, latencyMs: 0 };
}
if (!text) return { embedding: null, latencyMs: 0 };
const result = await openaiProvider.embed(text);
return {
embedding: result.embeddings[0],
latencyMs: result.latencyMs
};
return { embedding: result.embeddings[0], latencyMs: result.latencyMs };
}
/**
* Generate embeddings for multiple products
* @param {Object[]} products - Array of products
* @returns {Promise<{embeddings: Array<{index: number, embedding: number[]}>, latencyMs: number}>}
*/
async function getProductEmbeddings(products) {
if (!initialized || !openaiProvider) {
throw new Error('AI service not initialized');
}
export async function getProductEmbeddings(products) {
if (!initialized || !openaiProvider) throw new Error('AI service not initialized');
const texts = products.map(buildProductText);
const validIndices = texts.map((t, i) => t ? i : -1).filter((i) => i >= 0);
const validTexts = texts.filter((t) => t);
// Track which products have empty text
const validIndices = texts.map((t, i) => t ? i : -1).filter(i => i >= 0);
const validTexts = texts.filter(t => t);
if (validTexts.length === 0) {
return { embeddings: [], latencyMs: 0 };
}
if (validTexts.length === 0) return { embeddings: [], latencyMs: 0 };
const result = await openaiProvider.embed(validTexts);
// Map embeddings back to original indices
const embeddings = validIndices.map((originalIndex, resultIndex) => ({
index: originalIndex,
embedding: result.embeddings[resultIndex]
embedding: result.embeddings[resultIndex],
}));
return {
embeddings,
latencyMs: result.latencyMs
};
return { embeddings, latencyMs: result.latencyMs };
}
/**
* Find similar taxonomy items for a product embedding
* @param {number[]} productEmbedding
* @param {Object} options
* @returns {{categories: Array, themes: Array, colors: Array}}
*/
function findSimilarTaxonomy(productEmbedding, options = {}) {
if (!initialized || !taxonomyEmbeddings) {
throw new Error('AI service not initialized');
}
export function findSimilarTaxonomy(productEmbedding, options = {}) {
if (!initialized || !taxonomyEmbeddings) throw new Error('AI service not initialized');
const topCategories = options.topCategories ?? 10;
const topThemes = options.topThemes ?? 5;
@@ -250,25 +139,15 @@ function findSimilarTaxonomy(productEmbedding, options = {}) {
return {
categories: taxonomyEmbeddings.findSimilarCategories(productEmbedding, topCategories),
themes: taxonomyEmbeddings.findSimilarThemes(productEmbedding, topThemes),
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors)
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors),
};
}
/**
* Get product embedding and find similar taxonomy in one call
* @param {Object} product
* @param {Object} options
*/
async function getSuggestionsForProduct(product, options = {}) {
export async function getSuggestionsForProduct(product, options = {}) {
const { embedding, latencyMs: embeddingLatency } = await getProductEmbedding(product);
if (!embedding) {
return {
categories: [],
themes: [],
colors: [],
latencyMs: embeddingLatency
};
return { categories: [], themes: [], colors: [], latencyMs: embeddingLatency };
}
const startSearch = Date.now();
@@ -279,27 +158,17 @@ async function getSuggestionsForProduct(product, options = {}) {
...suggestions,
latencyMs: embeddingLatency + searchLatency,
embeddingLatencyMs: embeddingLatency,
searchLatencyMs: searchLatency
searchLatencyMs: searchLatency,
};
}
/**
* Get all taxonomy data (without embeddings) for frontend
*/
function getTaxonomyData() {
if (!initialized || !taxonomyEmbeddings) {
throw new Error('AI service not initialized');
}
export function getTaxonomyData() {
if (!initialized || !taxonomyEmbeddings) throw new Error('AI service not initialized');
return taxonomyEmbeddings.getTaxonomyData();
}
/**
* Get service status
*/
function getStatus() {
export function getStatus() {
const registry = getRegistry();
return {
initialized,
ready: isReady(),
@@ -309,90 +178,56 @@ function getStatus() {
taxonomyStats: taxonomyEmbeddings ? {
categories: taxonomyEmbeddings.categories?.length || 0,
themes: taxonomyEmbeddings.themes?.length || 0,
colors: taxonomyEmbeddings.colors?.length || 0
colors: taxonomyEmbeddings.colors?.length || 0,
} : null,
tasks: {
registered: registry.list(),
count: registry.size()
}
count: registry.size(),
},
};
}
/**
* Run an AI task by ID
* @param {string} taskId - Task identifier from TASK_IDS
* @param {Object} payload - Task-specific input
* @returns {Promise<Object>} Task result
*/
async function runTask(taskId, payload = {}) {
if (!initialized) {
throw new Error('AI service not initialized');
}
if (!groqProvider) {
throw new Error('Groq provider not available - chat completion tasks require GROQ_API_KEY');
}
export async function runTask(taskId, payload = {}) {
if (!initialized) throw new Error('AI service not initialized');
if (!groqProvider) throw new Error('Groq provider not available - chat completion tasks require GROQ_API_KEY');
const registry = getRegistry();
return registry.runTask(taskId, {
...payload,
// Inject dependencies tasks may need
provider: groqProvider,
// Use pool from payload if provided (from route), fall back to stored appPool
pool: payload.pool || appPool,
logger
logger,
});
}
/**
* Get the Groq provider instance (for direct use if needed)
* @returns {GroqProvider|null}
*/
function getGroqProvider() {
export function getGroqProvider() {
return groqProvider;
}
/**
* Get the PostgreSQL pool (for tasks that need DB access)
* @returns {Object|null}
*/
function getPool() {
export function getPool() {
return appPool;
}
/**
* Check if chat completion tasks are available
* @returns {boolean}
*/
function hasChatCompletion() {
export function hasChatCompletion() {
return !!groqProvider;
}
module.exports = {
// Initialization
export default {
initialize,
isReady,
getStatus,
startBackgroundCheck,
// Embeddings (OpenAI)
getProductEmbedding,
getProductEmbeddings,
findSimilarTaxonomy,
getSuggestionsForProduct,
getTaxonomyData,
// Chat completions (Groq)
runTask,
hasChatCompletion,
getGroqProvider,
getPool,
// Constants
TASK_IDS,
GROQ_MODELS,
// Re-export utilities
cosineSimilarity,
findTopMatches
findTopMatches,
};
inventory-server/src/services/ai/prompts/descriptionPrompts.js
+11 -88
View File
@@ -1,81 +1,41 @@
/**
* Description Validation Prompts
*
* Functions for building and parsing description validation prompts.
* System and general prompts are loaded from the database.
*/
/**
* Sanitize an issue string from AI response
* AI sometimes returns malformed strings with escape sequences
*
* @param {string} issue - Raw issue string
* @returns {string} Cleaned issue string
*/
function sanitizeIssue(issue) {
if (!issue || typeof issue !== 'string') return '';
let cleaned = issue
// Remove trailing backslashes (incomplete escapes)
return issue
.replace(/\\+$/, '')
// Fix malformed escaped quotes at end of string
.replace(/\\",?\)?$/, '')
// Clean up double-escaped quotes
.replace(/\\\\"/g, '"')
// Clean up single escaped quotes that aren't needed
.replace(/\\"/g, '"')
// Remove any remaining trailing punctuation artifacts
.replace(/[,\s]+$/, '')
// Trim whitespace
.trim();
return cleaned;
}
/**
* Build the user prompt for description validation
* Combines database prompts with product data
*
* @param {Object} product - Product data
* @param {string} product.name - Product name
* @param {string} product.description - Current description
* @param {string} [product.company_name] - Company name
* @param {string} [product.categories] - Product categories
* @param {Object} prompts - Prompts loaded from database
* @param {string} prompts.general - General description guidelines
* @param {string} [prompts.companySpecific] - Company-specific rules
* @returns {string} Complete user prompt
*/
function buildDescriptionUserPrompt(product, prompts) {
export function buildDescriptionUserPrompt(product, prompts) {
const parts = [];
// Add general prompt/guidelines if provided
if (prompts.general) {
parts.push(prompts.general);
parts.push(''); // Empty line for separation
parts.push('');
}
// Add company-specific rules if provided
if (prompts.companySpecific) {
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
parts.push(prompts.companySpecific);
parts.push(''); // Empty line for separation
parts.push('');
}
// Add product information
parts.push('PRODUCT TO VALIDATE:');
parts.push(`NAME: "${product.name || ''}"`);
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
if (product.categories) {
parts.push(`CATEGORIES: ${product.categories}`);
}
if (product.categories) parts.push(`CATEGORIES: ${product.categories}`);
parts.push('');
parts.push('CURRENT DESCRIPTION:');
parts.push(`"${product.description || '(empty)'}"`);
// Add response format instructions
parts.push('');
parts.push('CRITICAL RULES:');
parts.push('- If isValid is false, you MUST provide a suggestion with the improved description');
@@ -86,68 +46,40 @@ function buildDescriptionUserPrompt(product, prompts) {
parts.push(JSON.stringify({
isValid: 'true if perfect, false if ANY changes needed',
suggestion: 'REQUIRED when isValid is false - the complete improved description',
issues: ['list each problem found (empty array only if isValid is true)']
issues: ['list each problem found (empty array only if isValid is true)'],
}, null, 2));
return parts.join('\n');
}
/**
* Parse the AI response for description validation
*
* @param {Object|null} parsed - Parsed JSON from AI
* @param {string} content - Raw response content
* @returns {Object}
*/
function parseDescriptionResponse(parsed, content) {
// If we got valid parsed JSON, use it
export function parseDescriptionResponse(parsed, content) {
if (parsed && typeof parsed.isValid === 'boolean') {
// Sanitize issues - AI sometimes returns malformed escape sequences
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
const issues = rawIssues
.map(sanitizeIssue)
.filter(issue => issue.length > 0);
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
const suggestion = parsed.suggestion || null;
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
// If there are issues, treat as invalid regardless of what the AI said.
// Also if there's a suggestion, the AI thought something needed to change.
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
return { isValid, suggestion, issues };
}
// Handle case where isValid is a string "true"/"false" instead of boolean
if (parsed && typeof parsed.isValid === 'string') {
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
const issues = rawIssues
.map(sanitizeIssue)
.filter(issue => issue.length > 0);
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
const suggestion = parsed.suggestion || null;
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
// Same defensive logic: if there are issues, it's not valid
const isValid = rawIsValid && issues.length === 0 && !suggestion;
return { isValid, suggestion, issues };
}
// Try to extract from content if parsing failed
try {
// Look for isValid pattern
const isValidMatch = content.match(/"isValid"\s*:\s*(true|false)/i);
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
// Look for suggestion (might be multiline)
const suggestionMatch = content.match(/"suggestion"\s*:\s*"((?:[^"\\]|\\.)*)"/s);
let suggestion = suggestionMatch ? suggestionMatch[1] : null;
if (suggestion) {
// Unescape common escapes
suggestion = suggestion.replace(/\\n/g, '\n').replace(/\\"/g, '"');
}
// Look for issues array
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
let issues = [];
if (issuesMatch) {
@@ -155,22 +87,13 @@ function parseDescriptionResponse(parsed, content) {
const issueStrings = issuesContent.match(/"([^"]+)"/g);
if (issueStrings) {
issues = issueStrings
.map(s => sanitizeIssue(s.replace(/"/g, '')))
.filter(issue => issue.length > 0);
.map((s) => sanitizeIssue(s.replace(/"/g, '')))
.filter((issue) => issue.length > 0);
}
}
// Same logic: if there are issues, it's not valid
const finalIsValid = isValid && issues.length === 0 && !suggestion;
return { isValid: finalIsValid, suggestion, issues };
} catch {
// Default to valid if we can't parse anything
return { isValid: true, suggestion: null, issues: [] };
}
}
module.exports = {
buildDescriptionUserPrompt,
parseDescriptionResponse
};
inventory-server/src/services/ai/prompts/namePrompts.js
+14 -93
View File
@@ -1,164 +1,94 @@
/**
* Name Validation Prompts
*
* Functions for building and parsing name validation prompts.
* System and general prompts are loaded from the database.
*/
/**
* Sanitize an issue string from AI response
* AI sometimes returns malformed strings with escape sequences
*
* @param {string} issue - Raw issue string
* @returns {string} Cleaned issue string
*/
function sanitizeIssue(issue) {
if (!issue || typeof issue !== 'string') return '';
let cleaned = issue
// Remove trailing backslashes (incomplete escapes)
return issue
.replace(/\\+$/, '')
// Fix malformed escaped quotes at end of string
.replace(/\\",?\)?$/, '')
// Clean up double-escaped quotes
.replace(/\\\\"/g, '"')
// Clean up single escaped quotes that aren't needed
.replace(/\\"/g, '"')
// Remove any remaining trailing punctuation artifacts
.replace(/[,\s]+$/, '')
// Trim whitespace
.trim();
return cleaned;
}
/**
* Build the user prompt for name validation
* Combines database prompts with product data
*
* @param {Object} product - Product data
* @param {string} product.name - Current product name
* @param {string} [product.company_name] - Company name
* @param {string} [product.line_name] - Product line name
* @param {string} [product.subline_name] - Product subline name
* @param {string[]} [product.siblingNames] - Names of other products in the same line
* @param {Object} prompts - Prompts loaded from database
* @param {string} prompts.general - General naming conventions
* @param {string} [prompts.companySpecific] - Company-specific rules
* @returns {string} Complete user prompt
*/
function buildNameUserPrompt(product, prompts) {
export function buildNameUserPrompt(product, prompts) {
const parts = [];
// Add general prompt/conventions if provided
if (prompts.general) {
parts.push(prompts.general);
parts.push(''); // Empty line for separation
parts.push('');
}
// Add company-specific rules if provided
if (prompts.companySpecific) {
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
parts.push(prompts.companySpecific);
parts.push(''); // Empty line for separation
parts.push('');
}
// Add product information
parts.push('PRODUCT TO VALIDATE:');
parts.push(`NAME: "${product.name || ''}"`);
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
parts.push(`LINE: ${product.line_name || 'None'}`);
if (product.subline_name) {
parts.push(`SUBLINE: ${product.subline_name}`);
}
if (product.subline_name) parts.push(`SUBLINE: ${product.subline_name}`);
// Add sibling context for naming decisions
if (product.siblingNames && product.siblingNames.length > 0) {
parts.push('');
parts.push(`OTHER PRODUCTS IN THIS LINE (${product.siblingNames.length + 1} total including this one):`);
product.siblingNames.forEach(name => {
parts.push(`- ${name}`);
});
product.siblingNames.forEach((name) => parts.push(`- ${name}`));
}
// Add response format instructions
parts.push('');
parts.push('RESPOND WITH JSON:');
parts.push(JSON.stringify({
isValid: 'true/false',
suggestion: 'corrected name if changes needed, or null if valid',
issues: ['issue 1', 'issue 2 (empty array if valid)']
issues: ['issue 1', 'issue 2 (empty array if valid)'],
}, null, 2));
return parts.join('\n');
}
/**
* Parse the AI response for name validation
*
* @param {Object|null} parsed - Parsed JSON from AI
* @param {string} content - Raw response content
* @returns {Object}
*/
function parseNameResponse(parsed, content) {
// Debug: Log what we're trying to parse
export function parseNameResponse(parsed, content) {
console.log('[parseNameResponse] Input:', {
hasParsed: !!parsed,
parsedIsValid: parsed?.isValid,
parsedType: typeof parsed?.isValid,
contentPreview: content?.substring(0, 3000)
contentPreview: content?.substring(0, 3000),
});
// If we got valid parsed JSON, use it
if (parsed && typeof parsed.isValid === 'boolean') {
// Sanitize issues - AI sometimes returns malformed escape sequences
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
const issues = rawIssues
.map(sanitizeIssue)
.filter(issue => issue.length > 0);
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
const suggestion = parsed.suggestion || null;
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
// If there are issues, treat as invalid regardless of what the AI said.
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
return { isValid, suggestion, issues };
}
// Handle case where isValid is a string "true"/"false" instead of boolean
if (parsed && typeof parsed.isValid === 'string') {
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
const issues = rawIssues
.map(sanitizeIssue)
.filter(issue => issue.length > 0);
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
const suggestion = parsed.suggestion || null;
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
// Same defensive logic: if there are issues, it's not valid
const isValid = rawIsValid && issues.length === 0 && !suggestion;
console.log('[parseNameResponse] Parsed isValid as string:', parsed.isValid, '→', isValid);
return { isValid, suggestion, issues };
}
// Try to extract from content if parsing failed
try {
// Look for isValid pattern - handle both boolean and quoted string
// Matches: "isValid": true, "isValid": false, "isValid": "true", "isValid": "false"
const isValidMatch = content.match(/"isValid"\s*:\s*"?(true|false)"?/i);
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
console.log('[parseNameResponse] Regex extraction:', {
isValidMatch: isValidMatch?.[0],
isValidValue: isValidMatch?.[1],
resultIsValid: isValid
resultIsValid: isValid,
});
// Look for suggestion - handle escaped quotes and null
const suggestionMatch = content.match(/"suggestion"\s*:\s*(?:"([^"\\]*(?:\\.[^"\\]*)*)"|null)/);
const suggestion = suggestionMatch ? (suggestionMatch[1] || null) : null;
// Look for issues array
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
let issues = [];
if (issuesMatch) {
@@ -166,22 +96,13 @@ function parseNameResponse(parsed, content) {
const issueStrings = issuesContent.match(/"([^"]+)"/g);
if (issueStrings) {
issues = issueStrings
.map(s => sanitizeIssue(s.replace(/"/g, '')))
.filter(issue => issue.length > 0);
.map((s) => sanitizeIssue(s.replace(/"/g, '')))
.filter((issue) => issue.length > 0);
}
}
// Same defensive logic: if there are issues, it's not valid
const finalIsValid = isValid && issues.length === 0 && !suggestion;
return { isValid: finalIsValid, suggestion, issues };
} catch {
// Default to valid if we can't parse anything
return { isValid: true, suggestion: null, issues: [] };
}
}
module.exports = {
buildNameUserPrompt,
parseNameResponse
};
inventory-server/src/services/ai/prompts/promptLoader.js
+24 -126
View File
@@ -1,35 +1,18 @@
/**
* Prompt Loader
*
* Utilities to load AI prompts from the ai_prompts PostgreSQL table.
* Supports loading prompts by base type (e.g., 'name_validation' loads
* name_validation_system, name_validation_general, and optionally
* name_validation_company_specific).
* Prompt Loader — loads AI prompts from the ai_prompts PostgreSQL table.
*/
/**
* Load a single prompt by exact type
* @param {Object} pool - PostgreSQL pool
* @param {string} promptType - Exact prompt type (e.g., 'name_validation_system')
* @param {string} [company] - Company identifier (for company_specific types)
* @returns {Promise<string|null>} Prompt text or null if not found
*/
async function loadPromptByType(pool, promptType, company = null) {
export async function loadPromptByType(pool, promptType, company = null) {
try {
let result;
if (company) {
result = await pool.query(
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company = $2',
[promptType, company]
);
} else {
result = await pool.query(
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company IS NULL',
[promptType]
);
}
const result = company
? await pool.query(
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company = $2',
[promptType, company]
)
: await pool.query(
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company IS NULL',
[promptType]
);
return result.rows[0]?.prompt_text || null;
} catch (error) {
console.error(`[PromptLoader] Error loading ${promptType} prompt:`, error.message);
@@ -37,93 +20,46 @@ async function loadPromptByType(pool, promptType, company = null) {
}
}
/**
* Load all prompts for a task type (system, general, and optionally company-specific)
*
* @param {Object} pool - PostgreSQL pool
* @param {string} baseType - Base type name (e.g., 'name_validation', 'description_validation')
* @param {string|null} [company] - Optional company ID for company-specific prompts
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
*/
async function loadPromptsByType(pool, baseType, company = null) {
export async function loadPromptsByType(pool, baseType, company = null) {
const systemType = `${baseType}_system`;
const generalType = `${baseType}_general`;
const companyType = `${baseType}_company_specific`;
// Load system and general prompts in parallel
const [system, general] = await Promise.all([
loadPromptByType(pool, systemType),
loadPromptByType(pool, generalType)
loadPromptByType(pool, generalType),
]);
// Load company-specific prompt if company is provided
let companySpecific = null;
if (company) {
companySpecific = await loadPromptByType(pool, companyType, company);
}
return {
system,
general,
companySpecific
};
return { system, general, companySpecific };
}
/**
* Load name validation prompts
* @param {Object} pool - PostgreSQL pool
* @param {string|null} [company] - Optional company ID
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
*/
async function loadNameValidationPrompts(pool, company = null) {
export function loadNameValidationPrompts(pool, company = null) {
return loadPromptsByType(pool, 'name_validation', company);
}
/**
* Load description validation prompts
* @param {Object} pool - PostgreSQL pool
* @param {string|null} [company] - Optional company ID
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
*/
async function loadDescriptionValidationPrompts(pool, company = null) {
export function loadDescriptionValidationPrompts(pool, company = null) {
return loadPromptsByType(pool, 'description_validation', company);
}
/**
* Load sanity check prompts (no company-specific variant)
* @param {Object} pool - PostgreSQL pool
* @returns {Promise<{system: string|null, general: string|null, companySpecific: null}>}
*/
async function loadSanityCheckPrompts(pool) {
export function loadSanityCheckPrompts(pool) {
return loadPromptsByType(pool, 'sanity_check', null);
}
/**
* Load bulk validation prompts (GPT-5 validation)
* @param {Object} pool - PostgreSQL pool
* @param {string|null} [company] - Optional company ID
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
*/
async function loadBulkValidationPrompts(pool, company = null) {
export function loadBulkValidationPrompts(pool, company = null) {
return loadPromptsByType(pool, 'bulk_validation', company);
}
/**
* Load bulk validation prompts for multiple companies at once
* @param {Object} pool - PostgreSQL pool
* @param {string[]} companyIds - Array of company IDs
* @returns {Promise<{system: string|null, general: string|null, companyPrompts: Map<string, string>}>}
*/
async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
// Load system and general prompts
export async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
const [system, general] = await Promise.all([
loadPromptByType(pool, 'bulk_validation_system'),
loadPromptByType(pool, 'bulk_validation_general')
loadPromptByType(pool, 'bulk_validation_general'),
]);
// Load company-specific prompts for all provided companies
const companyPrompts = new Map();
if (companyIds.length > 0) {
try {
const result = await pool.query(
@@ -132,7 +68,6 @@ async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
AND company = ANY($1)`,
[companyIds]
);
for (const row of result.rows) {
companyPrompts.set(row.company, row.prompt_text);
}
@@ -140,35 +75,14 @@ async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
console.error('[PromptLoader] Error loading company-specific prompts:', error.message);
}
}
return {
system,
general,
companyPrompts
};
return { system, general, companyPrompts };
}
/**
* Validate that required prompts exist, throw error if missing
* @param {Object} prompts - Prompts object from loadPromptsByType
* @param {string} baseType - Base type for error messages
* @param {Object} options - Validation options
* @param {boolean} [options.requireSystem=true] - Require system prompt
* @param {boolean} [options.requireGeneral=true] - Require general prompt
* @throws {Error} If required prompts are missing
*/
function validateRequiredPrompts(prompts, baseType, options = {}) {
export function validateRequiredPrompts(prompts, baseType, options = {}) {
const { requireSystem = true, requireGeneral = true } = options;
const missing = [];
if (requireSystem && !prompts.system) {
missing.push(`${baseType}_system`);
}
if (requireGeneral && !prompts.general) {
missing.push(`${baseType}_general`);
}
if (requireSystem && !prompts.system) missing.push(`${baseType}_system`);
if (requireGeneral && !prompts.general) missing.push(`${baseType}_general`);
if (missing.length > 0) {
throw new Error(
`Missing required AI prompts: ${missing.join(', ')}. ` +
@@ -176,19 +90,3 @@ function validateRequiredPrompts(prompts, baseType, options = {}) {
);
}
}
module.exports = {
// Core loader
loadPromptByType,
loadPromptsByType,
// Task-specific loaders
loadNameValidationPrompts,
loadDescriptionValidationPrompts,
loadSanityCheckPrompts,
loadBulkValidationPrompts,
loadBulkValidationPromptsForCompanies,
// Validation
validateRequiredPrompts
};
inventory-server/src/services/ai/prompts/sanityCheckPrompts.js
+13 -51
View File
@@ -1,21 +1,8 @@
/**
* Sanity Check Prompts
*
* Functions for building and parsing batch product consistency validation prompts.
* System and general prompts are loaded from the database.
*/
/**
* Build the user prompt for sanity check
* Combines database prompts with product data
*
* @param {Object[]} products - Array of product data (limited fields for context)
* @param {Object} prompts - Prompts loaded from database
* @param {string} prompts.general - General sanity check rules
* @returns {string} Complete user prompt
*/
function buildSanityCheckUserPrompt(products, prompts) {
// Build a simplified product list for the prompt
export function buildSanityCheckUserPrompt(products, prompts) {
const productSummaries = products.map((p, index) => ({
index,
name: p.name,
@@ -33,22 +20,17 @@ function buildSanityCheckUserPrompt(products, prompts) {
weight: p.weight,
length: p.length,
width: p.width,
height: p.height
height: p.height,
}));
const parts = [];
// Add general prompt/rules if provided
if (prompts.general) {
parts.push(prompts.general);
parts.push(''); // Empty line for separation
parts.push('');
}
// Add products to review
parts.push(`PRODUCTS TO REVIEW (${products.length} items):`);
parts.push(JSON.stringify(productSummaries, null, 2));
// Add response format
parts.push('');
parts.push('RESPOND WITH JSON:');
parts.push(JSON.stringify({
@@ -57,10 +39,10 @@ function buildSanityCheckUserPrompt(products, prompts) {
productIndex: 0,
field: 'msrp',
issue: 'Description of the issue found',
suggestion: 'Suggested fix or verification (optional)'
}
suggestion: 'Suggested fix or verification (optional)',
},
],
summary: '2-3 sentences summarizing the overall product quality'
summary: '2-3 sentences summarizing the overall product quality',
}, null, 2));
parts.push('');
@@ -69,60 +51,40 @@ function buildSanityCheckUserPrompt(products, prompts) {
return parts.join('\n');
}
/**
* Parse the AI response for sanity check
*
* @param {Object|null} parsed - Parsed JSON from AI
* @param {string} content - Raw response content
* @returns {Object}
*/
function parseSanityCheckResponse(parsed, content) {
// If we got valid parsed JSON, use it
export function parseSanityCheckResponse(parsed, content) {
if (parsed && Array.isArray(parsed.issues)) {
return {
issues: parsed.issues.map(issue => ({
issues: parsed.issues.map((issue) => ({
productIndex: issue.productIndex ?? issue.index ?? 0,
field: issue.field || 'unknown',
issue: issue.issue || issue.message || '',
suggestion: issue.suggestion || null
suggestion: issue.suggestion || null,
})),
summary: parsed.summary || 'Review complete'
summary: parsed.summary || 'Review complete',
};
}
// Try to extract from content if parsing failed
try {
// Try to find issues array
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
let issues = [];
if (issuesMatch) {
// Try to parse the array content
try {
const arrayContent = `[${issuesMatch[1]}]`;
const parsedIssues = JSON.parse(arrayContent);
issues = parsedIssues.map(issue => ({
issues = parsedIssues.map((issue) => ({
productIndex: issue.productIndex ?? issue.index ?? 0,
field: issue.field || 'unknown',
issue: issue.issue || issue.message || '',
suggestion: issue.suggestion || null
suggestion: issue.suggestion || null,
}));
} catch {
// Couldn't parse the array
/* fall through */
}
}
// Try to find summary
const summaryMatch = content.match(/"summary"\s*:\s*"([^"]+)"/);
const summary = summaryMatch ? summaryMatch[1] : 'Review complete';
return { issues, summary };
} catch {
return { issues: [], summary: 'Could not parse review results' };
}
}
module.exports = {
buildSanityCheckUserPrompt,
parseSanityCheckResponse
};
inventory-server/src/services/ai/providers/groqProvider.js
+17 -76
View File
@@ -1,25 +1,15 @@
/**
* Groq Provider - Handles chat completions via Groq's OpenAI-compatible API
*
* Uses Groq's fast inference for real-time AI validation tasks.
* Supports models like openai/gpt-oss-120b (complex) and openai/gpt-oss-20b (simple).
* Groq Provider - chat completions via Groq's OpenAI-compatible API
*/
const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
export const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
// Default models
const MODELS = {
LARGE: 'openai/gpt-oss-120b', // For complex tasks (descriptions, sanity checks)
SMALL: 'openai/gpt-oss-20b' // For simple tasks (name validation)
export const MODELS = {
LARGE: 'openai/gpt-oss-120b',
SMALL: 'openai/gpt-oss-20b',
};
class GroqProvider {
/**
* @param {Object} options
* @param {string} options.apiKey - Groq API key
* @param {string} [options.baseUrl] - Override base URL
* @param {number} [options.timeoutMs=30000] - Default timeout
*/
export class GroqProvider {
constructor({ apiKey, baseUrl = GROQ_BASE_URL, timeoutMs = 30000 }) {
if (!apiKey) {
throw new Error('Groq API key is required');
@@ -29,41 +19,25 @@ class GroqProvider {
this.timeoutMs = timeoutMs;
}
/**
* Send a chat completion request
*
* @param {Object} params
* @param {Array<{role: string, content: string}>} params.messages - Conversation messages
* @param {string} [params.model] - Model to use (defaults to LARGE)
* @param {number} [params.temperature=0.3] - Response randomness (0-2)
* @param {number} [params.maxTokens=500] - Max tokens in response
* @param {Object} [params.responseFormat] - For JSON mode: { type: 'json_object' }
* @param {number} [params.timeoutMs] - Request timeout override
* @returns {Promise<{content: string, parsed: Object|null, usage: Object, latencyMs: number, model: string}>}
*/
async chatCompletion({
messages,
model = MODELS.LARGE,
temperature = 0.3,
maxTokens = 500,
responseFormat = null,
timeoutMs = this.timeoutMs
timeoutMs = this.timeoutMs,
}) {
const started = Date.now();
const body = {
model,
messages,
temperature,
max_completion_tokens: maxTokens
max_completion_tokens: maxTokens,
};
// Enable JSON mode if requested
if (responseFormat?.type === 'json_object') {
body.response_format = { type: 'json_object' };
}
// Debug: Log request being sent
console.log('[Groq] Request:', {
model: body.model,
temperature: body.temperature,
@@ -71,12 +45,11 @@ class GroqProvider {
hasResponseFormat: !!body.response_format,
messageCount: body.messages?.length,
systemPromptLength: body.messages?.[0]?.content?.length,
userPromptLength: body.messages?.[1]?.content?.length
userPromptLength: body.messages?.[1]?.content?.length,
});
const response = await this._makeRequest('chat/completions', body, timeoutMs);
// Debug: Log raw response structure
console.log('[Groq] Raw response:', {
hasChoices: !!response.choices,
choicesLength: response.choices?.length,
@@ -84,22 +57,20 @@ class GroqProvider {
finishReason: response.choices[0].finish_reason,
hasMessage: !!response.choices[0].message,
contentLength: response.choices[0].message?.content?.length,
contentPreview: response.choices[0].message?.content?.substring(0, 200)
contentPreview: response.choices[0].message?.content?.substring(0, 200),
} : null,
usage: response.usage,
model: response.model
model: response.model,
});
const content = response.choices?.[0]?.message?.content || '';
const usage = response.usage || {};
// Attempt to parse JSON if response format was requested
let parsed = null;
if (responseFormat && content) {
try {
parsed = JSON.parse(content);
} catch {
// Content isn't valid JSON - try to extract JSON from markdown
parsed = this._extractJson(content);
}
}
@@ -110,74 +81,50 @@ class GroqProvider {
usage: {
promptTokens: usage.prompt_tokens || 0,
completionTokens: usage.completion_tokens || 0,
totalTokens: usage.total_tokens || 0
totalTokens: usage.total_tokens || 0,
},
latencyMs: Date.now() - started,
model: response.model || model
model: response.model || model,
};
}
/**
* Extract JSON from content that might be wrapped in markdown code blocks
* @private
*/
_extractJson(content) {
// Try to find JSON in code blocks
const codeBlockMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
if (codeBlockMatch) {
try {
return JSON.parse(codeBlockMatch[1].trim());
} catch {
// Fall through
}
try { return JSON.parse(codeBlockMatch[1].trim()); } catch { /* fall through */ }
}
// Try to find JSON object/array directly
const jsonMatch = content.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
if (jsonMatch) {
try {
return JSON.parse(jsonMatch[1]);
} catch {
// Fall through
}
try { return JSON.parse(jsonMatch[1]); } catch { /* fall through */ }
}
return null;
}
/**
* Make an HTTP request to Groq API
* @private
*/
async _makeRequest(endpoint, body, timeoutMs) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
'Authorization': `Bearer ${this.apiKey}`,
},
body: JSON.stringify(body),
signal: controller.signal
signal: controller.signal,
});
if (!response.ok) {
const error = await response.json().catch(() => ({}));
const message = error.error?.message || `Groq API error: ${response.status}`;
const err = new Error(message);
err.status = response.status;
err.code = error.error?.code;
// Include failed_generation if available (for JSON mode failures)
if (error.error?.failed_generation) {
err.failedGeneration = error.error.failed_generation;
console.error('[Groq] JSON validation failed. Model output:', error.error.failed_generation);
}
throw err;
}
return response.json();
} catch (error) {
if (error.name === 'AbortError') {
@@ -191,13 +138,7 @@ class GroqProvider {
}
}
/**
* Check if the provider is properly configured
* @returns {boolean}
*/
isConfigured() {
return !!this.apiKey;
}
}
module.exports = { GroqProvider, MODELS, GROQ_BASE_URL };
inventory-server/src/services/ai/providers/openaiProvider.js
+12 -40
View File
@@ -2,11 +2,11 @@
* OpenAI Provider - Handles embedding generation
*/
const EMBEDDING_MODEL = 'text-embedding-3-small';
const EMBEDDING_DIMENSIONS = 1536;
export const EMBEDDING_MODEL = 'text-embedding-3-small';
export const EMBEDDING_DIMENSIONS = 1536;
const MAX_BATCH_SIZE = 2048;
class OpenAIProvider {
export class OpenAIProvider {
constructor({ apiKey, baseUrl = 'https://api.openai.com/v1', timeoutMs = 60000 }) {
if (!apiKey) {
throw new Error('OpenAI API key is required');
@@ -16,12 +16,6 @@ class OpenAIProvider {
this.timeoutMs = timeoutMs;
}
/**
* Generate embeddings for one or more texts
* @param {string|string[]} input - Text or array of texts
* @param {Object} options
* @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>}
*/
async embed(input, options = {}) {
const texts = Array.isArray(input) ? input : [input];
const model = options.model || EMBEDDING_MODEL;
@@ -33,56 +27,39 @@ class OpenAIProvider {
}
const started = Date.now();
// Clean and truncate input texts
const cleanedTexts = texts.map(t =>
const cleanedTexts = texts.map((t) =>
(t || '').replace(/\n+/g, ' ').trim().substring(0, 8000)
);
const body = {
input: cleanedTexts,
model,
encoding_format: 'float'
};
// Only embedding-3 models support dimensions parameter
if (model.includes('embedding-3')) {
body.dimensions = dimensions;
}
const body = { input: cleanedTexts, model, encoding_format: 'float' };
if (model.includes('embedding-3')) body.dimensions = dimensions;
const response = await this._makeRequest('embeddings', body, timeoutMs);
// Sort by index to ensure order matches input
const sortedData = response.data.sort((a, b) => a.index - b.index);
return {
embeddings: sortedData.map(item => item.embedding),
embeddings: sortedData.map((item) => item.embedding),
usage: {
promptTokens: response.usage?.prompt_tokens || 0,
totalTokens: response.usage?.total_tokens || 0
totalTokens: response.usage?.total_tokens || 0,
},
model: response.model || model,
latencyMs: Date.now() - started
latencyMs: Date.now() - started,
};
}
/**
* Generator for processing large batches in chunks
*/
async *embedBatchChunked(texts, options = {}) {
const batchSize = Math.min(options.batchSize || 100, MAX_BATCH_SIZE);
for (let i = 0; i < texts.length; i += batchSize) {
const chunk = texts.slice(i, i + batchSize);
const result = await this.embed(chunk, options);
yield {
embeddings: result.embeddings,
startIndex: i,
endIndex: i + chunk.length,
usage: result.usage,
model: result.model,
latencyMs: result.latencyMs
latencyMs: result.latencyMs,
};
}
}
@@ -90,28 +67,23 @@ class OpenAIProvider {
async _makeRequest(endpoint, body, timeoutMs) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
'Authorization': `Bearer ${this.apiKey}`,
},
body: JSON.stringify(body),
signal: controller.signal
signal: controller.signal,
});
if (!response.ok) {
const error = await response.json().catch(() => ({}));
throw new Error(error.error?.message || `OpenAI API error: ${response.status}`);
}
return response.json();
} finally {
clearTimeout(timeout);
}
}
}
module.exports = { OpenAIProvider, EMBEDDING_MODEL, EMBEDDING_DIMENSIONS };
inventory-server/src/services/ai/tasks/descriptionValidationTask.js
+25 -78
View File
@@ -1,132 +1,84 @@
/**
* Description Validation Task
*
* Validates a product description for quality, accuracy, and guideline compliance.
* Uses Groq with the larger model for better reasoning about content quality.
* Loads all prompts from the database (no hardcoded prompts).
*/
const { MODELS } = require('../providers/groqProvider');
const {
import { MODELS } from '../providers/groqProvider.js';
import {
loadDescriptionValidationPrompts,
validateRequiredPrompts
} = require('../prompts/promptLoader');
const {
validateRequiredPrompts,
} from '../prompts/promptLoader.js';
import {
buildDescriptionUserPrompt,
parseDescriptionResponse
} = require('../prompts/descriptionPrompts');
parseDescriptionResponse,
} from '../prompts/descriptionPrompts.js';
const TASK_ID = 'validate.description';
export const TASK_ID = 'validate.description';
/**
* Create the description validation task
*
* @returns {Object} Task definition
*/
function createDescriptionValidationTask() {
export function createDescriptionValidationTask() {
return {
id: TASK_ID,
description: 'Validate product description for quality and guideline compliance',
/**
* Run the description validation
*
* @param {Object} payload
* @param {Object} payload.product - Product data
* @param {string} payload.product.name - Product name (for context)
* @param {string} payload.product.description - Description to validate
* @param {string} [payload.product.company_name] - Company name
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
* @param {string} [payload.product.categories] - Product categories
* @param {Object} payload.provider - Groq provider instance
* @param {Object} payload.pool - PostgreSQL pool
* @param {Object} [payload.logger] - Logger instance
* @returns {Promise<Object>}
*/
async run(payload) {
const { product, provider, pool, logger } = payload;
const log = logger || console;
// Validate required input
if (!product?.name && !product?.description) {
return {
isValid: true,
suggestion: null,
issues: [],
skipped: true,
reason: 'No name or description provided'
};
}
if (!provider) {
throw new Error('Groq provider not available');
}
if (!pool) {
throw new Error('Database pool not available');
return { isValid: true, suggestion: null, issues: [], skipped: true, reason: 'No name or description provided' };
}
if (!provider) throw new Error('Groq provider not available');
if (!pool) throw new Error('Database pool not available');
try {
// Load prompts from database
const companyKey = product.company_id || product.company_name || product.company;
const prompts = await loadDescriptionValidationPrompts(pool, companyKey);
// Validate required prompts exist
validateRequiredPrompts(prompts, 'description_validation');
// Build the user prompt with database-loaded prompts
const userPrompt = buildDescriptionUserPrompt(product, prompts);
let response;
let result;
try {
// Try with JSON mode first
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.LARGE, // openai/gpt-oss-120b - better for content analysis
temperature: 0.3, // Slightly higher for creative suggestions
maxTokens: 2000, // Reasoning models need extra tokens for thinking
responseFormat: { type: 'json_object' }
model: MODELS.LARGE,
temperature: 0.3,
maxTokens: 2000,
responseFormat: { type: 'json_object' },
});
// Log full raw response for debugging
log.info('[DescriptionValidation] Raw AI response:', {
parsed: response.parsed,
content: response.content,
contentLength: response.content?.length
contentLength: response.content?.length,
});
// Parse the response
result = parseDescriptionResponse(response.parsed, response.content);
} catch (jsonError) {
// If JSON mode failed, check if we have failedGeneration to parse
if (jsonError.failedGeneration) {
log.warn('[DescriptionValidation] JSON mode failed, attempting to parse failed_generation:', {
failedGeneration: jsonError.failedGeneration
failedGeneration: jsonError.failedGeneration,
});
result = parseDescriptionResponse(null, jsonError.failedGeneration);
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
} else {
// Retry without JSON mode
log.warn('[DescriptionValidation] JSON mode failed, retrying without JSON mode');
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.LARGE,
temperature: 0.3,
maxTokens: 2000 // Reasoning models need extra tokens for thinking
// No responseFormat - let the model respond freely
maxTokens: 2000,
});
log.info('[DescriptionValidation] Raw AI response (no JSON mode):', {
parsed: response.parsed,
content: response.content,
contentLength: response.content?.length
contentLength: response.content?.length,
});
result = parseDescriptionResponse(response.parsed, response.content);
}
@@ -135,24 +87,19 @@ function createDescriptionValidationTask() {
log.info(`[DescriptionValidation] Validated description for "${product.name}" in ${response.latencyMs}ms`, {
isValid: result.isValid,
hasSuggestion: !!result.suggestion,
issueCount: result.issues.length
issueCount: result.issues.length,
});
return {
...result,
latencyMs: response.latencyMs,
usage: response.usage,
model: response.model
model: response.model,
};
} catch (error) {
log.error('[DescriptionValidation] Error:', error.message);
throw error;
}
}
},
};
}
module.exports = {
TASK_ID,
createDescriptionValidationTask
};
inventory-server/src/services/ai/tasks/index.js
+18 -113
View File
@@ -1,166 +1,87 @@
/**
* AI Task Registry
*
* Simple registry pattern for AI tasks. Each task has:
* - id: Unique identifier
* - run: Async function that executes the task
*
* This allows adding new AI capabilities without modifying core code.
*/
const { createNameValidationTask, TASK_ID: NAME_TASK_ID } = require('./nameValidationTask');
const { createDescriptionValidationTask, TASK_ID: DESC_TASK_ID } = require('./descriptionValidationTask');
const { createSanityCheckTask, TASK_ID: SANITY_TASK_ID } = require('./sanityCheckTask');
import { createNameValidationTask, TASK_ID as NAME_TASK_ID } from './nameValidationTask.js';
import { createDescriptionValidationTask, TASK_ID as DESC_TASK_ID } from './descriptionValidationTask.js';
import { createSanityCheckTask, TASK_ID as SANITY_TASK_ID } from './sanityCheckTask.js';
/**
* Task IDs - frozen constants for type safety
*/
const TASK_IDS = Object.freeze({
// Inline validation (triggered on field blur)
export { createNameValidationTask, createDescriptionValidationTask, createSanityCheckTask };
export const TASK_IDS = Object.freeze({
VALIDATE_NAME: NAME_TASK_ID,
VALIDATE_DESCRIPTION: DESC_TASK_ID,
// Batch operations (triggered on user action)
SANITY_CHECK: SANITY_TASK_ID
SANITY_CHECK: SANITY_TASK_ID,
});
/**
* Task Registry
*/
class TaskRegistry {
export class TaskRegistry {
constructor() {
this.tasks = new Map();
}
/**
* Register a task
* @param {Object} task
* @param {string} task.id - Unique task identifier
* @param {Function} task.run - Async function: (payload) => result
* @param {string} [task.description] - Human-readable description
*/
register(task) {
if (!task?.id) {
throw new Error('Task must have an id');
}
if (typeof task.run !== 'function') {
throw new Error(`Task ${task.id} must have a run function`);
}
if (this.tasks.has(task.id)) {
throw new Error(`Task ${task.id} is already registered`);
}
if (!task?.id) throw new Error('Task must have an id');
if (typeof task.run !== 'function') throw new Error(`Task ${task.id} must have a run function`);
if (this.tasks.has(task.id)) throw new Error(`Task ${task.id} is already registered`);
this.tasks.set(task.id, task);
return this;
}
/**
* Get a task by ID
* @param {string} taskId
* @returns {Object|null}
*/
get(taskId) {
return this.tasks.get(taskId) || null;
}
/**
* Check if a task exists
* @param {string} taskId
* @returns {boolean}
*/
has(taskId) {
return this.tasks.has(taskId);
}
/**
* Run a task by ID
* @param {string} taskId
* @param {Object} payload - Task-specific input
* @returns {Promise<Object>} Task result
*/
async runTask(taskId, payload = {}) {
const task = this.get(taskId);
if (!task) {
throw new Error(`Unknown task: ${taskId}`);
}
if (!task) throw new Error(`Unknown task: ${taskId}`);
try {
const result = await task.run(payload);
return {
success: true,
taskId,
...result
};
return { success: true, taskId, ...result };
} catch (error) {
return {
success: false,
taskId,
error: error.message,
code: error.code
code: error.code,
};
}
}
/**
* List all registered task IDs
* @returns {string[]}
*/
list() {
return Array.from(this.tasks.keys());
}
/**
* Get count of registered tasks
* @returns {number}
*/
size() {
return this.tasks.size;
}
}
// Singleton instance
let registry = null;
/**
* Get or create the task registry
* @returns {TaskRegistry}
*/
function getRegistry() {
if (!registry) {
registry = new TaskRegistry();
}
export function getRegistry() {
if (!registry) registry = new TaskRegistry();
return registry;
}
/**
* Reset the registry (mainly for testing)
*/
function resetRegistry() {
export function resetRegistry() {
registry = null;
}
/**
* Register all validation tasks with the registry
* Call this during initialization after the registry is created
*
* @param {Object} [logger] - Optional logger
*/
function registerAllTasks(logger = console) {
export function registerAllTasks(logger = console) {
const reg = getRegistry();
// Register name validation
if (!reg.has(TASK_IDS.VALIDATE_NAME)) {
reg.register(createNameValidationTask());
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_NAME}`);
}
// Register description validation
if (!reg.has(TASK_IDS.VALIDATE_DESCRIPTION)) {
reg.register(createDescriptionValidationTask());
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_DESCRIPTION}`);
}
// Register sanity check
if (!reg.has(TASK_IDS.SANITY_CHECK)) {
reg.register(createSanityCheckTask());
logger.info(`[Tasks] Registered: ${TASK_IDS.SANITY_CHECK}`);
@@ -168,19 +89,3 @@ function registerAllTasks(logger = console) {
return reg;
}
module.exports = {
// Constants
TASK_IDS,
// Registry
TaskRegistry,
getRegistry,
resetRegistry,
registerAllTasks,
// Task factories (for custom registration)
createNameValidationTask,
createDescriptionValidationTask,
createSanityCheckTask
};
inventory-server/src/services/ai/tasks/nameValidationTask.js
+26 -81
View File
@@ -1,77 +1,38 @@
/**
* Name Validation Task
*
* Validates a product name for spelling, grammar, and naming conventions.
* Uses Groq with the smaller model for fast response times.
* Loads all prompts from the database (no hardcoded prompts).
*/
const { MODELS } = require('../providers/groqProvider');
const {
import { MODELS } from '../providers/groqProvider.js';
import {
loadNameValidationPrompts,
validateRequiredPrompts
} = require('../prompts/promptLoader');
const {
validateRequiredPrompts,
} from '../prompts/promptLoader.js';
import {
buildNameUserPrompt,
parseNameResponse
} = require('../prompts/namePrompts');
parseNameResponse,
} from '../prompts/namePrompts.js';
const TASK_ID = 'validate.name';
export const TASK_ID = 'validate.name';
/**
* Create the name validation task
*
* @returns {Object} Task definition
*/
function createNameValidationTask() {
export function createNameValidationTask() {
return {
id: TASK_ID,
description: 'Validate product name for spelling, grammar, and conventions',
/**
* Run the name validation
*
* @param {Object} payload
* @param {Object} payload.product - Product data
* @param {string} payload.product.name - Product name to validate
* @param {string} [payload.product.company_name] - Company name
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
* @param {string} [payload.product.line_name] - Product line
* @param {string} [payload.product.description] - Description for context
* @param {Object} payload.provider - Groq provider instance
* @param {Object} payload.pool - PostgreSQL pool
* @param {Object} [payload.logger] - Logger instance
* @returns {Promise<Object>}
*/
async run(payload) {
const { product, provider, pool, logger } = payload;
const log = logger || console;
// Validate required input
if (!product?.name) {
return {
isValid: true,
suggestion: null,
issues: [],
skipped: true,
reason: 'No name provided'
};
}
if (!provider) {
throw new Error('Groq provider not available');
}
if (!pool) {
throw new Error('Database pool not available');
return { isValid: true, suggestion: null, issues: [], skipped: true, reason: 'No name provided' };
}
if (!provider) throw new Error('Groq provider not available');
if (!pool) throw new Error('Database pool not available');
try {
// Load prompts from database
const companyKey = product.company_id || product.company_name || product.company;
const prompts = await loadNameValidationPrompts(pool, companyKey);
// Debug: Log loaded prompts
log.info('[NameValidation] Loaded prompts:', {
hasSystem: !!prompts.system,
systemLength: prompts.system?.length || 0,
@@ -79,68 +40,57 @@ function createNameValidationTask() {
generalLength: prompts.general?.length || 0,
generalPreview: prompts.general?.substring(0, 100) || '(empty)',
hasCompanySpecific: !!prompts.companySpecific,
companyKey
companyKey,
});
// Validate required prompts exist
validateRequiredPrompts(prompts, 'name_validation');
// Build the user prompt with database-loaded prompts
const userPrompt = buildNameUserPrompt(product, prompts);
// Debug: Log the full user prompt being sent
log.info('[NameValidation] User prompt:', userPrompt.substring(0, 500));
let response;
let result;
try {
// Try with JSON mode first
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.LARGE, // openai/gpt-oss-120b - reasoning model
temperature: 0.2, // Low temperature for consistent results
maxTokens: 3000, // Reasoning models need extra tokens for thinking
responseFormat: { type: 'json_object' }
model: MODELS.LARGE,
temperature: 0.2,
maxTokens: 3000,
responseFormat: { type: 'json_object' },
});
// Log full raw response for debugging
log.info('[NameValidation] Raw AI response:', {
parsed: response.parsed,
content: response.content,
contentLength: response.content?.length
contentLength: response.content?.length,
});
// Parse the response
result = parseNameResponse(response.parsed, response.content);
} catch (jsonError) {
// If JSON mode failed, check if we have failedGeneration to parse
if (jsonError.failedGeneration) {
log.warn('[NameValidation] JSON mode failed, attempting to parse failed_generation:', {
failedGeneration: jsonError.failedGeneration
failedGeneration: jsonError.failedGeneration,
});
result = parseNameResponse(null, jsonError.failedGeneration);
response = { latencyMs: 0, usage: {}, model: MODELS.SMALL };
} else {
// Retry without JSON mode
log.warn('[NameValidation] JSON mode failed, retrying without JSON mode');
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.SMALL,
temperature: 0.2,
maxTokens: 1500 // Reasoning models need extra tokens for thinking
// No responseFormat - let the model respond freely
maxTokens: 1500,
});
log.info('[NameValidation] Raw AI response (no JSON mode):', {
parsed: response.parsed,
content: response.content,
contentLength: response.content?.length
contentLength: response.content?.length,
});
result = parseNameResponse(response.parsed, response.content);
}
@@ -149,24 +99,19 @@ function createNameValidationTask() {
log.info(`[NameValidation] Validated "${product.name}" in ${response.latencyMs}ms`, {
isValid: result.isValid,
hassuggestion: !!result.suggestion,
issueCount: result.issues.length
issueCount: result.issues.length,
});
return {
...result,
latencyMs: response.latencyMs,
usage: response.usage,
model: response.model
model: response.model,
};
} catch (error) {
log.error('[NameValidation] Error:', error.message);
throw error;
}
}
},
};
}
module.exports = {
TASK_ID,
createNameValidationTask
};
inventory-server/src/services/ai/tasks/sanityCheckTask.js
+27 -89
View File
@@ -1,96 +1,55 @@
/**
* Sanity Check Task
*
* Reviews a batch of products for consistency and appropriateness.
* Uses Groq with the larger model for complex batch analysis.
* Loads all prompts from the database (no hardcoded prompts).
*/
const { MODELS } = require('../providers/groqProvider');
const {
import { MODELS } from '../providers/groqProvider.js';
import {
loadSanityCheckPrompts,
validateRequiredPrompts
} = require('../prompts/promptLoader');
const {
validateRequiredPrompts,
} from '../prompts/promptLoader.js';
import {
buildSanityCheckUserPrompt,
parseSanityCheckResponse
} = require('../prompts/sanityCheckPrompts');
parseSanityCheckResponse,
} from '../prompts/sanityCheckPrompts.js';
const TASK_ID = 'sanity.check';
export const TASK_ID = 'sanity.check';
export const MAX_PRODUCTS_PER_REQUEST = 50;
// Maximum products to send in a single request (to avoid token limits)
const MAX_PRODUCTS_PER_REQUEST = 50;
/**
* Create the sanity check task
*
* @returns {Object} Task definition
*/
function createSanityCheckTask() {
export function createSanityCheckTask() {
return {
id: TASK_ID,
description: 'Review batch of products for consistency and appropriateness',
/**
* Run the sanity check
*
* @param {Object} payload
* @param {Object[]} payload.products - Array of products to check
* @param {Object} payload.provider - Groq provider instance
* @param {Object} payload.pool - PostgreSQL pool
* @param {Object} [payload.logger] - Logger instance
* @returns {Promise<Object>}
*/
async run(payload) {
const { products, provider, pool, logger } = payload;
const log = logger || console;
// Validate required input
if (!Array.isArray(products) || products.length === 0) {
return {
issues: [],
summary: 'No products to check',
skipped: true
};
}
if (!provider) {
throw new Error('Groq provider not available');
}
if (!pool) {
throw new Error('Database pool not available');
return { issues: [], summary: 'No products to check', skipped: true };
}
if (!provider) throw new Error('Groq provider not available');
if (!pool) throw new Error('Database pool not available');
try {
// Load prompts from database
const prompts = await loadSanityCheckPrompts(pool);
// Validate required prompts exist
validateRequiredPrompts(prompts, 'sanity_check');
// If batch is small enough, process in one request
if (products.length <= MAX_PRODUCTS_PER_REQUEST) {
return await checkBatch(products, prompts, provider, log);
}
// Otherwise, process in chunks and combine results
log.info(`[SanityCheck] Processing ${products.length} products in chunks`);
const allIssues = [];
const summaries = [];
for (let i = 0; i < products.length; i += MAX_PRODUCTS_PER_REQUEST) {
const chunk = products.slice(i, i + MAX_PRODUCTS_PER_REQUEST);
const chunkOffset = i; // To adjust product indices in results
const chunkOffset = i;
const result = await checkBatch(chunk, prompts, provider, log);
// Adjust product indices to match original array
const adjustedIssues = result.issues.map(issue => ({
const adjustedIssues = result.issues.map((issue) => ({
...issue,
productIndex: issue.productIndex + chunkOffset
productIndex: issue.productIndex + chunkOffset,
}));
allIssues.push(...adjustedIssues);
summaries.push(result.summary);
}
@@ -101,82 +60,61 @@ function createSanityCheckTask() {
? `Reviewed ${products.length} products in ${summaries.length} batches. ${allIssues.length} issues found.`
: summaries[0],
totalProducts: products.length,
issueCount: allIssues.length
issueCount: allIssues.length,
};
} catch (error) {
log.error('[SanityCheck] Error:', error.message);
throw error;
}
}
},
};
}
/**
* Check a single batch of products
*
* @param {Object[]} products - Products to check
* @param {Object} prompts - Loaded prompts from database
* @param {Object} provider - Groq provider
* @param {Object} log - Logger
* @returns {Promise<Object>}
*/
async function checkBatch(products, prompts, provider, log) {
const userPrompt = buildSanityCheckUserPrompt(products, prompts);
let response;
let result;
try {
// Try with JSON mode first
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.LARGE, // openai/gpt-oss-120b - needed for complex batch analysis
temperature: 0.2, // Low temperature for consistent analysis
maxTokens: 2000, // More tokens for batch results
responseFormat: { type: 'json_object' }
model: MODELS.LARGE,
temperature: 0.2,
maxTokens: 2000,
responseFormat: { type: 'json_object' },
});
result = parseSanityCheckResponse(response.parsed, response.content);
} catch (jsonError) {
// If JSON mode failed, check if we have failedGeneration to parse
if (jsonError.failedGeneration) {
log.warn('[SanityCheck] JSON mode failed, attempting to parse failed_generation');
result = parseSanityCheckResponse(null, jsonError.failedGeneration);
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
} else {
// Retry without JSON mode
log.warn('[SanityCheck] JSON mode failed, retrying without JSON mode');
response = await provider.chatCompletion({
messages: [
{ role: 'system', content: prompts.system },
{ role: 'user', content: userPrompt }
{ role: 'user', content: userPrompt },
],
model: MODELS.LARGE,
temperature: 0.2,
maxTokens: 2000
// No responseFormat - let the model respond freely
maxTokens: 2000,
});
result = parseSanityCheckResponse(response.parsed, response.content);
}
}
log.info(`[SanityCheck] Checked ${products.length} products in ${response.latencyMs}ms`, {
issueCount: result.issues.length
issueCount: result.issues.length,
});
return {
...result,
latencyMs: response.latencyMs,
usage: response.usage,
model: response.model
model: response.model,
};
}
module.exports = {
TASK_ID,
createSanityCheckTask,
MAX_PRODUCTS_PER_REQUEST
};
inventory-server/src/types/status-codes.js
+42 -58
View File
@@ -1,79 +1,63 @@
// Purchase Order Status Codes
const PurchaseOrderStatus = {
Canceled: 0,
Created: 1,
ElectronicallyReadySend: 10,
Ordered: 11,
Preordered: 12,
ElectronicallySent: 13,
ReceivingStarted: 15,
Done: 50
export const PurchaseOrderStatus = {
Canceled: 0,
Created: 1,
ElectronicallyReadySend: 10,
Ordered: 11,
Preordered: 12,
ElectronicallySent: 13,
ReceivingStarted: 15,
Done: 50,
};
// Receiving Status Codes
const ReceivingStatus = {
Canceled: 0,
Created: 1,
PartialReceived: 30,
FullReceived: 40,
Paid: 50
export const ReceivingStatus = {
Canceled: 0,
Created: 1,
PartialReceived: 30,
FullReceived: 40,
Paid: 50,
};
// Status Code Display Names
const PurchaseOrderStatusLabels = {
[PurchaseOrderStatus.Canceled]: 'Canceled',
[PurchaseOrderStatus.Created]: 'Created',
[PurchaseOrderStatus.ElectronicallyReadySend]: 'Ready to Send',
[PurchaseOrderStatus.Ordered]: 'Ordered',
[PurchaseOrderStatus.Preordered]: 'Preordered',
[PurchaseOrderStatus.ElectronicallySent]: 'Sent',
[PurchaseOrderStatus.ReceivingStarted]: 'Receiving Started',
[PurchaseOrderStatus.Done]: 'Done'
export const PurchaseOrderStatusLabels = {
[PurchaseOrderStatus.Canceled]: 'Canceled',
[PurchaseOrderStatus.Created]: 'Created',
[PurchaseOrderStatus.ElectronicallyReadySend]: 'Ready to Send',
[PurchaseOrderStatus.Ordered]: 'Ordered',
[PurchaseOrderStatus.Preordered]: 'Preordered',
[PurchaseOrderStatus.ElectronicallySent]: 'Sent',
[PurchaseOrderStatus.ReceivingStarted]: 'Receiving Started',
[PurchaseOrderStatus.Done]: 'Done',
};
const ReceivingStatusLabels = {
[ReceivingStatus.Canceled]: 'Canceled',
[ReceivingStatus.Created]: 'Created',
[ReceivingStatus.PartialReceived]: 'Partially Received',
[ReceivingStatus.FullReceived]: 'Fully Received',
[ReceivingStatus.Paid]: 'Paid'
export const ReceivingStatusLabels = {
[ReceivingStatus.Canceled]: 'Canceled',
[ReceivingStatus.Created]: 'Created',
[ReceivingStatus.PartialReceived]: 'Partially Received',
[ReceivingStatus.FullReceived]: 'Fully Received',
[ReceivingStatus.Paid]: 'Paid',
};
// Helper functions
function getPurchaseOrderStatusLabel(status) {
return PurchaseOrderStatusLabels[status] || 'Unknown';
export function getPurchaseOrderStatusLabel(status) {
return PurchaseOrderStatusLabels[status] || 'Unknown';
}
function getReceivingStatusLabel(status) {
return ReceivingStatusLabels[status] || 'Unknown';
export function getReceivingStatusLabel(status) {
return ReceivingStatusLabels[status] || 'Unknown';
}
// Status checks
function isReceivingComplete(status) {
return status >= ReceivingStatus.PartialReceived;
export function isReceivingComplete(status) {
return status >= ReceivingStatus.PartialReceived;
}
function isPurchaseOrderComplete(status) {
return status === PurchaseOrderStatus.Done;
export function isPurchaseOrderComplete(status) {
return status === PurchaseOrderStatus.Done;
}
function isPurchaseOrderCanceled(status) {
return status === PurchaseOrderStatus.Canceled;
export function isPurchaseOrderCanceled(status) {
return status === PurchaseOrderStatus.Canceled;
}
function isReceivingCanceled(status) {
return status === ReceivingStatus.Canceled;
export function isReceivingCanceled(status) {
return status === ReceivingStatus.Canceled;
}
module.exports = {
PurchaseOrderStatus,
ReceivingStatus,
PurchaseOrderStatusLabels,
ReceivingStatusLabels,
getPurchaseOrderStatusLabel,
getReceivingStatusLabel,
isReceivingComplete,
isPurchaseOrderComplete,
isPurchaseOrderCanceled,
isReceivingCanceled
};
inventory-server/src/utils/apiHelpers.js
+23 -39
View File
@@ -1,45 +1,29 @@
/**
* Parses a query parameter value based on its expected type.
* Throws error for invalid formats. Adjust date handling as needed.
* Throws on invalid formats.
*/
function parseValue(value, type) {
if (value === null || value === undefined || value === '') return null;
export function parseValue(value, type) {
if (value === null || value === undefined || value === '') return null;
console.log(`Parsing value: "${value}" as type: "${type}"`);
switch (type) {
case 'number':
const num = parseFloat(value);
if (isNaN(num)) {
console.error(`Invalid number format: "${value}"`);
throw new Error(`Invalid number format: "${value}"`);
}
return num;
case 'integer': // Specific type for integer IDs etc.
const int = parseInt(value, 10);
if (isNaN(int)) {
console.error(`Invalid integer format: "${value}"`);
throw new Error(`Invalid integer format: "${value}"`);
}
console.log(`Successfully parsed integer: ${int}`);
return int;
case 'boolean':
if (String(value).toLowerCase() === 'true') return true;
if (String(value).toLowerCase() === 'false') return false;
console.error(`Invalid boolean format: "${value}"`);
throw new Error(`Invalid boolean format: "${value}"`);
case 'date':
// Basic ISO date format validation (YYYY-MM-DD)
if (!String(value).match(/^\d{4}-\d{2}-\d{2}$/)) {
console.warn(`Potentially invalid date format passed: "${value}"`);
// Optionally throw an error or return null depending on strictness
// throw new Error(`Invalid date format (YYYY-MM-DD expected): "${value}"`);
}
return String(value); // Send as string, let DB handle casting/comparison
case 'string':
default:
return String(value);
switch (type) {
case 'number': {
const num = parseFloat(value);
if (Number.isNaN(num)) throw new Error(`Invalid number format: "${value}"`);
return num;
}
case 'integer': {
const int = parseInt(value, 10);
if (Number.isNaN(int)) throw new Error(`Invalid integer format: "${value}"`);
return int;
}
case 'boolean':
if (String(value).toLowerCase() === 'true') return true;
if (String(value).toLowerCase() === 'false') return false;
throw new Error(`Invalid boolean format: "${value}"`);
case 'date':
return String(value);
case 'string':
default:
return String(value);
}
}
module.exports = { parseValue };
inventory-server/src/utils/csvImporter.js
+9 -23
View File
@@ -1,45 +1,37 @@
const fs = require('fs');
const { parse } = require('csv-parse');
const { v4: uuidv4 } = require('uuid');
import fs from 'node:fs';
import { parse } from 'csv-parse';
import { v4 as uuidv4 } from 'uuid';
async function importProductsFromCSV(filePath, pool) {
export async function importProductsFromCSV(filePath, pool) {
return new Promise((resolve, reject) => {
const products = [];
fs.createReadStream(filePath)
.pipe(parse({
columns: true,
skip_empty_lines: true
}))
.on('data', async (row) => {
.pipe(parse({ columns: true, skip_empty_lines: true }))
.on('data', (row) => {
products.push({
id: uuidv4(),
sku: row.sku,
name: row.name,
description: row.description || null,
category: row.category || null
category: row.category || null,
});
})
.on('end', async () => {
try {
const connection = await pool.getConnection();
try {
await connection.beginTransaction();
for (const product of products) {
await connection.query(
'INSERT INTO products (id, sku, name, description, category) VALUES (?, ?, ?, ?, ?)',
[product.id, product.sku, product.name, product.description, product.category]
);
// Initialize inventory level for the product
await connection.query(
'INSERT INTO inventory_levels (id, product_id, quantity) VALUES (?, ?, 0)',
[uuidv4(), product.id]
);
}
await connection.commit();
resolve({ imported: products.length });
} catch (error) {
@@ -52,12 +44,6 @@ async function importProductsFromCSV(filePath, pool) {
reject(error);
}
})
.on('error', (error) => {
reject(error);
});
.on('error', reject);
});
}
module.exports = {
importProductsFromCSV
};
inventory-server/src/utils/db.js
+10 -8
View File
@@ -1,21 +1,23 @@
const { Pool } = require('pg');
import pg from 'pg';
const { Pool } = pg;
let pool;
function initPool(config) {
export function initPool(config) {
pool = new Pool(config);
return pool;
}
async function getConnection() {
export async function getConnection() {
if (!pool) {
throw new Error('Database pool not initialized');
}
return pool.connect();
}
module.exports = {
initPool,
getConnection,
getPool: () => pool
};
export function getPool() {
return pool;
}
export default { initPool, getConnection, getPool };
inventory-server/src/utils/dbConnection.js
+48 -150
View File
@@ -1,158 +1,94 @@
const { Client } = require('ssh2');
const mysql = require('mysql2/promise');
const fs = require('fs');
import { Client } from 'ssh2';
import mysql from 'mysql2/promise';
import fs from 'node:fs';
// Connection pooling and cache configuration
const connectionCache = {
ssh: null,
dbConnection: null,
lastUsed: 0,
isConnecting: false,
connectionPromise: null,
// Cache expiration time in milliseconds (5 minutes)
expirationTime: 5 * 60 * 1000,
// Cache for query results (key: query string, value: {data, timestamp})
queryCache: new Map(),
// Cache duration for different query types in milliseconds
cacheDuration: {
'field-options': 30 * 60 * 1000, // 30 minutes for field options
'product-lines': 10 * 60 * 1000, // 10 minutes for product lines
'sublines': 10 * 60 * 1000, // 10 minutes for sublines
'taxonomy': 30 * 60 * 1000, // 30 minutes for taxonomy data
'default': 60 * 1000 // 1 minute default
}
'field-options': 30 * 60 * 1000,
'product-lines': 10 * 60 * 1000,
'sublines': 10 * 60 * 1000,
'taxonomy': 30 * 60 * 1000,
'default': 60 * 1000,
},
};
/**
* Get a database connection with connection pooling
* @returns {Promise<{ssh: object, connection: object}>} The SSH and database connection
*/
async function getDbConnection() {
export async function getDbConnection() {
const now = Date.now();
// Check if we need to refresh the connection due to inactivity
const needsRefresh = !connectionCache.ssh ||
!connectionCache.dbConnection ||
(now - connectionCache.lastUsed > connectionCache.expirationTime);
// If connection is still valid, update last used time and return existing connection
const needsRefresh = !connectionCache.ssh
|| !connectionCache.dbConnection
|| (now - connectionCache.lastUsed > connectionCache.expirationTime);
if (!needsRefresh) {
connectionCache.lastUsed = now;
return {
ssh: connectionCache.ssh,
connection: connectionCache.dbConnection
};
return { ssh: connectionCache.ssh, connection: connectionCache.dbConnection };
}
// If another request is already establishing a connection, wait for that promise
if (connectionCache.isConnecting && connectionCache.connectionPromise) {
try {
await connectionCache.connectionPromise;
return {
ssh: connectionCache.ssh,
connection: connectionCache.dbConnection
};
return { ssh: connectionCache.ssh, connection: connectionCache.dbConnection };
} catch (error) {
// If that connection attempt failed, we'll try again below
console.error('Error waiting for existing connection:', error);
}
}
// Close existing connections if they exist
if (connectionCache.dbConnection) {
try {
await connectionCache.dbConnection.end();
} catch (error) {
console.error('Error closing existing database connection:', error);
}
try { await connectionCache.dbConnection.end(); }
catch (error) { console.error('Error closing existing database connection:', error); }
}
if (connectionCache.ssh) {
try {
connectionCache.ssh.end();
} catch (error) {
console.error('Error closing existing SSH connection:', error);
}
try { connectionCache.ssh.end(); }
catch (error) { console.error('Error closing existing SSH connection:', error); }
}
// Mark that we're establishing a new connection
connectionCache.isConnecting = true;
// Create a new promise for this connection attempt
connectionCache.connectionPromise = setupSshTunnel().then(tunnel => {
connectionCache.connectionPromise = setupSshTunnel().then((tunnel) => {
const { ssh, stream, dbConfig } = tunnel;
return mysql.createConnection({
...dbConfig,
stream
}).then(connection => {
// Store the new connections
return mysql.createConnection({ ...dbConfig, stream }).then((connection) => {
connectionCache.ssh = ssh;
connectionCache.dbConnection = connection;
connectionCache.lastUsed = Date.now();
connectionCache.isConnecting = false;
return {
ssh,
connection
};
return { ssh, connection };
});
}).catch(error => {
}).catch((error) => {
connectionCache.isConnecting = false;
throw error;
});
// Wait for the connection to be established
return connectionCache.connectionPromise;
}
/**
* Get cached query results or execute query if not cached
* @param {string} cacheKey - Unique key to identify the query
* @param {string} queryType - Type of query (field-options, product-lines, etc.)
* @param {Function} queryFn - Function to execute if cache miss
* @returns {Promise<any>} The query result
*/
async function getCachedQuery(cacheKey, queryType, queryFn) {
// Get cache duration based on query type
export async function getCachedQuery(cacheKey, queryType, queryFn) {
const cacheDuration = connectionCache.cacheDuration[queryType] || connectionCache.cacheDuration.default;
// Check if we have a valid cached result
const cachedResult = connectionCache.queryCache.get(cacheKey);
const now = Date.now();
if (cachedResult && (now - cachedResult.timestamp < cacheDuration)) {
console.log(`Cache hit for ${queryType} query: ${cacheKey}`);
return cachedResult.data;
}
// No valid cache found, execute the query
console.log(`Cache miss for ${queryType} query: ${cacheKey}`);
const result = await queryFn();
// Cache the result
connectionCache.queryCache.set(cacheKey, {
data: result,
timestamp: now
});
connectionCache.queryCache.set(cacheKey, { data: result, timestamp: now });
return result;
}
/**
* Setup SSH tunnel to production database
* @private - Should only be used by getDbConnection
* @returns {Promise<{ssh: object, stream: object, dbConfig: object}>}
*/
async function setupSshTunnel() {
const sshConfig = {
host: process.env.PROD_SSH_HOST,
port: process.env.PROD_SSH_PORT || 22,
port: Number(process.env.PROD_SSH_PORT) || 22,
username: process.env.PROD_SSH_USER,
privateKey: process.env.PROD_SSH_KEY_PATH
? fs.readFileSync(process.env.PROD_SSH_KEY_PATH)
: undefined,
compress: true
compress: true,
};
const dbConfig = {
@@ -160,80 +96,42 @@ async function setupSshTunnel() {
user: process.env.PROD_DB_USER,
password: process.env.PROD_DB_PASSWORD,
database: process.env.PROD_DB_NAME,
port: process.env.PROD_DB_PORT || 3306,
timezone: 'Z'
port: Number(process.env.PROD_DB_PORT) || 3306,
timezone: 'Z',
};
return new Promise((resolve, reject) => {
const ssh = new Client();
ssh.on('error', (err) => {
console.error('SSH connection error:', err);
reject(err);
});
ssh.on('ready', () => {
ssh.forwardOut(
'127.0.0.1',
0,
dbConfig.host,
dbConfig.port,
(err, stream) => {
if (err) reject(err);
resolve({ ssh, stream, dbConfig });
}
);
ssh.forwardOut('127.0.0.1', 0, dbConfig.host, dbConfig.port, (err, stream) => {
if (err) reject(err);
else resolve({ ssh, stream, dbConfig });
});
}).connect(sshConfig);
});
}
/**
* Clear cached query results
* @param {string} [cacheKey] - Specific cache key to clear (clears all if not provided)
*/
function clearQueryCache(cacheKey) {
if (cacheKey) {
connectionCache.queryCache.delete(cacheKey);
console.log(`Cleared cache for key: ${cacheKey}`);
} else {
connectionCache.queryCache.clear();
console.log('Cleared all query cache');
}
export function clearQueryCache(cacheKey) {
if (cacheKey) connectionCache.queryCache.delete(cacheKey);
else connectionCache.queryCache.clear();
}
/**
* Force close all active connections
* Useful for server shutdown or manual connection reset
*/
async function closeAllConnections() {
export async function closeAllConnections() {
if (connectionCache.dbConnection) {
try {
await connectionCache.dbConnection.end();
console.log('Closed database connection');
} catch (error) {
console.error('Error closing database connection:', error);
}
try { await connectionCache.dbConnection.end(); }
catch (error) { console.error('Error closing database connection:', error); }
connectionCache.dbConnection = null;
}
if (connectionCache.ssh) {
try {
connectionCache.ssh.end();
console.log('Closed SSH connection');
} catch (error) {
console.error('Error closing SSH connection:', error);
}
try { connectionCache.ssh.end(); }
catch (error) { console.error('Error closing SSH connection:', error); }
connectionCache.ssh = null;
}
connectionCache.lastUsed = 0;
connectionCache.isConnecting = false;
connectionCache.connectionPromise = null;
}
module.exports = {
getDbConnection,
getCachedQuery,
clearQueryCache,
closeAllConnections
};