Phase 3 + 6
This commit is contained in:
@@ -1,82 +1,36 @@
|
||||
/**
|
||||
* Vector similarity utilities
|
||||
*/
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
* @param {number[]} a
|
||||
* @param {number[]} b
|
||||
* @returns {number} Similarity score between -1 and 1
|
||||
*/
|
||||
function cosineSimilarity(a, b) {
|
||||
if (!a || !b || a.length !== b.length) {
|
||||
return 0;
|
||||
}
|
||||
export function cosineSimilarity(a, b) {
|
||||
if (!a || !b || a.length !== b.length) return 0;
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) return 0;
|
||||
|
||||
return dotProduct / denominator;
|
||||
return denominator === 0 ? 0 : dotProduct / denominator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find top K most similar items from a collection
|
||||
* @param {number[]} queryEmbedding - The embedding to search for
|
||||
* @param {Array<{id: any, embedding: number[]}>} items - Items with embeddings
|
||||
* @param {number} topK - Number of results to return
|
||||
* @returns {Array<{id: any, similarity: number}>}
|
||||
*/
|
||||
function findTopMatches(queryEmbedding, items, topK = 10) {
|
||||
if (!queryEmbedding || !items || items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const scored = items.map(item => ({
|
||||
export function findTopMatches(queryEmbedding, items, topK = 10) {
|
||||
if (!queryEmbedding || !items || items.length === 0) return [];
|
||||
const scored = items.map((item) => ({
|
||||
id: item.id,
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding)
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding),
|
||||
}));
|
||||
|
||||
scored.sort((a, b) => b.similarity - a.similarity);
|
||||
|
||||
return scored.slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find matches above a similarity threshold
|
||||
* @param {number[]} queryEmbedding
|
||||
* @param {Array<{id: any, embedding: number[]}>} items
|
||||
* @param {number} threshold - Minimum similarity (0-1)
|
||||
* @returns {Array<{id: any, similarity: number}>}
|
||||
*/
|
||||
function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
|
||||
if (!queryEmbedding || !items || items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
export function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
|
||||
if (!queryEmbedding || !items || items.length === 0) return [];
|
||||
const scored = items
|
||||
.map(item => ({
|
||||
.map((item) => ({
|
||||
id: item.id,
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding)
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding),
|
||||
}))
|
||||
.filter(item => item.similarity >= threshold);
|
||||
|
||||
.filter((item) => item.similarity >= threshold);
|
||||
scored.sort((a, b) => b.similarity - a.similarity);
|
||||
|
||||
return scored;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cosineSimilarity,
|
||||
findTopMatches,
|
||||
findMatchesAboveThreshold
|
||||
};
|
||||
|
||||
@@ -1,74 +1,52 @@
|
||||
/**
|
||||
* Taxonomy Embedding Service
|
||||
*
|
||||
* Generates and caches embeddings for categories, themes, and colors.
|
||||
* Excludes "Black Friday", "Gifts", "Deals" categories and their children.
|
||||
*
|
||||
* Disk cache: embeddings are saved to data/taxonomy-embeddings.json and reused
|
||||
* across server restarts. Cache is invalidated by content hash — if the taxonomy
|
||||
* rows in MySQL change, the next check will detect it and regenerate automatically.
|
||||
*
|
||||
* Background check: after initialization, call startBackgroundCheck(getConnectionFn)
|
||||
* to poll for taxonomy changes on a configurable interval (default 1h).
|
||||
* Disk cache at data/taxonomy-embeddings.json; content-hash invalidated.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { findTopMatches } = require('./similarity');
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { findTopMatches } from './similarity.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Categories to exclude (and all their children)
|
||||
const EXCLUDED_CATEGORY_NAMES = ['black friday', 'gifts', 'deals'];
|
||||
|
||||
// Disk cache config
|
||||
const CACHE_PATH = path.join(__dirname, '..', '..', '..', '..', 'data', 'taxonomy-embeddings.json');
|
||||
|
||||
class TaxonomyEmbeddings {
|
||||
export class TaxonomyEmbeddings {
|
||||
constructor({ provider, logger }) {
|
||||
this.provider = provider;
|
||||
this.logger = logger || console;
|
||||
|
||||
// Cached taxonomy with embeddings
|
||||
this.categories = [];
|
||||
this.themes = [];
|
||||
this.colors = [];
|
||||
|
||||
// Raw data without embeddings (for lookup)
|
||||
this.categoryMap = new Map();
|
||||
this.themeMap = new Map();
|
||||
this.colorMap = new Map();
|
||||
|
||||
// Content hash of the last successfully built taxonomy (from DB rows)
|
||||
this.contentHash = null;
|
||||
|
||||
this.initialized = false;
|
||||
this.initializing = false;
|
||||
this._checkInterval = null;
|
||||
this._regenerating = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize embeddings — fetches raw taxonomy rows to compute a content hash,
|
||||
* then either loads the matching disk cache or generates fresh embeddings.
|
||||
*/
|
||||
async initialize(connection) {
|
||||
if (this.initialized) {
|
||||
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
|
||||
}
|
||||
|
||||
if (this.initializing) {
|
||||
// Wait for existing initialization
|
||||
while (this.initializing) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
|
||||
}
|
||||
|
||||
this.initializing = true;
|
||||
|
||||
try {
|
||||
// Always fetch raw rows first — cheap (~10ms), no OpenAI calls.
|
||||
// Used to compute a content hash for cache validation.
|
||||
const rawRows = await this._fetchRawRows(connection);
|
||||
const freshHash = this._computeContentHash(rawRows);
|
||||
|
||||
@@ -77,9 +55,9 @@ class TaxonomyEmbeddings {
|
||||
this.categories = cached.categories;
|
||||
this.themes = cached.themes;
|
||||
this.colors = cached.colors;
|
||||
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
|
||||
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
|
||||
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
|
||||
this.categoryMap = new Map(this.categories.map((c) => [c.id, c]));
|
||||
this.themeMap = new Map(this.themes.map((t) => [t.id, t]));
|
||||
this.colorMap = new Map(this.colors.map((c) => [c.id, c]));
|
||||
this.contentHash = freshHash;
|
||||
this.initialized = true;
|
||||
this.logger.info(`[TaxonomyEmbeddings] Loaded from cache: ${this.categories.length} categories, ${this.themes.length} themes, ${this.colors.length} colors`);
|
||||
@@ -95,7 +73,6 @@ class TaxonomyEmbeddings {
|
||||
await this._buildAndEmbed(rawRows, freshHash);
|
||||
this.initialized = true;
|
||||
this.logger.info('[TaxonomyEmbeddings] Initialization complete');
|
||||
|
||||
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
|
||||
} catch (error) {
|
||||
this.logger.error('[TaxonomyEmbeddings] Initialization failed:', error);
|
||||
@@ -105,26 +82,16 @@ class TaxonomyEmbeddings {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a background interval that checks for taxonomy changes and regenerates
|
||||
* embeddings automatically if the content hash differs.
|
||||
*
|
||||
* @param {Function} getConnectionFn - async function returning { connection }
|
||||
* @param {number} intervalMs - check interval, default 1 hour
|
||||
*/
|
||||
startBackgroundCheck(getConnectionFn, intervalMs = 60 * 60 * 1000) {
|
||||
if (this._checkInterval) return;
|
||||
|
||||
this.logger.info(`[TaxonomyEmbeddings] Background taxonomy check started (every ${intervalMs / 60000} min)`);
|
||||
|
||||
this._checkInterval = setInterval(async () => {
|
||||
if (this._regenerating) return;
|
||||
|
||||
try {
|
||||
const { connection } = await getConnectionFn();
|
||||
const rawRows = await this._fetchRawRows(connection);
|
||||
const freshHash = this._computeContentHash(rawRows);
|
||||
|
||||
if (freshHash === this.contentHash) return;
|
||||
|
||||
this.logger.info('[TaxonomyEmbeddings] Taxonomy changed, regenerating embeddings in background...');
|
||||
@@ -146,123 +113,79 @@ class TaxonomyEmbeddings {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar categories for a product embedding
|
||||
*/
|
||||
findSimilarCategories(productEmbedding, topK = 10) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.initialized || !productEmbedding) return [];
|
||||
const matches = findTopMatches(productEmbedding, this.categories, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
return matches.map((match) => {
|
||||
const cat = this.categoryMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: cat?.name || '',
|
||||
fullPath: cat?.fullPath || '',
|
||||
similarity: match.similarity
|
||||
similarity: match.similarity,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar themes for a product embedding
|
||||
*/
|
||||
findSimilarThemes(productEmbedding, topK = 5) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.initialized || !productEmbedding) return [];
|
||||
const matches = findTopMatches(productEmbedding, this.themes, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
return matches.map((match) => {
|
||||
const theme = this.themeMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: theme?.name || '',
|
||||
fullPath: theme?.fullPath || '',
|
||||
similarity: match.similarity
|
||||
similarity: match.similarity,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar colors for a product embedding
|
||||
*/
|
||||
findSimilarColors(productEmbedding, topK = 5) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.initialized || !productEmbedding) return [];
|
||||
const matches = findTopMatches(productEmbedding, this.colors, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
return matches.map((match) => {
|
||||
const color = this.colorMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: color?.name || '',
|
||||
similarity: match.similarity
|
||||
similarity: match.similarity,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all taxonomy data (without embeddings) for frontend
|
||||
*/
|
||||
getTaxonomyData() {
|
||||
return {
|
||||
categories: this.categories.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
|
||||
themes: this.themes.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
|
||||
colors: this.colors.map(({ id, name }) => ({ id, name }))
|
||||
colors: this.colors.map(({ id, name }) => ({ id, name })),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if service is ready
|
||||
*/
|
||||
isReady() {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Fetch minimal raw rows from MySQL — used for content hash computation.
|
||||
* This is the cheap path: no path-building, no embeddings, just the raw data.
|
||||
*/
|
||||
async _fetchRawRows(connection) {
|
||||
const [[catRows], [themeRows], [colorRows]] = await Promise.all([
|
||||
connection.query('SELECT cat_id, name, master_cat_id, type FROM product_categories WHERE type IN (10, 11, 12, 13) ORDER BY cat_id'),
|
||||
connection.query('SELECT cat_id, name, master_cat_id, type FROM product_categories WHERE type IN (20, 21) ORDER BY cat_id'),
|
||||
connection.query('SELECT color, name, hex_color FROM product_color_list ORDER BY `order`')
|
||||
connection.query('SELECT color, name, hex_color FROM product_color_list ORDER BY `order`'),
|
||||
]);
|
||||
return { catRows, themeRows, colorRows };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a stable SHA-256 hash of the taxonomy row content.
|
||||
* Any change to IDs, names, or parent relationships will produce a different hash.
|
||||
*/
|
||||
_computeContentHash({ catRows, themeRows, colorRows }) {
|
||||
const content = JSON.stringify({
|
||||
cats: catRows.map(r => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
|
||||
themes: themeRows.map(r => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
|
||||
colors: colorRows.map(r => [r.color, r.name]).sort()
|
||||
cats: catRows.map((r) => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
|
||||
themes: themeRows.map((r) => [r.cat_id, r.name, r.master_cat_id]).sort((a, b) => a[0] - b[0]),
|
||||
colors: colorRows.map((r) => [r.color, r.name]).sort(),
|
||||
});
|
||||
return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build full taxonomy objects and generate embeddings, then atomically swap
|
||||
* the in-memory state. Called on cache miss and on background change detection.
|
||||
*/
|
||||
async _buildAndEmbed(rawRows, contentHash) {
|
||||
const { catRows, themeRows, colorRows } = rawRows;
|
||||
|
||||
const categories = this._buildCategories(catRows);
|
||||
const themes = this._buildThemes(themeRows);
|
||||
const colors = this._buildColors(colorRows);
|
||||
@@ -272,23 +195,22 @@ class TaxonomyEmbeddings {
|
||||
const [catEmbeddings, themeEmbeddings, colorEmbeddings] = await Promise.all([
|
||||
this._generateEmbeddings(categories, 'categories'),
|
||||
this._generateEmbeddings(themes, 'themes'),
|
||||
this._generateEmbeddings(colors, 'colors')
|
||||
this._generateEmbeddings(colors, 'colors'),
|
||||
]);
|
||||
|
||||
// Atomic in-memory swap (single-threaded JS — readers always see a consistent state)
|
||||
this.categories = catEmbeddings;
|
||||
this.themes = themeEmbeddings;
|
||||
this.colors = colorEmbeddings;
|
||||
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
|
||||
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
|
||||
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
|
||||
this.categoryMap = new Map(this.categories.map((c) => [c.id, c]));
|
||||
this.themeMap = new Map(this.themes.map((t) => [t.id, t]));
|
||||
this.colorMap = new Map(this.colors.map((c) => [c.id, c]));
|
||||
this.contentHash = contentHash;
|
||||
|
||||
this._saveCache();
|
||||
}
|
||||
|
||||
_buildCategories(rows) {
|
||||
const byId = new Map(rows.map(r => [r.cat_id, r]));
|
||||
const byId = new Map(rows.map((r) => [r.cat_id, r]));
|
||||
const excludedIds = new Set();
|
||||
|
||||
for (const row of rows) {
|
||||
@@ -297,7 +219,6 @@ class TaxonomyEmbeddings {
|
||||
}
|
||||
}
|
||||
|
||||
// Multiple passes to find all descendants of excluded categories
|
||||
let foundNew = true;
|
||||
while (foundNew) {
|
||||
foundNew = false;
|
||||
@@ -314,98 +235,80 @@ class TaxonomyEmbeddings {
|
||||
const categories = [];
|
||||
for (const row of rows) {
|
||||
if (excludedIds.has(row.cat_id)) continue;
|
||||
|
||||
const pathParts = [];
|
||||
let current = row;
|
||||
while (current) {
|
||||
pathParts.unshift(current.name);
|
||||
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
|
||||
}
|
||||
|
||||
categories.push({
|
||||
id: row.cat_id,
|
||||
name: row.name,
|
||||
parentId: row.master_cat_id,
|
||||
type: row.type,
|
||||
fullPath: pathParts.join(' > '),
|
||||
embeddingText: pathParts.join(' ')
|
||||
embeddingText: pathParts.join(' '),
|
||||
});
|
||||
}
|
||||
|
||||
return categories;
|
||||
}
|
||||
|
||||
_buildThemes(rows) {
|
||||
const byId = new Map(rows.map(r => [r.cat_id, r]));
|
||||
|
||||
return rows.map(row => {
|
||||
const byId = new Map(rows.map((r) => [r.cat_id, r]));
|
||||
return rows.map((row) => {
|
||||
const pathParts = [];
|
||||
let current = row;
|
||||
while (current) {
|
||||
pathParts.unshift(current.name);
|
||||
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
|
||||
}
|
||||
|
||||
return {
|
||||
id: row.cat_id,
|
||||
name: row.name,
|
||||
parentId: row.master_cat_id,
|
||||
type: row.type,
|
||||
fullPath: pathParts.join(' > '),
|
||||
embeddingText: pathParts.join(' ')
|
||||
embeddingText: pathParts.join(' '),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
_buildColors(rows) {
|
||||
return rows.map(row => ({
|
||||
return rows.map((row) => ({
|
||||
id: row.color,
|
||||
name: row.name,
|
||||
hexColor: row.hex_color,
|
||||
embeddingText: row.name
|
||||
embeddingText: row.name,
|
||||
}));
|
||||
}
|
||||
|
||||
async _generateEmbeddings(items, label) {
|
||||
if (items.length === 0) {
|
||||
return items;
|
||||
}
|
||||
if (items.length === 0) return items;
|
||||
|
||||
const startTime = Date.now();
|
||||
const texts = items.map(item => item.embeddingText);
|
||||
const texts = items.map((item) => item.embeddingText);
|
||||
const results = [...items];
|
||||
|
||||
// Process in batches
|
||||
for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) {
|
||||
for (let i = 0; i < chunk.embeddings.length; i++) {
|
||||
const globalIndex = chunk.startIndex + i;
|
||||
results[globalIndex] = {
|
||||
...results[globalIndex],
|
||||
embedding: chunk.embeddings[i]
|
||||
};
|
||||
results[globalIndex] = { ...results[globalIndex], embedding: chunk.embeddings[i] };
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logger.info(`[TaxonomyEmbeddings] Generated ${items.length} ${label} embeddings in ${elapsed}ms`);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Disk Cache Methods
|
||||
// ============================================================================
|
||||
|
||||
_loadCache() {
|
||||
try {
|
||||
if (!fs.existsSync(CACHE_PATH)) return null;
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(CACHE_PATH, 'utf8'));
|
||||
if (!data.contentHash || !data.categories?.length || !data.themes?.length || !data.colors?.length) {
|
||||
this.logger.warn('[TaxonomyEmbeddings] Disk cache malformed or missing content hash, will regenerate');
|
||||
return null;
|
||||
}
|
||||
|
||||
return data;
|
||||
} catch (err) {
|
||||
this.logger.warn('[TaxonomyEmbeddings] Failed to load disk cache:', err.message);
|
||||
@@ -429,5 +332,3 @@ class TaxonomyEmbeddings {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { TaxonomyEmbeddings };
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
/**
|
||||
* AI Service
|
||||
*
|
||||
* Main entry point for AI functionality including:
|
||||
* - Embeddings for taxonomy suggestions (OpenAI)
|
||||
* - Chat completions for validation tasks (Groq)
|
||||
* - Task registry for AI operations
|
||||
* Main entry point for AI functionality (embeddings + chat completions + task registry).
|
||||
*/
|
||||
|
||||
const { OpenAIProvider } = require('./providers/openaiProvider');
|
||||
const { GroqProvider, MODELS: GROQ_MODELS } = require('./providers/groqProvider');
|
||||
const { TaxonomyEmbeddings } = require('./embeddings/taxonomyEmbeddings');
|
||||
const { cosineSimilarity, findTopMatches } = require('./embeddings/similarity');
|
||||
const { getRegistry, TASK_IDS, registerAllTasks } = require('./tasks');
|
||||
import { OpenAIProvider } from './providers/openaiProvider.js';
|
||||
import { GroqProvider, MODELS as GROQ_MODELS } from './providers/groqProvider.js';
|
||||
import { TaxonomyEmbeddings } from './embeddings/taxonomyEmbeddings.js';
|
||||
import { cosineSimilarity, findTopMatches } from './embeddings/similarity.js';
|
||||
import { getRegistry, TASK_IDS, registerAllTasks } from './tasks/index.js';
|
||||
|
||||
export { TASK_IDS, GROQ_MODELS, cosineSimilarity, findTopMatches };
|
||||
|
||||
let initialized = false;
|
||||
let initializing = false;
|
||||
@@ -19,54 +18,28 @@ let openaiProvider = null;
|
||||
let groqProvider = null;
|
||||
let taxonomyEmbeddings = null;
|
||||
let logger = console;
|
||||
|
||||
// Store pool reference for task access
|
||||
let appPool = null;
|
||||
|
||||
/**
|
||||
* Initialize the AI service
|
||||
* @param {Object} options
|
||||
* @param {string} options.openaiApiKey - OpenAI API key (for embeddings)
|
||||
* @param {string} [options.groqApiKey] - Groq API key (for chat completions)
|
||||
* @param {Object} options.mysqlConnection - MySQL connection for taxonomy data
|
||||
* @param {Object} [options.pool] - PostgreSQL pool for prompt loading
|
||||
* @param {Object} [options.logger] - Logger instance
|
||||
*/
|
||||
async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, logger: customLogger }) {
|
||||
if (initialized) {
|
||||
return { success: true, message: 'Already initialized' };
|
||||
}
|
||||
export async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, logger: customLogger }) {
|
||||
if (initialized) return { success: true, message: 'Already initialized' };
|
||||
|
||||
if (initializing) {
|
||||
// Wait for existing initialization
|
||||
while (initializing) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
return { success: initialized, message: initialized ? 'Initialized' : 'Initialization failed' };
|
||||
}
|
||||
|
||||
initializing = true;
|
||||
|
||||
try {
|
||||
if (customLogger) {
|
||||
logger = customLogger;
|
||||
}
|
||||
|
||||
if (!openaiApiKey) {
|
||||
throw new Error('OpenAI API key is required');
|
||||
}
|
||||
if (customLogger) logger = customLogger;
|
||||
if (!openaiApiKey) throw new Error('OpenAI API key is required');
|
||||
|
||||
logger.info('[AI] Initializing AI service...');
|
||||
if (pool) appPool = pool;
|
||||
|
||||
// Store pool reference for tasks
|
||||
if (pool) {
|
||||
appPool = pool;
|
||||
}
|
||||
|
||||
// Create OpenAI provider (for embeddings)
|
||||
openaiProvider = new OpenAIProvider({ apiKey: openaiApiKey });
|
||||
|
||||
// Create Groq provider (for chat completions) if API key provided
|
||||
if (groqApiKey) {
|
||||
groqProvider = new GroqProvider({ apiKey: groqApiKey });
|
||||
logger.info('[AI] Groq provider initialized for chat completions');
|
||||
@@ -74,32 +47,19 @@ async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, log
|
||||
logger.warn('[AI] No Groq API key provided - chat completion tasks will not be available');
|
||||
}
|
||||
|
||||
// Create and initialize taxonomy embeddings
|
||||
taxonomyEmbeddings = new TaxonomyEmbeddings({
|
||||
provider: openaiProvider,
|
||||
logger
|
||||
});
|
||||
|
||||
taxonomyEmbeddings = new TaxonomyEmbeddings({ provider: openaiProvider, logger });
|
||||
const stats = await taxonomyEmbeddings.initialize(mysqlConnection);
|
||||
|
||||
// Register validation tasks if Groq is available
|
||||
if (groqProvider) {
|
||||
registerValidationTasks();
|
||||
}
|
||||
if (groqProvider) registerValidationTasks();
|
||||
|
||||
initialized = true;
|
||||
logger.info('[AI] AI service initialized', {
|
||||
...stats,
|
||||
groqEnabled: !!groqProvider,
|
||||
tasksRegistered: getRegistry().list()
|
||||
tasksRegistered: getRegistry().list(),
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Initialized',
|
||||
stats,
|
||||
groqEnabled: !!groqProvider
|
||||
};
|
||||
return { success: true, message: 'Initialized', stats, groqEnabled: !!groqProvider };
|
||||
} catch (error) {
|
||||
logger.error('[AI] Initialization failed:', error);
|
||||
return { success: false, message: error.message };
|
||||
@@ -108,41 +68,20 @@ async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, log
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register validation tasks with the task registry
|
||||
* Called during initialization if Groq is available
|
||||
*/
|
||||
function registerValidationTasks() {
|
||||
registerAllTasks(logger);
|
||||
logger.info('[AI] Validation tasks registered');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if service is ready
|
||||
*/
|
||||
function isReady() {
|
||||
export function isReady() {
|
||||
return initialized && taxonomyEmbeddings?.isReady();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start background taxonomy change detection.
|
||||
* Call once after initialization, passing a function that returns { connection }.
|
||||
* @param {Function} getConnectionFn
|
||||
* @param {number} [intervalMs] - default 1 hour
|
||||
*/
|
||||
function startBackgroundCheck(getConnectionFn, intervalMs) {
|
||||
export function startBackgroundCheck(getConnectionFn, intervalMs) {
|
||||
if (!initialized || !taxonomyEmbeddings) return;
|
||||
taxonomyEmbeddings.startBackgroundCheck(getConnectionFn, intervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build weighted product text for embedding.
|
||||
* Weights the product name heavily by repeating it, and truncates long descriptions
|
||||
* to prevent verbose marketing copy from drowning out the product signal.
|
||||
*
|
||||
* @param {Object} product - Product with name, description, company, line
|
||||
* @returns {string} - Combined text for embedding
|
||||
*/
|
||||
function buildProductText(product) {
|
||||
const parts = [];
|
||||
const name = product.name?.trim();
|
||||
@@ -150,20 +89,9 @@ function buildProductText(product) {
|
||||
const company = (product.company_name || product.company)?.trim();
|
||||
const line = (product.line_name || product.line)?.trim();
|
||||
|
||||
// Name is most important - repeat 3x to weight it heavily in the embedding
|
||||
if (name) {
|
||||
parts.push(name, name, name);
|
||||
}
|
||||
|
||||
// Company and line provide context
|
||||
if (company) {
|
||||
parts.push(company);
|
||||
}
|
||||
if (line) {
|
||||
parts.push(line);
|
||||
}
|
||||
|
||||
// Truncate description to prevent it from overwhelming the signal
|
||||
if (name) parts.push(name, name, name);
|
||||
if (company) parts.push(company);
|
||||
if (line) parts.push(line);
|
||||
if (description) {
|
||||
const truncated = description.length > 500
|
||||
? description.substring(0, 500) + '...'
|
||||
@@ -174,74 +102,35 @@ function buildProductText(product) {
|
||||
return parts.join(' ').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for a product
|
||||
* @param {Object} product - Product with name, description, company, line
|
||||
* @returns {Promise<{embedding: number[], latencyMs: number}>}
|
||||
*/
|
||||
async function getProductEmbedding(product) {
|
||||
if (!initialized || !openaiProvider) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
export async function getProductEmbedding(product) {
|
||||
if (!initialized || !openaiProvider) throw new Error('AI service not initialized');
|
||||
|
||||
const text = buildProductText(product);
|
||||
|
||||
if (!text) {
|
||||
return { embedding: null, latencyMs: 0 };
|
||||
}
|
||||
if (!text) return { embedding: null, latencyMs: 0 };
|
||||
|
||||
const result = await openaiProvider.embed(text);
|
||||
|
||||
return {
|
||||
embedding: result.embeddings[0],
|
||||
latencyMs: result.latencyMs
|
||||
};
|
||||
return { embedding: result.embeddings[0], latencyMs: result.latencyMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for multiple products
|
||||
* @param {Object[]} products - Array of products
|
||||
* @returns {Promise<{embeddings: Array<{index: number, embedding: number[]}>, latencyMs: number}>}
|
||||
*/
|
||||
async function getProductEmbeddings(products) {
|
||||
if (!initialized || !openaiProvider) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
export async function getProductEmbeddings(products) {
|
||||
if (!initialized || !openaiProvider) throw new Error('AI service not initialized');
|
||||
|
||||
const texts = products.map(buildProductText);
|
||||
const validIndices = texts.map((t, i) => t ? i : -1).filter((i) => i >= 0);
|
||||
const validTexts = texts.filter((t) => t);
|
||||
|
||||
// Track which products have empty text
|
||||
const validIndices = texts.map((t, i) => t ? i : -1).filter(i => i >= 0);
|
||||
const validTexts = texts.filter(t => t);
|
||||
|
||||
if (validTexts.length === 0) {
|
||||
return { embeddings: [], latencyMs: 0 };
|
||||
}
|
||||
if (validTexts.length === 0) return { embeddings: [], latencyMs: 0 };
|
||||
|
||||
const result = await openaiProvider.embed(validTexts);
|
||||
|
||||
// Map embeddings back to original indices
|
||||
const embeddings = validIndices.map((originalIndex, resultIndex) => ({
|
||||
index: originalIndex,
|
||||
embedding: result.embeddings[resultIndex]
|
||||
embedding: result.embeddings[resultIndex],
|
||||
}));
|
||||
|
||||
return {
|
||||
embeddings,
|
||||
latencyMs: result.latencyMs
|
||||
};
|
||||
return { embeddings, latencyMs: result.latencyMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar taxonomy items for a product embedding
|
||||
* @param {number[]} productEmbedding
|
||||
* @param {Object} options
|
||||
* @returns {{categories: Array, themes: Array, colors: Array}}
|
||||
*/
|
||||
function findSimilarTaxonomy(productEmbedding, options = {}) {
|
||||
if (!initialized || !taxonomyEmbeddings) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
export function findSimilarTaxonomy(productEmbedding, options = {}) {
|
||||
if (!initialized || !taxonomyEmbeddings) throw new Error('AI service not initialized');
|
||||
|
||||
const topCategories = options.topCategories ?? 10;
|
||||
const topThemes = options.topThemes ?? 5;
|
||||
@@ -250,25 +139,15 @@ function findSimilarTaxonomy(productEmbedding, options = {}) {
|
||||
return {
|
||||
categories: taxonomyEmbeddings.findSimilarCategories(productEmbedding, topCategories),
|
||||
themes: taxonomyEmbeddings.findSimilarThemes(productEmbedding, topThemes),
|
||||
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors)
|
||||
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product embedding and find similar taxonomy in one call
|
||||
* @param {Object} product
|
||||
* @param {Object} options
|
||||
*/
|
||||
async function getSuggestionsForProduct(product, options = {}) {
|
||||
export async function getSuggestionsForProduct(product, options = {}) {
|
||||
const { embedding, latencyMs: embeddingLatency } = await getProductEmbedding(product);
|
||||
|
||||
if (!embedding) {
|
||||
return {
|
||||
categories: [],
|
||||
themes: [],
|
||||
colors: [],
|
||||
latencyMs: embeddingLatency
|
||||
};
|
||||
return { categories: [], themes: [], colors: [], latencyMs: embeddingLatency };
|
||||
}
|
||||
|
||||
const startSearch = Date.now();
|
||||
@@ -279,27 +158,17 @@ async function getSuggestionsForProduct(product, options = {}) {
|
||||
...suggestions,
|
||||
latencyMs: embeddingLatency + searchLatency,
|
||||
embeddingLatencyMs: embeddingLatency,
|
||||
searchLatencyMs: searchLatency
|
||||
searchLatencyMs: searchLatency,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all taxonomy data (without embeddings) for frontend
|
||||
*/
|
||||
function getTaxonomyData() {
|
||||
if (!initialized || !taxonomyEmbeddings) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
export function getTaxonomyData() {
|
||||
if (!initialized || !taxonomyEmbeddings) throw new Error('AI service not initialized');
|
||||
return taxonomyEmbeddings.getTaxonomyData();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get service status
|
||||
*/
|
||||
function getStatus() {
|
||||
export function getStatus() {
|
||||
const registry = getRegistry();
|
||||
|
||||
return {
|
||||
initialized,
|
||||
ready: isReady(),
|
||||
@@ -309,90 +178,56 @@ function getStatus() {
|
||||
taxonomyStats: taxonomyEmbeddings ? {
|
||||
categories: taxonomyEmbeddings.categories?.length || 0,
|
||||
themes: taxonomyEmbeddings.themes?.length || 0,
|
||||
colors: taxonomyEmbeddings.colors?.length || 0
|
||||
colors: taxonomyEmbeddings.colors?.length || 0,
|
||||
} : null,
|
||||
tasks: {
|
||||
registered: registry.list(),
|
||||
count: registry.size()
|
||||
}
|
||||
count: registry.size(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an AI task by ID
|
||||
* @param {string} taskId - Task identifier from TASK_IDS
|
||||
* @param {Object} payload - Task-specific input
|
||||
* @returns {Promise<Object>} Task result
|
||||
*/
|
||||
async function runTask(taskId, payload = {}) {
|
||||
if (!initialized) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
if (!groqProvider) {
|
||||
throw new Error('Groq provider not available - chat completion tasks require GROQ_API_KEY');
|
||||
}
|
||||
export async function runTask(taskId, payload = {}) {
|
||||
if (!initialized) throw new Error('AI service not initialized');
|
||||
if (!groqProvider) throw new Error('Groq provider not available - chat completion tasks require GROQ_API_KEY');
|
||||
|
||||
const registry = getRegistry();
|
||||
return registry.runTask(taskId, {
|
||||
...payload,
|
||||
// Inject dependencies tasks may need
|
||||
provider: groqProvider,
|
||||
// Use pool from payload if provided (from route), fall back to stored appPool
|
||||
pool: payload.pool || appPool,
|
||||
logger
|
||||
logger,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Groq provider instance (for direct use if needed)
|
||||
* @returns {GroqProvider|null}
|
||||
*/
|
||||
function getGroqProvider() {
|
||||
export function getGroqProvider() {
|
||||
return groqProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the PostgreSQL pool (for tasks that need DB access)
|
||||
* @returns {Object|null}
|
||||
*/
|
||||
function getPool() {
|
||||
export function getPool() {
|
||||
return appPool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if chat completion tasks are available
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function hasChatCompletion() {
|
||||
export function hasChatCompletion() {
|
||||
return !!groqProvider;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Initialization
|
||||
export default {
|
||||
initialize,
|
||||
isReady,
|
||||
getStatus,
|
||||
startBackgroundCheck,
|
||||
|
||||
// Embeddings (OpenAI)
|
||||
getProductEmbedding,
|
||||
getProductEmbeddings,
|
||||
findSimilarTaxonomy,
|
||||
getSuggestionsForProduct,
|
||||
getTaxonomyData,
|
||||
|
||||
// Chat completions (Groq)
|
||||
runTask,
|
||||
hasChatCompletion,
|
||||
getGroqProvider,
|
||||
getPool,
|
||||
|
||||
// Constants
|
||||
TASK_IDS,
|
||||
GROQ_MODELS,
|
||||
|
||||
// Re-export utilities
|
||||
cosineSimilarity,
|
||||
findTopMatches
|
||||
findTopMatches,
|
||||
};
|
||||
|
||||
@@ -1,81 +1,41 @@
|
||||
/**
|
||||
* Description Validation Prompts
|
||||
*
|
||||
* Functions for building and parsing description validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sanitize an issue string from AI response
|
||||
* AI sometimes returns malformed strings with escape sequences
|
||||
*
|
||||
* @param {string} issue - Raw issue string
|
||||
* @returns {string} Cleaned issue string
|
||||
*/
|
||||
function sanitizeIssue(issue) {
|
||||
if (!issue || typeof issue !== 'string') return '';
|
||||
|
||||
let cleaned = issue
|
||||
// Remove trailing backslashes (incomplete escapes)
|
||||
return issue
|
||||
.replace(/\\+$/, '')
|
||||
// Fix malformed escaped quotes at end of string
|
||||
.replace(/\\",?\)?$/, '')
|
||||
// Clean up double-escaped quotes
|
||||
.replace(/\\\\"/g, '"')
|
||||
// Clean up single escaped quotes that aren't needed
|
||||
.replace(/\\"/g, '"')
|
||||
// Remove any remaining trailing punctuation artifacts
|
||||
.replace(/[,\s]+$/, '')
|
||||
// Trim whitespace
|
||||
.trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt for description validation
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object} product - Product data
|
||||
* @param {string} product.name - Product name
|
||||
* @param {string} product.description - Current description
|
||||
* @param {string} [product.company_name] - Company name
|
||||
* @param {string} [product.categories] - Product categories
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General description guidelines
|
||||
* @param {string} [prompts.companySpecific] - Company-specific rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildDescriptionUserPrompt(product, prompts) {
|
||||
export function buildDescriptionUserPrompt(product, prompts) {
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/guidelines if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// Add company-specific rules if provided
|
||||
if (prompts.companySpecific) {
|
||||
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
|
||||
parts.push(prompts.companySpecific);
|
||||
parts.push(''); // Empty line for separation
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// Add product information
|
||||
parts.push('PRODUCT TO VALIDATE:');
|
||||
parts.push(`NAME: "${product.name || ''}"`);
|
||||
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
|
||||
|
||||
if (product.categories) {
|
||||
parts.push(`CATEGORIES: ${product.categories}`);
|
||||
}
|
||||
if (product.categories) parts.push(`CATEGORIES: ${product.categories}`);
|
||||
|
||||
parts.push('');
|
||||
parts.push('CURRENT DESCRIPTION:');
|
||||
parts.push(`"${product.description || '(empty)'}"`);
|
||||
|
||||
// Add response format instructions
|
||||
parts.push('');
|
||||
parts.push('CRITICAL RULES:');
|
||||
parts.push('- If isValid is false, you MUST provide a suggestion with the improved description');
|
||||
@@ -86,68 +46,40 @@ function buildDescriptionUserPrompt(product, prompts) {
|
||||
parts.push(JSON.stringify({
|
||||
isValid: 'true if perfect, false if ANY changes needed',
|
||||
suggestion: 'REQUIRED when isValid is false - the complete improved description',
|
||||
issues: ['list each problem found (empty array only if isValid is true)']
|
||||
issues: ['list each problem found (empty array only if isValid is true)'],
|
||||
}, null, 2));
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for description validation
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseDescriptionResponse(parsed, content) {
|
||||
// If we got valid parsed JSON, use it
|
||||
export function parseDescriptionResponse(parsed, content) {
|
||||
if (parsed && typeof parsed.isValid === 'boolean') {
|
||||
// Sanitize issues - AI sometimes returns malformed escape sequences
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
|
||||
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
|
||||
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
|
||||
// If there are issues, treat as invalid regardless of what the AI said.
|
||||
// Also if there's a suggestion, the AI thought something needed to change.
|
||||
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Handle case where isValid is a string "true"/"false" instead of boolean
|
||||
if (parsed && typeof parsed.isValid === 'string') {
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const isValid = rawIsValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Look for isValid pattern
|
||||
const isValidMatch = content.match(/"isValid"\s*:\s*(true|false)/i);
|
||||
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
|
||||
|
||||
// Look for suggestion (might be multiline)
|
||||
const suggestionMatch = content.match(/"suggestion"\s*:\s*"((?:[^"\\]|\\.)*)"/s);
|
||||
let suggestion = suggestionMatch ? suggestionMatch[1] : null;
|
||||
if (suggestion) {
|
||||
// Unescape common escapes
|
||||
suggestion = suggestion.replace(/\\n/g, '\n').replace(/\\"/g, '"');
|
||||
}
|
||||
|
||||
// Look for issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
if (issuesMatch) {
|
||||
@@ -155,22 +87,13 @@ function parseDescriptionResponse(parsed, content) {
|
||||
const issueStrings = issuesContent.match(/"([^"]+)"/g);
|
||||
if (issueStrings) {
|
||||
issues = issueStrings
|
||||
.map(s => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter(issue => issue.length > 0);
|
||||
.map((s) => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter((issue) => issue.length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Same logic: if there are issues, it's not valid
|
||||
const finalIsValid = isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid: finalIsValid, suggestion, issues };
|
||||
} catch {
|
||||
// Default to valid if we can't parse anything
|
||||
return { isValid: true, suggestion: null, issues: [] };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildDescriptionUserPrompt,
|
||||
parseDescriptionResponse
|
||||
};
|
||||
|
||||
@@ -1,164 +1,94 @@
|
||||
/**
|
||||
* Name Validation Prompts
|
||||
*
|
||||
* Functions for building and parsing name validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sanitize an issue string from AI response
|
||||
* AI sometimes returns malformed strings with escape sequences
|
||||
*
|
||||
* @param {string} issue - Raw issue string
|
||||
* @returns {string} Cleaned issue string
|
||||
*/
|
||||
function sanitizeIssue(issue) {
|
||||
if (!issue || typeof issue !== 'string') return '';
|
||||
|
||||
let cleaned = issue
|
||||
// Remove trailing backslashes (incomplete escapes)
|
||||
return issue
|
||||
.replace(/\\+$/, '')
|
||||
// Fix malformed escaped quotes at end of string
|
||||
.replace(/\\",?\)?$/, '')
|
||||
// Clean up double-escaped quotes
|
||||
.replace(/\\\\"/g, '"')
|
||||
// Clean up single escaped quotes that aren't needed
|
||||
.replace(/\\"/g, '"')
|
||||
// Remove any remaining trailing punctuation artifacts
|
||||
.replace(/[,\s]+$/, '')
|
||||
// Trim whitespace
|
||||
.trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt for name validation
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object} product - Product data
|
||||
* @param {string} product.name - Current product name
|
||||
* @param {string} [product.company_name] - Company name
|
||||
* @param {string} [product.line_name] - Product line name
|
||||
* @param {string} [product.subline_name] - Product subline name
|
||||
* @param {string[]} [product.siblingNames] - Names of other products in the same line
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General naming conventions
|
||||
* @param {string} [prompts.companySpecific] - Company-specific rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildNameUserPrompt(product, prompts) {
|
||||
export function buildNameUserPrompt(product, prompts) {
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/conventions if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// Add company-specific rules if provided
|
||||
if (prompts.companySpecific) {
|
||||
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
|
||||
parts.push(prompts.companySpecific);
|
||||
parts.push(''); // Empty line for separation
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// Add product information
|
||||
parts.push('PRODUCT TO VALIDATE:');
|
||||
parts.push(`NAME: "${product.name || ''}"`);
|
||||
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
|
||||
parts.push(`LINE: ${product.line_name || 'None'}`);
|
||||
if (product.subline_name) {
|
||||
parts.push(`SUBLINE: ${product.subline_name}`);
|
||||
}
|
||||
if (product.subline_name) parts.push(`SUBLINE: ${product.subline_name}`);
|
||||
|
||||
// Add sibling context for naming decisions
|
||||
if (product.siblingNames && product.siblingNames.length > 0) {
|
||||
parts.push('');
|
||||
parts.push(`OTHER PRODUCTS IN THIS LINE (${product.siblingNames.length + 1} total including this one):`);
|
||||
product.siblingNames.forEach(name => {
|
||||
parts.push(`- ${name}`);
|
||||
});
|
||||
product.siblingNames.forEach((name) => parts.push(`- ${name}`));
|
||||
}
|
||||
|
||||
// Add response format instructions
|
||||
parts.push('');
|
||||
parts.push('RESPOND WITH JSON:');
|
||||
parts.push(JSON.stringify({
|
||||
isValid: 'true/false',
|
||||
suggestion: 'corrected name if changes needed, or null if valid',
|
||||
issues: ['issue 1', 'issue 2 (empty array if valid)']
|
||||
issues: ['issue 1', 'issue 2 (empty array if valid)'],
|
||||
}, null, 2));
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for name validation
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseNameResponse(parsed, content) {
|
||||
// Debug: Log what we're trying to parse
|
||||
export function parseNameResponse(parsed, content) {
|
||||
console.log('[parseNameResponse] Input:', {
|
||||
hasParsed: !!parsed,
|
||||
parsedIsValid: parsed?.isValid,
|
||||
parsedType: typeof parsed?.isValid,
|
||||
contentPreview: content?.substring(0, 3000)
|
||||
contentPreview: content?.substring(0, 3000),
|
||||
});
|
||||
|
||||
// If we got valid parsed JSON, use it
|
||||
if (parsed && typeof parsed.isValid === 'boolean') {
|
||||
// Sanitize issues - AI sometimes returns malformed escape sequences
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
|
||||
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
|
||||
// If there are issues, treat as invalid regardless of what the AI said.
|
||||
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Handle case where isValid is a string "true"/"false" instead of boolean
|
||||
if (parsed && typeof parsed.isValid === 'string') {
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const issues = rawIssues.map(sanitizeIssue).filter((issue) => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const isValid = rawIsValid && issues.length === 0 && !suggestion;
|
||||
|
||||
console.log('[parseNameResponse] Parsed isValid as string:', parsed.isValid, '→', isValid);
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Look for isValid pattern - handle both boolean and quoted string
|
||||
// Matches: "isValid": true, "isValid": false, "isValid": "true", "isValid": "false"
|
||||
const isValidMatch = content.match(/"isValid"\s*:\s*"?(true|false)"?/i);
|
||||
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
|
||||
|
||||
console.log('[parseNameResponse] Regex extraction:', {
|
||||
isValidMatch: isValidMatch?.[0],
|
||||
isValidValue: isValidMatch?.[1],
|
||||
resultIsValid: isValid
|
||||
resultIsValid: isValid,
|
||||
});
|
||||
|
||||
// Look for suggestion - handle escaped quotes and null
|
||||
const suggestionMatch = content.match(/"suggestion"\s*:\s*(?:"([^"\\]*(?:\\.[^"\\]*)*)"|null)/);
|
||||
const suggestion = suggestionMatch ? (suggestionMatch[1] || null) : null;
|
||||
|
||||
// Look for issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
if (issuesMatch) {
|
||||
@@ -166,22 +96,13 @@ function parseNameResponse(parsed, content) {
|
||||
const issueStrings = issuesContent.match(/"([^"]+)"/g);
|
||||
if (issueStrings) {
|
||||
issues = issueStrings
|
||||
.map(s => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter(issue => issue.length > 0);
|
||||
.map((s) => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter((issue) => issue.length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const finalIsValid = isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid: finalIsValid, suggestion, issues };
|
||||
} catch {
|
||||
// Default to valid if we can't parse anything
|
||||
return { isValid: true, suggestion: null, issues: [] };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildNameUserPrompt,
|
||||
parseNameResponse
|
||||
};
|
||||
|
||||
@@ -1,35 +1,18 @@
|
||||
/**
|
||||
* Prompt Loader
|
||||
*
|
||||
* Utilities to load AI prompts from the ai_prompts PostgreSQL table.
|
||||
* Supports loading prompts by base type (e.g., 'name_validation' loads
|
||||
* name_validation_system, name_validation_general, and optionally
|
||||
* name_validation_company_specific).
|
||||
* Prompt Loader — loads AI prompts from the ai_prompts PostgreSQL table.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Load a single prompt by exact type
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string} promptType - Exact prompt type (e.g., 'name_validation_system')
|
||||
* @param {string} [company] - Company identifier (for company_specific types)
|
||||
* @returns {Promise<string|null>} Prompt text or null if not found
|
||||
*/
|
||||
async function loadPromptByType(pool, promptType, company = null) {
|
||||
export async function loadPromptByType(pool, promptType, company = null) {
|
||||
try {
|
||||
let result;
|
||||
|
||||
if (company) {
|
||||
result = await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company = $2',
|
||||
[promptType, company]
|
||||
);
|
||||
} else {
|
||||
result = await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company IS NULL',
|
||||
[promptType]
|
||||
);
|
||||
}
|
||||
|
||||
const result = company
|
||||
? await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company = $2',
|
||||
[promptType, company]
|
||||
)
|
||||
: await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company IS NULL',
|
||||
[promptType]
|
||||
);
|
||||
return result.rows[0]?.prompt_text || null;
|
||||
} catch (error) {
|
||||
console.error(`[PromptLoader] Error loading ${promptType} prompt:`, error.message);
|
||||
@@ -37,93 +20,46 @@ async function loadPromptByType(pool, promptType, company = null) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all prompts for a task type (system, general, and optionally company-specific)
|
||||
*
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string} baseType - Base type name (e.g., 'name_validation', 'description_validation')
|
||||
* @param {string|null} [company] - Optional company ID for company-specific prompts
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadPromptsByType(pool, baseType, company = null) {
|
||||
export async function loadPromptsByType(pool, baseType, company = null) {
|
||||
const systemType = `${baseType}_system`;
|
||||
const generalType = `${baseType}_general`;
|
||||
const companyType = `${baseType}_company_specific`;
|
||||
|
||||
// Load system and general prompts in parallel
|
||||
const [system, general] = await Promise.all([
|
||||
loadPromptByType(pool, systemType),
|
||||
loadPromptByType(pool, generalType)
|
||||
loadPromptByType(pool, generalType),
|
||||
]);
|
||||
|
||||
// Load company-specific prompt if company is provided
|
||||
let companySpecific = null;
|
||||
if (company) {
|
||||
companySpecific = await loadPromptByType(pool, companyType, company);
|
||||
}
|
||||
|
||||
return {
|
||||
system,
|
||||
general,
|
||||
companySpecific
|
||||
};
|
||||
return { system, general, companySpecific };
|
||||
}
|
||||
|
||||
/**
|
||||
* Load name validation prompts
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadNameValidationPrompts(pool, company = null) {
|
||||
export function loadNameValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'name_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load description validation prompts
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadDescriptionValidationPrompts(pool, company = null) {
|
||||
export function loadDescriptionValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'description_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load sanity check prompts (no company-specific variant)
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: null}>}
|
||||
*/
|
||||
async function loadSanityCheckPrompts(pool) {
|
||||
export function loadSanityCheckPrompts(pool) {
|
||||
return loadPromptsByType(pool, 'sanity_check', null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load bulk validation prompts (GPT-5 validation)
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadBulkValidationPrompts(pool, company = null) {
|
||||
export function loadBulkValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'bulk_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load bulk validation prompts for multiple companies at once
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string[]} companyIds - Array of company IDs
|
||||
* @returns {Promise<{system: string|null, general: string|null, companyPrompts: Map<string, string>}>}
|
||||
*/
|
||||
async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
|
||||
// Load system and general prompts
|
||||
export async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
|
||||
const [system, general] = await Promise.all([
|
||||
loadPromptByType(pool, 'bulk_validation_system'),
|
||||
loadPromptByType(pool, 'bulk_validation_general')
|
||||
loadPromptByType(pool, 'bulk_validation_general'),
|
||||
]);
|
||||
|
||||
// Load company-specific prompts for all provided companies
|
||||
const companyPrompts = new Map();
|
||||
|
||||
if (companyIds.length > 0) {
|
||||
try {
|
||||
const result = await pool.query(
|
||||
@@ -132,7 +68,6 @@ async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
|
||||
AND company = ANY($1)`,
|
||||
[companyIds]
|
||||
);
|
||||
|
||||
for (const row of result.rows) {
|
||||
companyPrompts.set(row.company, row.prompt_text);
|
||||
}
|
||||
@@ -140,35 +75,14 @@ async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
|
||||
console.error('[PromptLoader] Error loading company-specific prompts:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
system,
|
||||
general,
|
||||
companyPrompts
|
||||
};
|
||||
return { system, general, companyPrompts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that required prompts exist, throw error if missing
|
||||
* @param {Object} prompts - Prompts object from loadPromptsByType
|
||||
* @param {string} baseType - Base type for error messages
|
||||
* @param {Object} options - Validation options
|
||||
* @param {boolean} [options.requireSystem=true] - Require system prompt
|
||||
* @param {boolean} [options.requireGeneral=true] - Require general prompt
|
||||
* @throws {Error} If required prompts are missing
|
||||
*/
|
||||
function validateRequiredPrompts(prompts, baseType, options = {}) {
|
||||
export function validateRequiredPrompts(prompts, baseType, options = {}) {
|
||||
const { requireSystem = true, requireGeneral = true } = options;
|
||||
const missing = [];
|
||||
|
||||
if (requireSystem && !prompts.system) {
|
||||
missing.push(`${baseType}_system`);
|
||||
}
|
||||
|
||||
if (requireGeneral && !prompts.general) {
|
||||
missing.push(`${baseType}_general`);
|
||||
}
|
||||
|
||||
if (requireSystem && !prompts.system) missing.push(`${baseType}_system`);
|
||||
if (requireGeneral && !prompts.general) missing.push(`${baseType}_general`);
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`Missing required AI prompts: ${missing.join(', ')}. ` +
|
||||
@@ -176,19 +90,3 @@ function validateRequiredPrompts(prompts, baseType, options = {}) {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Core loader
|
||||
loadPromptByType,
|
||||
loadPromptsByType,
|
||||
|
||||
// Task-specific loaders
|
||||
loadNameValidationPrompts,
|
||||
loadDescriptionValidationPrompts,
|
||||
loadSanityCheckPrompts,
|
||||
loadBulkValidationPrompts,
|
||||
loadBulkValidationPromptsForCompanies,
|
||||
|
||||
// Validation
|
||||
validateRequiredPrompts
|
||||
};
|
||||
|
||||
@@ -1,21 +1,8 @@
|
||||
/**
|
||||
* Sanity Check Prompts
|
||||
*
|
||||
* Functions for building and parsing batch product consistency validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Build the user prompt for sanity check
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object[]} products - Array of product data (limited fields for context)
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General sanity check rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildSanityCheckUserPrompt(products, prompts) {
|
||||
// Build a simplified product list for the prompt
|
||||
export function buildSanityCheckUserPrompt(products, prompts) {
|
||||
const productSummaries = products.map((p, index) => ({
|
||||
index,
|
||||
name: p.name,
|
||||
@@ -33,22 +20,17 @@ function buildSanityCheckUserPrompt(products, prompts) {
|
||||
weight: p.weight,
|
||||
length: p.length,
|
||||
width: p.width,
|
||||
height: p.height
|
||||
height: p.height,
|
||||
}));
|
||||
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/rules if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// Add products to review
|
||||
parts.push(`PRODUCTS TO REVIEW (${products.length} items):`);
|
||||
parts.push(JSON.stringify(productSummaries, null, 2));
|
||||
|
||||
// Add response format
|
||||
parts.push('');
|
||||
parts.push('RESPOND WITH JSON:');
|
||||
parts.push(JSON.stringify({
|
||||
@@ -57,10 +39,10 @@ function buildSanityCheckUserPrompt(products, prompts) {
|
||||
productIndex: 0,
|
||||
field: 'msrp',
|
||||
issue: 'Description of the issue found',
|
||||
suggestion: 'Suggested fix or verification (optional)'
|
||||
}
|
||||
suggestion: 'Suggested fix or verification (optional)',
|
||||
},
|
||||
],
|
||||
summary: '2-3 sentences summarizing the overall product quality'
|
||||
summary: '2-3 sentences summarizing the overall product quality',
|
||||
}, null, 2));
|
||||
|
||||
parts.push('');
|
||||
@@ -69,60 +51,40 @@ function buildSanityCheckUserPrompt(products, prompts) {
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for sanity check
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseSanityCheckResponse(parsed, content) {
|
||||
// If we got valid parsed JSON, use it
|
||||
export function parseSanityCheckResponse(parsed, content) {
|
||||
if (parsed && Array.isArray(parsed.issues)) {
|
||||
return {
|
||||
issues: parsed.issues.map(issue => ({
|
||||
issues: parsed.issues.map((issue) => ({
|
||||
productIndex: issue.productIndex ?? issue.index ?? 0,
|
||||
field: issue.field || 'unknown',
|
||||
issue: issue.issue || issue.message || '',
|
||||
suggestion: issue.suggestion || null
|
||||
suggestion: issue.suggestion || null,
|
||||
})),
|
||||
summary: parsed.summary || 'Review complete'
|
||||
summary: parsed.summary || 'Review complete',
|
||||
};
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Try to find issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
|
||||
if (issuesMatch) {
|
||||
// Try to parse the array content
|
||||
try {
|
||||
const arrayContent = `[${issuesMatch[1]}]`;
|
||||
const parsedIssues = JSON.parse(arrayContent);
|
||||
issues = parsedIssues.map(issue => ({
|
||||
issues = parsedIssues.map((issue) => ({
|
||||
productIndex: issue.productIndex ?? issue.index ?? 0,
|
||||
field: issue.field || 'unknown',
|
||||
issue: issue.issue || issue.message || '',
|
||||
suggestion: issue.suggestion || null
|
||||
suggestion: issue.suggestion || null,
|
||||
}));
|
||||
} catch {
|
||||
// Couldn't parse the array
|
||||
/* fall through */
|
||||
}
|
||||
}
|
||||
|
||||
// Try to find summary
|
||||
const summaryMatch = content.match(/"summary"\s*:\s*"([^"]+)"/);
|
||||
const summary = summaryMatch ? summaryMatch[1] : 'Review complete';
|
||||
|
||||
return { issues, summary };
|
||||
} catch {
|
||||
return { issues: [], summary: 'Could not parse review results' };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildSanityCheckUserPrompt,
|
||||
parseSanityCheckResponse
|
||||
};
|
||||
|
||||
@@ -1,25 +1,15 @@
|
||||
/**
|
||||
* Groq Provider - Handles chat completions via Groq's OpenAI-compatible API
|
||||
*
|
||||
* Uses Groq's fast inference for real-time AI validation tasks.
|
||||
* Supports models like openai/gpt-oss-120b (complex) and openai/gpt-oss-20b (simple).
|
||||
* Groq Provider - chat completions via Groq's OpenAI-compatible API
|
||||
*/
|
||||
|
||||
const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
|
||||
export const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
|
||||
|
||||
// Default models
|
||||
const MODELS = {
|
||||
LARGE: 'openai/gpt-oss-120b', // For complex tasks (descriptions, sanity checks)
|
||||
SMALL: 'openai/gpt-oss-20b' // For simple tasks (name validation)
|
||||
export const MODELS = {
|
||||
LARGE: 'openai/gpt-oss-120b',
|
||||
SMALL: 'openai/gpt-oss-20b',
|
||||
};
|
||||
|
||||
class GroqProvider {
|
||||
/**
|
||||
* @param {Object} options
|
||||
* @param {string} options.apiKey - Groq API key
|
||||
* @param {string} [options.baseUrl] - Override base URL
|
||||
* @param {number} [options.timeoutMs=30000] - Default timeout
|
||||
*/
|
||||
export class GroqProvider {
|
||||
constructor({ apiKey, baseUrl = GROQ_BASE_URL, timeoutMs = 30000 }) {
|
||||
if (!apiKey) {
|
||||
throw new Error('Groq API key is required');
|
||||
@@ -29,41 +19,25 @@ class GroqProvider {
|
||||
this.timeoutMs = timeoutMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a chat completion request
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {Array<{role: string, content: string}>} params.messages - Conversation messages
|
||||
* @param {string} [params.model] - Model to use (defaults to LARGE)
|
||||
* @param {number} [params.temperature=0.3] - Response randomness (0-2)
|
||||
* @param {number} [params.maxTokens=500] - Max tokens in response
|
||||
* @param {Object} [params.responseFormat] - For JSON mode: { type: 'json_object' }
|
||||
* @param {number} [params.timeoutMs] - Request timeout override
|
||||
* @returns {Promise<{content: string, parsed: Object|null, usage: Object, latencyMs: number, model: string}>}
|
||||
*/
|
||||
async chatCompletion({
|
||||
messages,
|
||||
model = MODELS.LARGE,
|
||||
temperature = 0.3,
|
||||
maxTokens = 500,
|
||||
responseFormat = null,
|
||||
timeoutMs = this.timeoutMs
|
||||
timeoutMs = this.timeoutMs,
|
||||
}) {
|
||||
const started = Date.now();
|
||||
|
||||
const body = {
|
||||
model,
|
||||
messages,
|
||||
temperature,
|
||||
max_completion_tokens: maxTokens
|
||||
max_completion_tokens: maxTokens,
|
||||
};
|
||||
|
||||
// Enable JSON mode if requested
|
||||
if (responseFormat?.type === 'json_object') {
|
||||
body.response_format = { type: 'json_object' };
|
||||
}
|
||||
|
||||
// Debug: Log request being sent
|
||||
console.log('[Groq] Request:', {
|
||||
model: body.model,
|
||||
temperature: body.temperature,
|
||||
@@ -71,12 +45,11 @@ class GroqProvider {
|
||||
hasResponseFormat: !!body.response_format,
|
||||
messageCount: body.messages?.length,
|
||||
systemPromptLength: body.messages?.[0]?.content?.length,
|
||||
userPromptLength: body.messages?.[1]?.content?.length
|
||||
userPromptLength: body.messages?.[1]?.content?.length,
|
||||
});
|
||||
|
||||
const response = await this._makeRequest('chat/completions', body, timeoutMs);
|
||||
|
||||
// Debug: Log raw response structure
|
||||
console.log('[Groq] Raw response:', {
|
||||
hasChoices: !!response.choices,
|
||||
choicesLength: response.choices?.length,
|
||||
@@ -84,22 +57,20 @@ class GroqProvider {
|
||||
finishReason: response.choices[0].finish_reason,
|
||||
hasMessage: !!response.choices[0].message,
|
||||
contentLength: response.choices[0].message?.content?.length,
|
||||
contentPreview: response.choices[0].message?.content?.substring(0, 200)
|
||||
contentPreview: response.choices[0].message?.content?.substring(0, 200),
|
||||
} : null,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
model: response.model,
|
||||
});
|
||||
|
||||
const content = response.choices?.[0]?.message?.content || '';
|
||||
const usage = response.usage || {};
|
||||
|
||||
// Attempt to parse JSON if response format was requested
|
||||
let parsed = null;
|
||||
if (responseFormat && content) {
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch {
|
||||
// Content isn't valid JSON - try to extract JSON from markdown
|
||||
parsed = this._extractJson(content);
|
||||
}
|
||||
}
|
||||
@@ -110,74 +81,50 @@ class GroqProvider {
|
||||
usage: {
|
||||
promptTokens: usage.prompt_tokens || 0,
|
||||
completionTokens: usage.completion_tokens || 0,
|
||||
totalTokens: usage.total_tokens || 0
|
||||
totalTokens: usage.total_tokens || 0,
|
||||
},
|
||||
latencyMs: Date.now() - started,
|
||||
model: response.model || model
|
||||
model: response.model || model,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON from content that might be wrapped in markdown code blocks
|
||||
* @private
|
||||
*/
|
||||
_extractJson(content) {
|
||||
// Try to find JSON in code blocks
|
||||
const codeBlockMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
||||
if (codeBlockMatch) {
|
||||
try {
|
||||
return JSON.parse(codeBlockMatch[1].trim());
|
||||
} catch {
|
||||
// Fall through
|
||||
}
|
||||
try { return JSON.parse(codeBlockMatch[1].trim()); } catch { /* fall through */ }
|
||||
}
|
||||
|
||||
// Try to find JSON object/array directly
|
||||
const jsonMatch = content.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
||||
if (jsonMatch) {
|
||||
try {
|
||||
return JSON.parse(jsonMatch[1]);
|
||||
} catch {
|
||||
// Fall through
|
||||
}
|
||||
try { return JSON.parse(jsonMatch[1]); } catch { /* fall through */ }
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an HTTP request to Groq API
|
||||
* @private
|
||||
*/
|
||||
async _makeRequest(endpoint, body, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`
|
||||
'Authorization': `Bearer ${this.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({}));
|
||||
const message = error.error?.message || `Groq API error: ${response.status}`;
|
||||
const err = new Error(message);
|
||||
err.status = response.status;
|
||||
err.code = error.error?.code;
|
||||
// Include failed_generation if available (for JSON mode failures)
|
||||
if (error.error?.failed_generation) {
|
||||
err.failedGeneration = error.error.failed_generation;
|
||||
console.error('[Groq] JSON validation failed. Model output:', error.error.failed_generation);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
@@ -191,13 +138,7 @@ class GroqProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the provider is properly configured
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isConfigured() {
|
||||
return !!this.apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { GroqProvider, MODELS, GROQ_BASE_URL };
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
* OpenAI Provider - Handles embedding generation
|
||||
*/
|
||||
|
||||
const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||
const EMBEDDING_DIMENSIONS = 1536;
|
||||
export const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||
export const EMBEDDING_DIMENSIONS = 1536;
|
||||
const MAX_BATCH_SIZE = 2048;
|
||||
|
||||
class OpenAIProvider {
|
||||
export class OpenAIProvider {
|
||||
constructor({ apiKey, baseUrl = 'https://api.openai.com/v1', timeoutMs = 60000 }) {
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key is required');
|
||||
@@ -16,12 +16,6 @@ class OpenAIProvider {
|
||||
this.timeoutMs = timeoutMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for one or more texts
|
||||
* @param {string|string[]} input - Text or array of texts
|
||||
* @param {Object} options
|
||||
* @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>}
|
||||
*/
|
||||
async embed(input, options = {}) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
const model = options.model || EMBEDDING_MODEL;
|
||||
@@ -33,56 +27,39 @@ class OpenAIProvider {
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
|
||||
// Clean and truncate input texts
|
||||
const cleanedTexts = texts.map(t =>
|
||||
const cleanedTexts = texts.map((t) =>
|
||||
(t || '').replace(/\n+/g, ' ').trim().substring(0, 8000)
|
||||
);
|
||||
|
||||
const body = {
|
||||
input: cleanedTexts,
|
||||
model,
|
||||
encoding_format: 'float'
|
||||
};
|
||||
|
||||
// Only embedding-3 models support dimensions parameter
|
||||
if (model.includes('embedding-3')) {
|
||||
body.dimensions = dimensions;
|
||||
}
|
||||
const body = { input: cleanedTexts, model, encoding_format: 'float' };
|
||||
if (model.includes('embedding-3')) body.dimensions = dimensions;
|
||||
|
||||
const response = await this._makeRequest('embeddings', body, timeoutMs);
|
||||
|
||||
// Sort by index to ensure order matches input
|
||||
const sortedData = response.data.sort((a, b) => a.index - b.index);
|
||||
|
||||
return {
|
||||
embeddings: sortedData.map(item => item.embedding),
|
||||
embeddings: sortedData.map((item) => item.embedding),
|
||||
usage: {
|
||||
promptTokens: response.usage?.prompt_tokens || 0,
|
||||
totalTokens: response.usage?.total_tokens || 0
|
||||
totalTokens: response.usage?.total_tokens || 0,
|
||||
},
|
||||
model: response.model || model,
|
||||
latencyMs: Date.now() - started
|
||||
latencyMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generator for processing large batches in chunks
|
||||
*/
|
||||
async *embedBatchChunked(texts, options = {}) {
|
||||
const batchSize = Math.min(options.batchSize || 100, MAX_BATCH_SIZE);
|
||||
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
const chunk = texts.slice(i, i + batchSize);
|
||||
const result = await this.embed(chunk, options);
|
||||
|
||||
yield {
|
||||
embeddings: result.embeddings,
|
||||
startIndex: i,
|
||||
endIndex: i + chunk.length,
|
||||
usage: result.usage,
|
||||
model: result.model,
|
||||
latencyMs: result.latencyMs
|
||||
latencyMs: result.latencyMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -90,28 +67,23 @@ class OpenAIProvider {
|
||||
async _makeRequest(endpoint, body, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`
|
||||
'Authorization': `Bearer ${this.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({}));
|
||||
throw new Error(error.error?.message || `OpenAI API error: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { OpenAIProvider, EMBEDDING_MODEL, EMBEDDING_DIMENSIONS };
|
||||
|
||||
@@ -1,132 +1,84 @@
|
||||
/**
|
||||
* Description Validation Task
|
||||
*
|
||||
* Validates a product description for quality, accuracy, and guideline compliance.
|
||||
* Uses Groq with the larger model for better reasoning about content quality.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
import { MODELS } from '../providers/groqProvider.js';
|
||||
import {
|
||||
loadDescriptionValidationPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
validateRequiredPrompts,
|
||||
} from '../prompts/promptLoader.js';
|
||||
import {
|
||||
buildDescriptionUserPrompt,
|
||||
parseDescriptionResponse
|
||||
} = require('../prompts/descriptionPrompts');
|
||||
parseDescriptionResponse,
|
||||
} from '../prompts/descriptionPrompts.js';
|
||||
|
||||
const TASK_ID = 'validate.description';
|
||||
export const TASK_ID = 'validate.description';
|
||||
|
||||
/**
|
||||
* Create the description validation task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createDescriptionValidationTask() {
|
||||
export function createDescriptionValidationTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Validate product description for quality and guideline compliance',
|
||||
|
||||
/**
|
||||
* Run the description validation
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object} payload.product - Product data
|
||||
* @param {string} payload.product.name - Product name (for context)
|
||||
* @param {string} payload.product.description - Description to validate
|
||||
* @param {string} [payload.product.company_name] - Company name
|
||||
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
|
||||
* @param {string} [payload.product.categories] - Product categories
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { product, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!product?.name && !product?.description) {
|
||||
return {
|
||||
isValid: true,
|
||||
suggestion: null,
|
||||
issues: [],
|
||||
skipped: true,
|
||||
reason: 'No name or description provided'
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
return { isValid: true, suggestion: null, issues: [], skipped: true, reason: 'No name or description provided' };
|
||||
}
|
||||
if (!provider) throw new Error('Groq provider not available');
|
||||
if (!pool) throw new Error('Database pool not available');
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const companyKey = product.company_id || product.company_name || product.company;
|
||||
const prompts = await loadDescriptionValidationPrompts(pool, companyKey);
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'description_validation');
|
||||
|
||||
// Build the user prompt with database-loaded prompts
|
||||
const userPrompt = buildDescriptionUserPrompt(product, prompts);
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - better for content analysis
|
||||
temperature: 0.3, // Slightly higher for creative suggestions
|
||||
maxTokens: 2000, // Reasoning models need extra tokens for thinking
|
||||
responseFormat: { type: 'json_object' }
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.3,
|
||||
maxTokens: 2000,
|
||||
responseFormat: { type: 'json_object' },
|
||||
});
|
||||
|
||||
// Log full raw response for debugging
|
||||
log.info('[DescriptionValidation] Raw AI response:', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
contentLength: response.content?.length,
|
||||
});
|
||||
|
||||
// Parse the response
|
||||
result = parseDescriptionResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[DescriptionValidation] JSON mode failed, attempting to parse failed_generation:', {
|
||||
failedGeneration: jsonError.failedGeneration
|
||||
failedGeneration: jsonError.failedGeneration,
|
||||
});
|
||||
result = parseDescriptionResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[DescriptionValidation] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.3,
|
||||
maxTokens: 2000 // Reasoning models need extra tokens for thinking
|
||||
// No responseFormat - let the model respond freely
|
||||
maxTokens: 2000,
|
||||
});
|
||||
log.info('[DescriptionValidation] Raw AI response (no JSON mode):', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
contentLength: response.content?.length,
|
||||
});
|
||||
result = parseDescriptionResponse(response.parsed, response.content);
|
||||
}
|
||||
@@ -135,24 +87,19 @@ function createDescriptionValidationTask() {
|
||||
log.info(`[DescriptionValidation] Validated description for "${product.name}" in ${response.latencyMs}ms`, {
|
||||
isValid: result.isValid,
|
||||
hasSuggestion: !!result.suggestion,
|
||||
issueCount: result.issues.length
|
||||
issueCount: result.issues.length,
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
model: response.model,
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[DescriptionValidation] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createDescriptionValidationTask
|
||||
};
|
||||
|
||||
@@ -1,166 +1,87 @@
|
||||
/**
|
||||
* AI Task Registry
|
||||
*
|
||||
* Simple registry pattern for AI tasks. Each task has:
|
||||
* - id: Unique identifier
|
||||
* - run: Async function that executes the task
|
||||
*
|
||||
* This allows adding new AI capabilities without modifying core code.
|
||||
*/
|
||||
|
||||
const { createNameValidationTask, TASK_ID: NAME_TASK_ID } = require('./nameValidationTask');
|
||||
const { createDescriptionValidationTask, TASK_ID: DESC_TASK_ID } = require('./descriptionValidationTask');
|
||||
const { createSanityCheckTask, TASK_ID: SANITY_TASK_ID } = require('./sanityCheckTask');
|
||||
import { createNameValidationTask, TASK_ID as NAME_TASK_ID } from './nameValidationTask.js';
|
||||
import { createDescriptionValidationTask, TASK_ID as DESC_TASK_ID } from './descriptionValidationTask.js';
|
||||
import { createSanityCheckTask, TASK_ID as SANITY_TASK_ID } from './sanityCheckTask.js';
|
||||
|
||||
/**
|
||||
* Task IDs - frozen constants for type safety
|
||||
*/
|
||||
const TASK_IDS = Object.freeze({
|
||||
// Inline validation (triggered on field blur)
|
||||
export { createNameValidationTask, createDescriptionValidationTask, createSanityCheckTask };
|
||||
|
||||
export const TASK_IDS = Object.freeze({
|
||||
VALIDATE_NAME: NAME_TASK_ID,
|
||||
VALIDATE_DESCRIPTION: DESC_TASK_ID,
|
||||
|
||||
// Batch operations (triggered on user action)
|
||||
SANITY_CHECK: SANITY_TASK_ID
|
||||
SANITY_CHECK: SANITY_TASK_ID,
|
||||
});
|
||||
|
||||
/**
|
||||
* Task Registry
|
||||
*/
|
||||
class TaskRegistry {
|
||||
export class TaskRegistry {
|
||||
constructor() {
|
||||
this.tasks = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a task
|
||||
* @param {Object} task
|
||||
* @param {string} task.id - Unique task identifier
|
||||
* @param {Function} task.run - Async function: (payload) => result
|
||||
* @param {string} [task.description] - Human-readable description
|
||||
*/
|
||||
register(task) {
|
||||
if (!task?.id) {
|
||||
throw new Error('Task must have an id');
|
||||
}
|
||||
if (typeof task.run !== 'function') {
|
||||
throw new Error(`Task ${task.id} must have a run function`);
|
||||
}
|
||||
if (this.tasks.has(task.id)) {
|
||||
throw new Error(`Task ${task.id} is already registered`);
|
||||
}
|
||||
|
||||
if (!task?.id) throw new Error('Task must have an id');
|
||||
if (typeof task.run !== 'function') throw new Error(`Task ${task.id} must have a run function`);
|
||||
if (this.tasks.has(task.id)) throw new Error(`Task ${task.id} is already registered`);
|
||||
this.tasks.set(task.id, task);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a task by ID
|
||||
* @param {string} taskId
|
||||
* @returns {Object|null}
|
||||
*/
|
||||
get(taskId) {
|
||||
return this.tasks.get(taskId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a task exists
|
||||
* @param {string} taskId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
has(taskId) {
|
||||
return this.tasks.has(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a task by ID
|
||||
* @param {string} taskId
|
||||
* @param {Object} payload - Task-specific input
|
||||
* @returns {Promise<Object>} Task result
|
||||
*/
|
||||
async runTask(taskId, payload = {}) {
|
||||
const task = this.get(taskId);
|
||||
if (!task) {
|
||||
throw new Error(`Unknown task: ${taskId}`);
|
||||
}
|
||||
|
||||
if (!task) throw new Error(`Unknown task: ${taskId}`);
|
||||
try {
|
||||
const result = await task.run(payload);
|
||||
return {
|
||||
success: true,
|
||||
taskId,
|
||||
...result
|
||||
};
|
||||
return { success: true, taskId, ...result };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
taskId,
|
||||
error: error.message,
|
||||
code: error.code
|
||||
code: error.code,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all registered task IDs
|
||||
* @returns {string[]}
|
||||
*/
|
||||
list() {
|
||||
return Array.from(this.tasks.keys());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of registered tasks
|
||||
* @returns {number}
|
||||
*/
|
||||
size() {
|
||||
return this.tasks.size;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
let registry = null;
|
||||
|
||||
/**
|
||||
* Get or create the task registry
|
||||
* @returns {TaskRegistry}
|
||||
*/
|
||||
function getRegistry() {
|
||||
if (!registry) {
|
||||
registry = new TaskRegistry();
|
||||
}
|
||||
export function getRegistry() {
|
||||
if (!registry) registry = new TaskRegistry();
|
||||
return registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the registry (mainly for testing)
|
||||
*/
|
||||
function resetRegistry() {
|
||||
export function resetRegistry() {
|
||||
registry = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all validation tasks with the registry
|
||||
* Call this during initialization after the registry is created
|
||||
*
|
||||
* @param {Object} [logger] - Optional logger
|
||||
*/
|
||||
function registerAllTasks(logger = console) {
|
||||
export function registerAllTasks(logger = console) {
|
||||
const reg = getRegistry();
|
||||
|
||||
// Register name validation
|
||||
if (!reg.has(TASK_IDS.VALIDATE_NAME)) {
|
||||
reg.register(createNameValidationTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_NAME}`);
|
||||
}
|
||||
|
||||
// Register description validation
|
||||
if (!reg.has(TASK_IDS.VALIDATE_DESCRIPTION)) {
|
||||
reg.register(createDescriptionValidationTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_DESCRIPTION}`);
|
||||
}
|
||||
|
||||
// Register sanity check
|
||||
if (!reg.has(TASK_IDS.SANITY_CHECK)) {
|
||||
reg.register(createSanityCheckTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.SANITY_CHECK}`);
|
||||
@@ -168,19 +89,3 @@ function registerAllTasks(logger = console) {
|
||||
|
||||
return reg;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Constants
|
||||
TASK_IDS,
|
||||
|
||||
// Registry
|
||||
TaskRegistry,
|
||||
getRegistry,
|
||||
resetRegistry,
|
||||
registerAllTasks,
|
||||
|
||||
// Task factories (for custom registration)
|
||||
createNameValidationTask,
|
||||
createDescriptionValidationTask,
|
||||
createSanityCheckTask
|
||||
};
|
||||
|
||||
@@ -1,77 +1,38 @@
|
||||
/**
|
||||
* Name Validation Task
|
||||
*
|
||||
* Validates a product name for spelling, grammar, and naming conventions.
|
||||
* Uses Groq with the smaller model for fast response times.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
import { MODELS } from '../providers/groqProvider.js';
|
||||
import {
|
||||
loadNameValidationPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
validateRequiredPrompts,
|
||||
} from '../prompts/promptLoader.js';
|
||||
import {
|
||||
buildNameUserPrompt,
|
||||
parseNameResponse
|
||||
} = require('../prompts/namePrompts');
|
||||
parseNameResponse,
|
||||
} from '../prompts/namePrompts.js';
|
||||
|
||||
const TASK_ID = 'validate.name';
|
||||
export const TASK_ID = 'validate.name';
|
||||
|
||||
/**
|
||||
* Create the name validation task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createNameValidationTask() {
|
||||
export function createNameValidationTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Validate product name for spelling, grammar, and conventions',
|
||||
|
||||
/**
|
||||
* Run the name validation
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object} payload.product - Product data
|
||||
* @param {string} payload.product.name - Product name to validate
|
||||
* @param {string} [payload.product.company_name] - Company name
|
||||
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
|
||||
* @param {string} [payload.product.line_name] - Product line
|
||||
* @param {string} [payload.product.description] - Description for context
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { product, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!product?.name) {
|
||||
return {
|
||||
isValid: true,
|
||||
suggestion: null,
|
||||
issues: [],
|
||||
skipped: true,
|
||||
reason: 'No name provided'
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
return { isValid: true, suggestion: null, issues: [], skipped: true, reason: 'No name provided' };
|
||||
}
|
||||
if (!provider) throw new Error('Groq provider not available');
|
||||
if (!pool) throw new Error('Database pool not available');
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const companyKey = product.company_id || product.company_name || product.company;
|
||||
const prompts = await loadNameValidationPrompts(pool, companyKey);
|
||||
|
||||
// Debug: Log loaded prompts
|
||||
log.info('[NameValidation] Loaded prompts:', {
|
||||
hasSystem: !!prompts.system,
|
||||
systemLength: prompts.system?.length || 0,
|
||||
@@ -79,68 +40,57 @@ function createNameValidationTask() {
|
||||
generalLength: prompts.general?.length || 0,
|
||||
generalPreview: prompts.general?.substring(0, 100) || '(empty)',
|
||||
hasCompanySpecific: !!prompts.companySpecific,
|
||||
companyKey
|
||||
companyKey,
|
||||
});
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'name_validation');
|
||||
|
||||
// Build the user prompt with database-loaded prompts
|
||||
const userPrompt = buildNameUserPrompt(product, prompts);
|
||||
|
||||
// Debug: Log the full user prompt being sent
|
||||
log.info('[NameValidation] User prompt:', userPrompt.substring(0, 500));
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - reasoning model
|
||||
temperature: 0.2, // Low temperature for consistent results
|
||||
maxTokens: 3000, // Reasoning models need extra tokens for thinking
|
||||
responseFormat: { type: 'json_object' }
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.2,
|
||||
maxTokens: 3000,
|
||||
responseFormat: { type: 'json_object' },
|
||||
});
|
||||
|
||||
// Log full raw response for debugging
|
||||
log.info('[NameValidation] Raw AI response:', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
contentLength: response.content?.length,
|
||||
});
|
||||
|
||||
// Parse the response
|
||||
result = parseNameResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[NameValidation] JSON mode failed, attempting to parse failed_generation:', {
|
||||
failedGeneration: jsonError.failedGeneration
|
||||
failedGeneration: jsonError.failedGeneration,
|
||||
});
|
||||
result = parseNameResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.SMALL };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[NameValidation] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.SMALL,
|
||||
temperature: 0.2,
|
||||
maxTokens: 1500 // Reasoning models need extra tokens for thinking
|
||||
// No responseFormat - let the model respond freely
|
||||
maxTokens: 1500,
|
||||
});
|
||||
log.info('[NameValidation] Raw AI response (no JSON mode):', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
contentLength: response.content?.length,
|
||||
});
|
||||
result = parseNameResponse(response.parsed, response.content);
|
||||
}
|
||||
@@ -149,24 +99,19 @@ function createNameValidationTask() {
|
||||
log.info(`[NameValidation] Validated "${product.name}" in ${response.latencyMs}ms`, {
|
||||
isValid: result.isValid,
|
||||
hassuggestion: !!result.suggestion,
|
||||
issueCount: result.issues.length
|
||||
issueCount: result.issues.length,
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
model: response.model,
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[NameValidation] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createNameValidationTask
|
||||
};
|
||||
|
||||
@@ -1,96 +1,55 @@
|
||||
/**
|
||||
* Sanity Check Task
|
||||
*
|
||||
* Reviews a batch of products for consistency and appropriateness.
|
||||
* Uses Groq with the larger model for complex batch analysis.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
import { MODELS } from '../providers/groqProvider.js';
|
||||
import {
|
||||
loadSanityCheckPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
validateRequiredPrompts,
|
||||
} from '../prompts/promptLoader.js';
|
||||
import {
|
||||
buildSanityCheckUserPrompt,
|
||||
parseSanityCheckResponse
|
||||
} = require('../prompts/sanityCheckPrompts');
|
||||
parseSanityCheckResponse,
|
||||
} from '../prompts/sanityCheckPrompts.js';
|
||||
|
||||
const TASK_ID = 'sanity.check';
|
||||
export const TASK_ID = 'sanity.check';
|
||||
export const MAX_PRODUCTS_PER_REQUEST = 50;
|
||||
|
||||
// Maximum products to send in a single request (to avoid token limits)
|
||||
const MAX_PRODUCTS_PER_REQUEST = 50;
|
||||
|
||||
/**
|
||||
* Create the sanity check task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createSanityCheckTask() {
|
||||
export function createSanityCheckTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Review batch of products for consistency and appropriateness',
|
||||
|
||||
/**
|
||||
* Run the sanity check
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object[]} payload.products - Array of products to check
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { products, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!Array.isArray(products) || products.length === 0) {
|
||||
return {
|
||||
issues: [],
|
||||
summary: 'No products to check',
|
||||
skipped: true
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
return { issues: [], summary: 'No products to check', skipped: true };
|
||||
}
|
||||
if (!provider) throw new Error('Groq provider not available');
|
||||
if (!pool) throw new Error('Database pool not available');
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const prompts = await loadSanityCheckPrompts(pool);
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'sanity_check');
|
||||
|
||||
// If batch is small enough, process in one request
|
||||
if (products.length <= MAX_PRODUCTS_PER_REQUEST) {
|
||||
return await checkBatch(products, prompts, provider, log);
|
||||
}
|
||||
|
||||
// Otherwise, process in chunks and combine results
|
||||
log.info(`[SanityCheck] Processing ${products.length} products in chunks`);
|
||||
const allIssues = [];
|
||||
const summaries = [];
|
||||
|
||||
for (let i = 0; i < products.length; i += MAX_PRODUCTS_PER_REQUEST) {
|
||||
const chunk = products.slice(i, i + MAX_PRODUCTS_PER_REQUEST);
|
||||
const chunkOffset = i; // To adjust product indices in results
|
||||
|
||||
const chunkOffset = i;
|
||||
const result = await checkBatch(chunk, prompts, provider, log);
|
||||
|
||||
// Adjust product indices to match original array
|
||||
const adjustedIssues = result.issues.map(issue => ({
|
||||
const adjustedIssues = result.issues.map((issue) => ({
|
||||
...issue,
|
||||
productIndex: issue.productIndex + chunkOffset
|
||||
productIndex: issue.productIndex + chunkOffset,
|
||||
}));
|
||||
|
||||
allIssues.push(...adjustedIssues);
|
||||
summaries.push(result.summary);
|
||||
}
|
||||
@@ -101,82 +60,61 @@ function createSanityCheckTask() {
|
||||
? `Reviewed ${products.length} products in ${summaries.length} batches. ${allIssues.length} issues found.`
|
||||
: summaries[0],
|
||||
totalProducts: products.length,
|
||||
issueCount: allIssues.length
|
||||
issueCount: allIssues.length,
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[SanityCheck] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a single batch of products
|
||||
*
|
||||
* @param {Object[]} products - Products to check
|
||||
* @param {Object} prompts - Loaded prompts from database
|
||||
* @param {Object} provider - Groq provider
|
||||
* @param {Object} log - Logger
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async function checkBatch(products, prompts, provider, log) {
|
||||
const userPrompt = buildSanityCheckUserPrompt(products, prompts);
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - needed for complex batch analysis
|
||||
temperature: 0.2, // Low temperature for consistent analysis
|
||||
maxTokens: 2000, // More tokens for batch results
|
||||
responseFormat: { type: 'json_object' }
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.2,
|
||||
maxTokens: 2000,
|
||||
responseFormat: { type: 'json_object' },
|
||||
});
|
||||
|
||||
result = parseSanityCheckResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[SanityCheck] JSON mode failed, attempting to parse failed_generation');
|
||||
result = parseSanityCheckResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[SanityCheck] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.2,
|
||||
maxTokens: 2000
|
||||
// No responseFormat - let the model respond freely
|
||||
maxTokens: 2000,
|
||||
});
|
||||
result = parseSanityCheckResponse(response.parsed, response.content);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[SanityCheck] Checked ${products.length} products in ${response.latencyMs}ms`, {
|
||||
issueCount: result.issues.length
|
||||
issueCount: result.issues.length,
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
model: response.model,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createSanityCheckTask,
|
||||
MAX_PRODUCTS_PER_REQUEST
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user