Restore accidentally removed files, a few forecast tweaks
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Vector similarity utilities
|
||||
*/
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two vectors
|
||||
* @param {number[]} a
|
||||
* @param {number[]} b
|
||||
* @returns {number} Similarity score between -1 and 1
|
||||
*/
|
||||
function cosineSimilarity(a, b) {
|
||||
if (!a || !b || a.length !== b.length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) return 0;
|
||||
|
||||
return dotProduct / denominator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find top K most similar items from a collection
|
||||
* @param {number[]} queryEmbedding - The embedding to search for
|
||||
* @param {Array<{id: any, embedding: number[]}>} items - Items with embeddings
|
||||
* @param {number} topK - Number of results to return
|
||||
* @returns {Array<{id: any, similarity: number}>}
|
||||
*/
|
||||
function findTopMatches(queryEmbedding, items, topK = 10) {
|
||||
if (!queryEmbedding || !items || items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const scored = items.map(item => ({
|
||||
id: item.id,
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding)
|
||||
}));
|
||||
|
||||
scored.sort((a, b) => b.similarity - a.similarity);
|
||||
|
||||
return scored.slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find matches above a similarity threshold
|
||||
* @param {number[]} queryEmbedding
|
||||
* @param {Array<{id: any, embedding: number[]}>} items
|
||||
* @param {number} threshold - Minimum similarity (0-1)
|
||||
* @returns {Array<{id: any, similarity: number}>}
|
||||
*/
|
||||
function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
|
||||
if (!queryEmbedding || !items || items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const scored = items
|
||||
.map(item => ({
|
||||
id: item.id,
|
||||
similarity: cosineSimilarity(queryEmbedding, item.embedding)
|
||||
}))
|
||||
.filter(item => item.similarity >= threshold);
|
||||
|
||||
scored.sort((a, b) => b.similarity - a.similarity);
|
||||
|
||||
return scored;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cosineSimilarity,
|
||||
findTopMatches,
|
||||
findMatchesAboveThreshold
|
||||
};
|
||||
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Taxonomy Embedding Service
|
||||
*
|
||||
* Generates and caches embeddings for categories, themes, and colors.
|
||||
* Excludes "Black Friday", "Gifts", "Deals" categories and their children.
|
||||
*/
|
||||
|
||||
const { findTopMatches } = require('./similarity');
|
||||
|
||||
// Categories to exclude (and all their children)
|
||||
const EXCLUDED_CATEGORY_NAMES = ['black friday', 'gifts', 'deals'];
|
||||
|
||||
class TaxonomyEmbeddings {
|
||||
constructor({ provider, logger }) {
|
||||
this.provider = provider;
|
||||
this.logger = logger || console;
|
||||
|
||||
// Cached taxonomy with embeddings
|
||||
this.categories = [];
|
||||
this.themes = [];
|
||||
this.colors = [];
|
||||
|
||||
// Raw data without embeddings (for lookup)
|
||||
this.categoryMap = new Map();
|
||||
this.themeMap = new Map();
|
||||
this.colorMap = new Map();
|
||||
|
||||
this.initialized = false;
|
||||
this.initializing = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize embeddings - fetch taxonomy and generate embeddings
|
||||
*/
|
||||
async initialize(connection) {
|
||||
if (this.initialized) {
|
||||
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
|
||||
}
|
||||
|
||||
if (this.initializing) {
|
||||
// Wait for existing initialization
|
||||
while (this.initializing) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
|
||||
}
|
||||
|
||||
this.initializing = true;
|
||||
|
||||
try {
|
||||
this.logger.info('[TaxonomyEmbeddings] Starting initialization...');
|
||||
|
||||
// Fetch raw taxonomy data
|
||||
const [categories, themes, colors] = await Promise.all([
|
||||
this._fetchCategories(connection),
|
||||
this._fetchThemes(connection),
|
||||
this._fetchColors(connection)
|
||||
]);
|
||||
|
||||
this.logger.info(`[TaxonomyEmbeddings] Fetched ${categories.length} categories, ${themes.length} themes, ${colors.length} colors`);
|
||||
|
||||
// Generate embeddings in parallel
|
||||
const [catEmbeddings, themeEmbeddings, colorEmbeddings] = await Promise.all([
|
||||
this._generateEmbeddings(categories, 'categories'),
|
||||
this._generateEmbeddings(themes, 'themes'),
|
||||
this._generateEmbeddings(colors, 'colors')
|
||||
]);
|
||||
|
||||
// Store with embeddings
|
||||
this.categories = catEmbeddings;
|
||||
this.themes = themeEmbeddings;
|
||||
this.colors = colorEmbeddings;
|
||||
|
||||
// Build lookup maps
|
||||
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
|
||||
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
|
||||
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
|
||||
|
||||
this.initialized = true;
|
||||
this.logger.info('[TaxonomyEmbeddings] Initialization complete');
|
||||
|
||||
return {
|
||||
categories: this.categories.length,
|
||||
themes: this.themes.length,
|
||||
colors: this.colors.length
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error('[TaxonomyEmbeddings] Initialization failed:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
this.initializing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar categories for a product embedding
|
||||
*/
|
||||
findSimilarCategories(productEmbedding, topK = 10) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const matches = findTopMatches(productEmbedding, this.categories, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
const cat = this.categoryMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: cat?.name || '',
|
||||
fullPath: cat?.fullPath || '',
|
||||
similarity: match.similarity
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar themes for a product embedding
|
||||
*/
|
||||
findSimilarThemes(productEmbedding, topK = 5) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const matches = findTopMatches(productEmbedding, this.themes, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
const theme = this.themeMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: theme?.name || '',
|
||||
fullPath: theme?.fullPath || '',
|
||||
similarity: match.similarity
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar colors for a product embedding
|
||||
*/
|
||||
findSimilarColors(productEmbedding, topK = 5) {
|
||||
if (!this.initialized || !productEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const matches = findTopMatches(productEmbedding, this.colors, topK);
|
||||
|
||||
return matches.map(match => {
|
||||
const color = this.colorMap.get(match.id);
|
||||
return {
|
||||
id: match.id,
|
||||
name: color?.name || '',
|
||||
similarity: match.similarity
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all taxonomy data (without embeddings) for frontend
|
||||
*/
|
||||
getTaxonomyData() {
|
||||
return {
|
||||
categories: this.categories.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
|
||||
themes: this.themes.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
|
||||
colors: this.colors.map(({ id, name }) => ({ id, name }))
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if service is ready
|
||||
*/
|
||||
isReady() {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods
|
||||
// ============================================================================
|
||||
|
||||
async _fetchCategories(connection) {
|
||||
// Fetch hierarchical categories (types 10-13)
|
||||
const [rows] = await connection.query(`
|
||||
SELECT cat_id, name, master_cat_id, type
|
||||
FROM product_categories
|
||||
WHERE type IN (10, 11, 12, 13)
|
||||
ORDER BY type, name
|
||||
`);
|
||||
|
||||
// Build lookup for hierarchy
|
||||
const byId = new Map(rows.map(r => [r.cat_id, r]));
|
||||
|
||||
// Find IDs of excluded top-level categories and all their descendants
|
||||
const excludedIds = new Set();
|
||||
|
||||
// First pass: find excluded top-level categories
|
||||
for (const row of rows) {
|
||||
if (row.type === 10 && EXCLUDED_CATEGORY_NAMES.includes(row.name.toLowerCase())) {
|
||||
excludedIds.add(row.cat_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Multiple passes to find all descendants
|
||||
let foundNew = true;
|
||||
while (foundNew) {
|
||||
foundNew = false;
|
||||
for (const row of rows) {
|
||||
if (!excludedIds.has(row.cat_id) && excludedIds.has(row.master_cat_id)) {
|
||||
excludedIds.add(row.cat_id);
|
||||
foundNew = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info(`[TaxonomyEmbeddings] Excluding ${excludedIds.size} categories (Black Friday, Gifts, Deals and children)`);
|
||||
|
||||
// Build category objects with full paths, excluding filtered ones
|
||||
const categories = [];
|
||||
|
||||
for (const row of rows) {
|
||||
if (excludedIds.has(row.cat_id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const path = [];
|
||||
let current = row;
|
||||
|
||||
// Walk up the tree to build full path
|
||||
while (current) {
|
||||
path.unshift(current.name);
|
||||
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
|
||||
}
|
||||
|
||||
categories.push({
|
||||
id: row.cat_id,
|
||||
name: row.name,
|
||||
parentId: row.master_cat_id,
|
||||
type: row.type,
|
||||
fullPath: path.join(' > '),
|
||||
embeddingText: path.join(' ')
|
||||
});
|
||||
}
|
||||
|
||||
return categories;
|
||||
}
|
||||
|
||||
async _fetchThemes(connection) {
|
||||
// Fetch themes (types 20-21)
|
||||
const [rows] = await connection.query(`
|
||||
SELECT cat_id, name, master_cat_id, type
|
||||
FROM product_categories
|
||||
WHERE type IN (20, 21)
|
||||
ORDER BY type, name
|
||||
`);
|
||||
|
||||
const byId = new Map(rows.map(r => [r.cat_id, r]));
|
||||
const themes = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const path = [];
|
||||
let current = row;
|
||||
|
||||
while (current) {
|
||||
path.unshift(current.name);
|
||||
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
|
||||
}
|
||||
|
||||
themes.push({
|
||||
id: row.cat_id,
|
||||
name: row.name,
|
||||
parentId: row.master_cat_id,
|
||||
type: row.type,
|
||||
fullPath: path.join(' > '),
|
||||
embeddingText: path.join(' ')
|
||||
});
|
||||
}
|
||||
|
||||
return themes;
|
||||
}
|
||||
|
||||
async _fetchColors(connection) {
|
||||
const [rows] = await connection.query(`
|
||||
SELECT color, name, hex_color
|
||||
FROM product_color_list
|
||||
ORDER BY \`order\`
|
||||
`);
|
||||
|
||||
return rows.map(row => ({
|
||||
id: row.color,
|
||||
name: row.name,
|
||||
hexColor: row.hex_color,
|
||||
embeddingText: row.name
|
||||
}));
|
||||
}
|
||||
|
||||
async _generateEmbeddings(items, label) {
|
||||
if (items.length === 0) {
|
||||
return items;
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
const texts = items.map(item => item.embeddingText);
|
||||
const results = [...items];
|
||||
|
||||
// Process in batches
|
||||
let batchNum = 0;
|
||||
for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) {
|
||||
batchNum++;
|
||||
for (let i = 0; i < chunk.embeddings.length; i++) {
|
||||
const globalIndex = chunk.startIndex + i;
|
||||
results[globalIndex] = {
|
||||
...results[globalIndex],
|
||||
embedding: chunk.embeddings[i]
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logger.info(`[TaxonomyEmbeddings] Generated ${items.length} ${label} embeddings in ${elapsed}ms`);
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { TaxonomyEmbeddings };
|
||||
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* AI Service
|
||||
*
|
||||
* Main entry point for AI functionality including:
|
||||
* - Embeddings for taxonomy suggestions (OpenAI)
|
||||
* - Chat completions for validation tasks (Groq)
|
||||
* - Task registry for AI operations
|
||||
*/
|
||||
|
||||
const { OpenAIProvider } = require('./providers/openaiProvider');
|
||||
const { GroqProvider, MODELS: GROQ_MODELS } = require('./providers/groqProvider');
|
||||
const { TaxonomyEmbeddings } = require('./embeddings/taxonomyEmbeddings');
|
||||
const { cosineSimilarity, findTopMatches } = require('./embeddings/similarity');
|
||||
const { getRegistry, TASK_IDS, registerAllTasks } = require('./tasks');
|
||||
|
||||
let initialized = false;
|
||||
let initializing = false;
|
||||
let openaiProvider = null;
|
||||
let groqProvider = null;
|
||||
let taxonomyEmbeddings = null;
|
||||
let logger = console;
|
||||
|
||||
// Store pool reference for task access
|
||||
let appPool = null;
|
||||
|
||||
/**
|
||||
* Initialize the AI service
|
||||
* @param {Object} options
|
||||
* @param {string} options.openaiApiKey - OpenAI API key (for embeddings)
|
||||
* @param {string} [options.groqApiKey] - Groq API key (for chat completions)
|
||||
* @param {Object} options.mysqlConnection - MySQL connection for taxonomy data
|
||||
* @param {Object} [options.pool] - PostgreSQL pool for prompt loading
|
||||
* @param {Object} [options.logger] - Logger instance
|
||||
*/
|
||||
async function initialize({ openaiApiKey, groqApiKey, mysqlConnection, pool, logger: customLogger }) {
|
||||
if (initialized) {
|
||||
return { success: true, message: 'Already initialized' };
|
||||
}
|
||||
|
||||
if (initializing) {
|
||||
// Wait for existing initialization
|
||||
while (initializing) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
return { success: initialized, message: initialized ? 'Initialized' : 'Initialization failed' };
|
||||
}
|
||||
|
||||
initializing = true;
|
||||
|
||||
try {
|
||||
if (customLogger) {
|
||||
logger = customLogger;
|
||||
}
|
||||
|
||||
if (!openaiApiKey) {
|
||||
throw new Error('OpenAI API key is required');
|
||||
}
|
||||
|
||||
logger.info('[AI] Initializing AI service...');
|
||||
|
||||
// Store pool reference for tasks
|
||||
if (pool) {
|
||||
appPool = pool;
|
||||
}
|
||||
|
||||
// Create OpenAI provider (for embeddings)
|
||||
openaiProvider = new OpenAIProvider({ apiKey: openaiApiKey });
|
||||
|
||||
// Create Groq provider (for chat completions) if API key provided
|
||||
if (groqApiKey) {
|
||||
groqProvider = new GroqProvider({ apiKey: groqApiKey });
|
||||
logger.info('[AI] Groq provider initialized for chat completions');
|
||||
} else {
|
||||
logger.warn('[AI] No Groq API key provided - chat completion tasks will not be available');
|
||||
}
|
||||
|
||||
// Create and initialize taxonomy embeddings
|
||||
taxonomyEmbeddings = new TaxonomyEmbeddings({
|
||||
provider: openaiProvider,
|
||||
logger
|
||||
});
|
||||
|
||||
const stats = await taxonomyEmbeddings.initialize(mysqlConnection);
|
||||
|
||||
// Register validation tasks if Groq is available
|
||||
if (groqProvider) {
|
||||
registerValidationTasks();
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
logger.info('[AI] AI service initialized', {
|
||||
...stats,
|
||||
groqEnabled: !!groqProvider,
|
||||
tasksRegistered: getRegistry().list()
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Initialized',
|
||||
stats,
|
||||
groqEnabled: !!groqProvider
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('[AI] Initialization failed:', error);
|
||||
return { success: false, message: error.message };
|
||||
} finally {
|
||||
initializing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register validation tasks with the task registry
|
||||
* Called during initialization if Groq is available
|
||||
*/
|
||||
function registerValidationTasks() {
|
||||
registerAllTasks(logger);
|
||||
logger.info('[AI] Validation tasks registered');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if service is ready
|
||||
*/
|
||||
function isReady() {
|
||||
return initialized && taxonomyEmbeddings?.isReady();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build weighted product text for embedding.
|
||||
* Weights the product name heavily by repeating it, and truncates long descriptions
|
||||
* to prevent verbose marketing copy from drowning out the product signal.
|
||||
*
|
||||
* @param {Object} product - Product with name, description, company, line
|
||||
* @returns {string} - Combined text for embedding
|
||||
*/
|
||||
function buildProductText(product) {
|
||||
const parts = [];
|
||||
const name = product.name?.trim();
|
||||
const description = product.description?.trim();
|
||||
const company = (product.company_name || product.company)?.trim();
|
||||
const line = (product.line_name || product.line)?.trim();
|
||||
|
||||
// Name is most important - repeat 3x to weight it heavily in the embedding
|
||||
if (name) {
|
||||
parts.push(name, name, name);
|
||||
}
|
||||
|
||||
// Company and line provide context
|
||||
if (company) {
|
||||
parts.push(company);
|
||||
}
|
||||
if (line) {
|
||||
parts.push(line);
|
||||
}
|
||||
|
||||
// Truncate description to prevent it from overwhelming the signal
|
||||
if (description) {
|
||||
const truncated = description.length > 500
|
||||
? description.substring(0, 500) + '...'
|
||||
: description;
|
||||
parts.push(truncated);
|
||||
}
|
||||
|
||||
return parts.join(' ').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for a product
|
||||
* @param {Object} product - Product with name, description, company, line
|
||||
* @returns {Promise<{embedding: number[], latencyMs: number}>}
|
||||
*/
|
||||
async function getProductEmbedding(product) {
|
||||
if (!initialized || !openaiProvider) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
const text = buildProductText(product);
|
||||
|
||||
if (!text) {
|
||||
return { embedding: null, latencyMs: 0 };
|
||||
}
|
||||
|
||||
const result = await openaiProvider.embed(text);
|
||||
|
||||
return {
|
||||
embedding: result.embeddings[0],
|
||||
latencyMs: result.latencyMs
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for multiple products
|
||||
* @param {Object[]} products - Array of products
|
||||
* @returns {Promise<{embeddings: Array<{index: number, embedding: number[]}>, latencyMs: number}>}
|
||||
*/
|
||||
async function getProductEmbeddings(products) {
|
||||
if (!initialized || !openaiProvider) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
const texts = products.map(buildProductText);
|
||||
|
||||
// Track which products have empty text
|
||||
const validIndices = texts.map((t, i) => t ? i : -1).filter(i => i >= 0);
|
||||
const validTexts = texts.filter(t => t);
|
||||
|
||||
if (validTexts.length === 0) {
|
||||
return { embeddings: [], latencyMs: 0 };
|
||||
}
|
||||
|
||||
const result = await openaiProvider.embed(validTexts);
|
||||
|
||||
// Map embeddings back to original indices
|
||||
const embeddings = validIndices.map((originalIndex, resultIndex) => ({
|
||||
index: originalIndex,
|
||||
embedding: result.embeddings[resultIndex]
|
||||
}));
|
||||
|
||||
return {
|
||||
embeddings,
|
||||
latencyMs: result.latencyMs
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar taxonomy items for a product embedding
|
||||
* @param {number[]} productEmbedding
|
||||
* @param {Object} options
|
||||
* @returns {{categories: Array, themes: Array, colors: Array}}
|
||||
*/
|
||||
function findSimilarTaxonomy(productEmbedding, options = {}) {
|
||||
if (!initialized || !taxonomyEmbeddings) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
const topCategories = options.topCategories ?? 10;
|
||||
const topThemes = options.topThemes ?? 5;
|
||||
const topColors = options.topColors ?? 5;
|
||||
|
||||
return {
|
||||
categories: taxonomyEmbeddings.findSimilarCategories(productEmbedding, topCategories),
|
||||
themes: taxonomyEmbeddings.findSimilarThemes(productEmbedding, topThemes),
|
||||
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product embedding and find similar taxonomy in one call
|
||||
* @param {Object} product
|
||||
* @param {Object} options
|
||||
*/
|
||||
async function getSuggestionsForProduct(product, options = {}) {
|
||||
const { embedding, latencyMs: embeddingLatency } = await getProductEmbedding(product);
|
||||
|
||||
if (!embedding) {
|
||||
return {
|
||||
categories: [],
|
||||
themes: [],
|
||||
colors: [],
|
||||
latencyMs: embeddingLatency
|
||||
};
|
||||
}
|
||||
|
||||
const startSearch = Date.now();
|
||||
const suggestions = findSimilarTaxonomy(embedding, options);
|
||||
const searchLatency = Date.now() - startSearch;
|
||||
|
||||
return {
|
||||
...suggestions,
|
||||
latencyMs: embeddingLatency + searchLatency,
|
||||
embeddingLatencyMs: embeddingLatency,
|
||||
searchLatencyMs: searchLatency
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all taxonomy data (without embeddings) for frontend
|
||||
*/
|
||||
function getTaxonomyData() {
|
||||
if (!initialized || !taxonomyEmbeddings) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
return taxonomyEmbeddings.getTaxonomyData();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get service status
|
||||
*/
|
||||
function getStatus() {
|
||||
const registry = getRegistry();
|
||||
|
||||
return {
|
||||
initialized,
|
||||
ready: isReady(),
|
||||
hasOpenAI: !!openaiProvider,
|
||||
hasGroq: !!groqProvider,
|
||||
hasTaxonomy: !!taxonomyEmbeddings,
|
||||
taxonomyStats: taxonomyEmbeddings ? {
|
||||
categories: taxonomyEmbeddings.categories?.length || 0,
|
||||
themes: taxonomyEmbeddings.themes?.length || 0,
|
||||
colors: taxonomyEmbeddings.colors?.length || 0
|
||||
} : null,
|
||||
tasks: {
|
||||
registered: registry.list(),
|
||||
count: registry.size()
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an AI task by ID
|
||||
* @param {string} taskId - Task identifier from TASK_IDS
|
||||
* @param {Object} payload - Task-specific input
|
||||
* @returns {Promise<Object>} Task result
|
||||
*/
|
||||
async function runTask(taskId, payload = {}) {
|
||||
if (!initialized) {
|
||||
throw new Error('AI service not initialized');
|
||||
}
|
||||
|
||||
if (!groqProvider) {
|
||||
throw new Error('Groq provider not available - chat completion tasks require GROQ_API_KEY');
|
||||
}
|
||||
|
||||
const registry = getRegistry();
|
||||
return registry.runTask(taskId, {
|
||||
...payload,
|
||||
// Inject dependencies tasks may need
|
||||
provider: groqProvider,
|
||||
// Use pool from payload if provided (from route), fall back to stored appPool
|
||||
pool: payload.pool || appPool,
|
||||
logger
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Groq provider instance (for direct use if needed)
|
||||
* @returns {GroqProvider|null}
|
||||
*/
|
||||
function getGroqProvider() {
|
||||
return groqProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the PostgreSQL pool (for tasks that need DB access)
|
||||
* @returns {Object|null}
|
||||
*/
|
||||
function getPool() {
|
||||
return appPool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if chat completion tasks are available
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function hasChatCompletion() {
|
||||
return !!groqProvider;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Initialization
|
||||
initialize,
|
||||
isReady,
|
||||
getStatus,
|
||||
|
||||
// Embeddings (OpenAI)
|
||||
getProductEmbedding,
|
||||
getProductEmbeddings,
|
||||
findSimilarTaxonomy,
|
||||
getSuggestionsForProduct,
|
||||
getTaxonomyData,
|
||||
|
||||
// Chat completions (Groq)
|
||||
runTask,
|
||||
hasChatCompletion,
|
||||
getGroqProvider,
|
||||
getPool,
|
||||
|
||||
// Constants
|
||||
TASK_IDS,
|
||||
GROQ_MODELS,
|
||||
|
||||
// Re-export utilities
|
||||
cosineSimilarity,
|
||||
findTopMatches
|
||||
};
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Description Validation Prompts
|
||||
*
|
||||
* Functions for building and parsing description validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sanitize an issue string from AI response
|
||||
* AI sometimes returns malformed strings with escape sequences
|
||||
*
|
||||
* @param {string} issue - Raw issue string
|
||||
* @returns {string} Cleaned issue string
|
||||
*/
|
||||
function sanitizeIssue(issue) {
|
||||
if (!issue || typeof issue !== 'string') return '';
|
||||
|
||||
let cleaned = issue
|
||||
// Remove trailing backslashes (incomplete escapes)
|
||||
.replace(/\\+$/, '')
|
||||
// Fix malformed escaped quotes at end of string
|
||||
.replace(/\\",?\)?$/, '')
|
||||
// Clean up double-escaped quotes
|
||||
.replace(/\\\\"/g, '"')
|
||||
// Clean up single escaped quotes that aren't needed
|
||||
.replace(/\\"/g, '"')
|
||||
// Remove any remaining trailing punctuation artifacts
|
||||
.replace(/[,\s]+$/, '')
|
||||
// Trim whitespace
|
||||
.trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt for description validation
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object} product - Product data
|
||||
* @param {string} product.name - Product name
|
||||
* @param {string} product.description - Current description
|
||||
* @param {string} [product.company_name] - Company name
|
||||
* @param {string} [product.categories] - Product categories
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General description guidelines
|
||||
* @param {string} [prompts.companySpecific] - Company-specific rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildDescriptionUserPrompt(product, prompts) {
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/guidelines if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
}
|
||||
|
||||
// Add company-specific rules if provided
|
||||
if (prompts.companySpecific) {
|
||||
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
|
||||
parts.push(prompts.companySpecific);
|
||||
parts.push(''); // Empty line for separation
|
||||
}
|
||||
|
||||
// Add product information
|
||||
parts.push('PRODUCT TO VALIDATE:');
|
||||
parts.push(`NAME: "${product.name || ''}"`);
|
||||
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
|
||||
|
||||
if (product.categories) {
|
||||
parts.push(`CATEGORIES: ${product.categories}`);
|
||||
}
|
||||
|
||||
parts.push('');
|
||||
parts.push('CURRENT DESCRIPTION:');
|
||||
parts.push(`"${product.description || '(empty)'}"`);
|
||||
|
||||
// Add response format instructions
|
||||
parts.push('');
|
||||
parts.push('CRITICAL RULES:');
|
||||
parts.push('- If isValid is false, you MUST provide a suggestion with the improved description');
|
||||
parts.push('- If there are ANY issues, isValid MUST be false and suggestion MUST contain the corrected text');
|
||||
parts.push('- Only set isValid to true if there are ZERO issues and the description needs no changes');
|
||||
parts.push('');
|
||||
parts.push('RESPOND WITH JSON:');
|
||||
parts.push(JSON.stringify({
|
||||
isValid: 'true if perfect, false if ANY changes needed',
|
||||
suggestion: 'REQUIRED when isValid is false - the complete improved description',
|
||||
issues: ['list each problem found (empty array only if isValid is true)']
|
||||
}, null, 2));
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for description validation
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseDescriptionResponse(parsed, content) {
|
||||
// If we got valid parsed JSON, use it
|
||||
if (parsed && typeof parsed.isValid === 'boolean') {
|
||||
// Sanitize issues - AI sometimes returns malformed escape sequences
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
|
||||
const suggestion = parsed.suggestion || null;
|
||||
|
||||
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
|
||||
// If there are issues, treat as invalid regardless of what the AI said.
|
||||
// Also if there's a suggestion, the AI thought something needed to change.
|
||||
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Handle case where isValid is a string "true"/"false" instead of boolean
|
||||
if (parsed && typeof parsed.isValid === 'string') {
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const isValid = rawIsValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Look for isValid pattern
|
||||
const isValidMatch = content.match(/"isValid"\s*:\s*(true|false)/i);
|
||||
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
|
||||
|
||||
// Look for suggestion (might be multiline)
|
||||
const suggestionMatch = content.match(/"suggestion"\s*:\s*"((?:[^"\\]|\\.)*)"/s);
|
||||
let suggestion = suggestionMatch ? suggestionMatch[1] : null;
|
||||
if (suggestion) {
|
||||
// Unescape common escapes
|
||||
suggestion = suggestion.replace(/\\n/g, '\n').replace(/\\"/g, '"');
|
||||
}
|
||||
|
||||
// Look for issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
if (issuesMatch) {
|
||||
const issuesContent = issuesMatch[1];
|
||||
const issueStrings = issuesContent.match(/"([^"]+)"/g);
|
||||
if (issueStrings) {
|
||||
issues = issueStrings
|
||||
.map(s => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter(issue => issue.length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Same logic: if there are issues, it's not valid
|
||||
const finalIsValid = isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid: finalIsValid, suggestion, issues };
|
||||
} catch {
|
||||
// Default to valid if we can't parse anything
|
||||
return { isValid: true, suggestion: null, issues: [] };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildDescriptionUserPrompt,
|
||||
parseDescriptionResponse
|
||||
};
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Name Validation Prompts
|
||||
*
|
||||
* Functions for building and parsing name validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sanitize an issue string from AI response
|
||||
* AI sometimes returns malformed strings with escape sequences
|
||||
*
|
||||
* @param {string} issue - Raw issue string
|
||||
* @returns {string} Cleaned issue string
|
||||
*/
|
||||
function sanitizeIssue(issue) {
|
||||
if (!issue || typeof issue !== 'string') return '';
|
||||
|
||||
let cleaned = issue
|
||||
// Remove trailing backslashes (incomplete escapes)
|
||||
.replace(/\\+$/, '')
|
||||
// Fix malformed escaped quotes at end of string
|
||||
.replace(/\\",?\)?$/, '')
|
||||
// Clean up double-escaped quotes
|
||||
.replace(/\\\\"/g, '"')
|
||||
// Clean up single escaped quotes that aren't needed
|
||||
.replace(/\\"/g, '"')
|
||||
// Remove any remaining trailing punctuation artifacts
|
||||
.replace(/[,\s]+$/, '')
|
||||
// Trim whitespace
|
||||
.trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt for name validation
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object} product - Product data
|
||||
* @param {string} product.name - Current product name
|
||||
* @param {string} [product.company_name] - Company name
|
||||
* @param {string} [product.line_name] - Product line name
|
||||
* @param {string} [product.subline_name] - Product subline name
|
||||
* @param {string[]} [product.siblingNames] - Names of other products in the same line
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General naming conventions
|
||||
* @param {string} [prompts.companySpecific] - Company-specific rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildNameUserPrompt(product, prompts) {
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/conventions if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
}
|
||||
|
||||
// Add company-specific rules if provided
|
||||
if (prompts.companySpecific) {
|
||||
parts.push(`COMPANY-SPECIFIC RULES FOR ${product.company_name || 'THIS COMPANY'}:`);
|
||||
parts.push(prompts.companySpecific);
|
||||
parts.push(''); // Empty line for separation
|
||||
}
|
||||
|
||||
// Add product information
|
||||
parts.push('PRODUCT TO VALIDATE:');
|
||||
parts.push(`NAME: "${product.name || ''}"`);
|
||||
parts.push(`COMPANY: ${product.company_name || 'Unknown'}`);
|
||||
parts.push(`LINE: ${product.line_name || 'None'}`);
|
||||
if (product.subline_name) {
|
||||
parts.push(`SUBLINE: ${product.subline_name}`);
|
||||
}
|
||||
|
||||
// Add sibling context for naming decisions
|
||||
if (product.siblingNames && product.siblingNames.length > 0) {
|
||||
parts.push('');
|
||||
parts.push(`OTHER PRODUCTS IN THIS LINE (${product.siblingNames.length + 1} total including this one):`);
|
||||
product.siblingNames.forEach(name => {
|
||||
parts.push(`- ${name}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Add response format instructions
|
||||
parts.push('');
|
||||
parts.push('RESPOND WITH JSON:');
|
||||
parts.push(JSON.stringify({
|
||||
isValid: 'true/false',
|
||||
suggestion: 'corrected name if changes needed, or null if valid',
|
||||
issues: ['issue 1', 'issue 2 (empty array if valid)']
|
||||
}, null, 2));
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for name validation
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseNameResponse(parsed, content) {
|
||||
// Debug: Log what we're trying to parse
|
||||
console.log('[parseNameResponse] Input:', {
|
||||
hasParsed: !!parsed,
|
||||
parsedIsValid: parsed?.isValid,
|
||||
parsedType: typeof parsed?.isValid,
|
||||
contentPreview: content?.substring(0, 3000)
|
||||
});
|
||||
|
||||
// If we got valid parsed JSON, use it
|
||||
if (parsed && typeof parsed.isValid === 'boolean') {
|
||||
// Sanitize issues - AI sometimes returns malformed escape sequences
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
|
||||
// IMPORTANT: LLMs sometimes return contradictory data (isValid: true with issues).
|
||||
// If there are issues, treat as invalid regardless of what the AI said.
|
||||
const isValid = parsed.isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Handle case where isValid is a string "true"/"false" instead of boolean
|
||||
if (parsed && typeof parsed.isValid === 'string') {
|
||||
const rawIssues = Array.isArray(parsed.issues) ? parsed.issues : [];
|
||||
const issues = rawIssues
|
||||
.map(sanitizeIssue)
|
||||
.filter(issue => issue.length > 0);
|
||||
const suggestion = parsed.suggestion || null;
|
||||
const rawIsValid = parsed.isValid.toLowerCase() !== 'false';
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const isValid = rawIsValid && issues.length === 0 && !suggestion;
|
||||
|
||||
console.log('[parseNameResponse] Parsed isValid as string:', parsed.isValid, '→', isValid);
|
||||
return { isValid, suggestion, issues };
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Look for isValid pattern - handle both boolean and quoted string
|
||||
// Matches: "isValid": true, "isValid": false, "isValid": "true", "isValid": "false"
|
||||
const isValidMatch = content.match(/"isValid"\s*:\s*"?(true|false)"?/i);
|
||||
const isValid = isValidMatch ? isValidMatch[1].toLowerCase() === 'true' : true;
|
||||
|
||||
console.log('[parseNameResponse] Regex extraction:', {
|
||||
isValidMatch: isValidMatch?.[0],
|
||||
isValidValue: isValidMatch?.[1],
|
||||
resultIsValid: isValid
|
||||
});
|
||||
|
||||
// Look for suggestion - handle escaped quotes and null
|
||||
const suggestionMatch = content.match(/"suggestion"\s*:\s*(?:"([^"\\]*(?:\\.[^"\\]*)*)"|null)/);
|
||||
const suggestion = suggestionMatch ? (suggestionMatch[1] || null) : null;
|
||||
|
||||
// Look for issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
if (issuesMatch) {
|
||||
const issuesContent = issuesMatch[1];
|
||||
const issueStrings = issuesContent.match(/"([^"]+)"/g);
|
||||
if (issueStrings) {
|
||||
issues = issueStrings
|
||||
.map(s => sanitizeIssue(s.replace(/"/g, '')))
|
||||
.filter(issue => issue.length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Same defensive logic: if there are issues, it's not valid
|
||||
const finalIsValid = isValid && issues.length === 0 && !suggestion;
|
||||
|
||||
return { isValid: finalIsValid, suggestion, issues };
|
||||
} catch {
|
||||
// Default to valid if we can't parse anything
|
||||
return { isValid: true, suggestion: null, issues: [] };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildNameUserPrompt,
|
||||
parseNameResponse
|
||||
};
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Prompt Loader
|
||||
*
|
||||
* Utilities to load AI prompts from the ai_prompts PostgreSQL table.
|
||||
* Supports loading prompts by base type (e.g., 'name_validation' loads
|
||||
* name_validation_system, name_validation_general, and optionally
|
||||
* name_validation_company_specific).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Load a single prompt by exact type
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string} promptType - Exact prompt type (e.g., 'name_validation_system')
|
||||
* @param {string} [company] - Company identifier (for company_specific types)
|
||||
* @returns {Promise<string|null>} Prompt text or null if not found
|
||||
*/
|
||||
async function loadPromptByType(pool, promptType, company = null) {
|
||||
try {
|
||||
let result;
|
||||
|
||||
if (company) {
|
||||
result = await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company = $2',
|
||||
[promptType, company]
|
||||
);
|
||||
} else {
|
||||
result = await pool.query(
|
||||
'SELECT prompt_text FROM ai_prompts WHERE prompt_type = $1 AND company IS NULL',
|
||||
[promptType]
|
||||
);
|
||||
}
|
||||
|
||||
return result.rows[0]?.prompt_text || null;
|
||||
} catch (error) {
|
||||
console.error(`[PromptLoader] Error loading ${promptType} prompt:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all prompts for a task type (system, general, and optionally company-specific)
|
||||
*
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string} baseType - Base type name (e.g., 'name_validation', 'description_validation')
|
||||
* @param {string|null} [company] - Optional company ID for company-specific prompts
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadPromptsByType(pool, baseType, company = null) {
|
||||
const systemType = `${baseType}_system`;
|
||||
const generalType = `${baseType}_general`;
|
||||
const companyType = `${baseType}_company_specific`;
|
||||
|
||||
// Load system and general prompts in parallel
|
||||
const [system, general] = await Promise.all([
|
||||
loadPromptByType(pool, systemType),
|
||||
loadPromptByType(pool, generalType)
|
||||
]);
|
||||
|
||||
// Load company-specific prompt if company is provided
|
||||
let companySpecific = null;
|
||||
if (company) {
|
||||
companySpecific = await loadPromptByType(pool, companyType, company);
|
||||
}
|
||||
|
||||
return {
|
||||
system,
|
||||
general,
|
||||
companySpecific
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load name validation prompts
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadNameValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'name_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load description validation prompts
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadDescriptionValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'description_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load sanity check prompts (no company-specific variant)
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: null}>}
|
||||
*/
|
||||
async function loadSanityCheckPrompts(pool) {
|
||||
return loadPromptsByType(pool, 'sanity_check', null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load bulk validation prompts (GPT-5 validation)
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string|null} [company] - Optional company ID
|
||||
* @returns {Promise<{system: string|null, general: string|null, companySpecific: string|null}>}
|
||||
*/
|
||||
async function loadBulkValidationPrompts(pool, company = null) {
|
||||
return loadPromptsByType(pool, 'bulk_validation', company);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load bulk validation prompts for multiple companies at once
|
||||
* @param {Object} pool - PostgreSQL pool
|
||||
* @param {string[]} companyIds - Array of company IDs
|
||||
* @returns {Promise<{system: string|null, general: string|null, companyPrompts: Map<string, string>}>}
|
||||
*/
|
||||
async function loadBulkValidationPromptsForCompanies(pool, companyIds = []) {
|
||||
// Load system and general prompts
|
||||
const [system, general] = await Promise.all([
|
||||
loadPromptByType(pool, 'bulk_validation_system'),
|
||||
loadPromptByType(pool, 'bulk_validation_general')
|
||||
]);
|
||||
|
||||
// Load company-specific prompts for all provided companies
|
||||
const companyPrompts = new Map();
|
||||
|
||||
if (companyIds.length > 0) {
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`SELECT company, prompt_text FROM ai_prompts
|
||||
WHERE prompt_type = 'bulk_validation_company_specific'
|
||||
AND company = ANY($1)`,
|
||||
[companyIds]
|
||||
);
|
||||
|
||||
for (const row of result.rows) {
|
||||
companyPrompts.set(row.company, row.prompt_text);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[PromptLoader] Error loading company-specific prompts:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
system,
|
||||
general,
|
||||
companyPrompts
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that required prompts exist, throw error if missing
|
||||
* @param {Object} prompts - Prompts object from loadPromptsByType
|
||||
* @param {string} baseType - Base type for error messages
|
||||
* @param {Object} options - Validation options
|
||||
* @param {boolean} [options.requireSystem=true] - Require system prompt
|
||||
* @param {boolean} [options.requireGeneral=true] - Require general prompt
|
||||
* @throws {Error} If required prompts are missing
|
||||
*/
|
||||
function validateRequiredPrompts(prompts, baseType, options = {}) {
|
||||
const { requireSystem = true, requireGeneral = true } = options;
|
||||
const missing = [];
|
||||
|
||||
if (requireSystem && !prompts.system) {
|
||||
missing.push(`${baseType}_system`);
|
||||
}
|
||||
|
||||
if (requireGeneral && !prompts.general) {
|
||||
missing.push(`${baseType}_general`);
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`Missing required AI prompts: ${missing.join(', ')}. ` +
|
||||
`Please add these prompts in Settings > AI Validation Prompts.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Core loader
|
||||
loadPromptByType,
|
||||
loadPromptsByType,
|
||||
|
||||
// Task-specific loaders
|
||||
loadNameValidationPrompts,
|
||||
loadDescriptionValidationPrompts,
|
||||
loadSanityCheckPrompts,
|
||||
loadBulkValidationPrompts,
|
||||
loadBulkValidationPromptsForCompanies,
|
||||
|
||||
// Validation
|
||||
validateRequiredPrompts
|
||||
};
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Sanity Check Prompts
|
||||
*
|
||||
* Functions for building and parsing batch product consistency validation prompts.
|
||||
* System and general prompts are loaded from the database.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Build the user prompt for sanity check
|
||||
* Combines database prompts with product data
|
||||
*
|
||||
* @param {Object[]} products - Array of product data (limited fields for context)
|
||||
* @param {Object} prompts - Prompts loaded from database
|
||||
* @param {string} prompts.general - General sanity check rules
|
||||
* @returns {string} Complete user prompt
|
||||
*/
|
||||
function buildSanityCheckUserPrompt(products, prompts) {
|
||||
// Build a simplified product list for the prompt
|
||||
const productSummaries = products.map((p, index) => ({
|
||||
index,
|
||||
name: p.name,
|
||||
supplier: p.supplier_name || p.supplier,
|
||||
company: p.company_name || p.company,
|
||||
supplier_no: p.supplier_no,
|
||||
msrp: p.msrp,
|
||||
cost_each: p.cost_each,
|
||||
qty_per_unit: p.qty_per_unit,
|
||||
case_qty: p.case_qty,
|
||||
tax_cat: p.tax_cat_name || p.tax_cat,
|
||||
size_cat: p.size_cat_name || p.size_cat,
|
||||
themes: p.theme_names || p.themes,
|
||||
categories: p.category_names || p.categories,
|
||||
weight: p.weight,
|
||||
length: p.length,
|
||||
width: p.width,
|
||||
height: p.height
|
||||
}));
|
||||
|
||||
const parts = [];
|
||||
|
||||
// Add general prompt/rules if provided
|
||||
if (prompts.general) {
|
||||
parts.push(prompts.general);
|
||||
parts.push(''); // Empty line for separation
|
||||
}
|
||||
|
||||
// Add products to review
|
||||
parts.push(`PRODUCTS TO REVIEW (${products.length} items):`);
|
||||
parts.push(JSON.stringify(productSummaries, null, 2));
|
||||
|
||||
// Add response format
|
||||
parts.push('');
|
||||
parts.push('RESPOND WITH JSON:');
|
||||
parts.push(JSON.stringify({
|
||||
issues: [
|
||||
{
|
||||
productIndex: 0,
|
||||
field: 'msrp',
|
||||
issue: 'Description of the issue found',
|
||||
suggestion: 'Suggested fix or verification (optional)'
|
||||
}
|
||||
],
|
||||
summary: '2-3 sentences summarizing the overall product quality'
|
||||
}, null, 2));
|
||||
|
||||
parts.push('');
|
||||
parts.push('If no issues are found, return empty issues array with positive summary.');
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the AI response for sanity check
|
||||
*
|
||||
* @param {Object|null} parsed - Parsed JSON from AI
|
||||
* @param {string} content - Raw response content
|
||||
* @returns {Object}
|
||||
*/
|
||||
function parseSanityCheckResponse(parsed, content) {
|
||||
// If we got valid parsed JSON, use it
|
||||
if (parsed && Array.isArray(parsed.issues)) {
|
||||
return {
|
||||
issues: parsed.issues.map(issue => ({
|
||||
productIndex: issue.productIndex ?? issue.index ?? 0,
|
||||
field: issue.field || 'unknown',
|
||||
issue: issue.issue || issue.message || '',
|
||||
suggestion: issue.suggestion || null
|
||||
})),
|
||||
summary: parsed.summary || 'Review complete'
|
||||
};
|
||||
}
|
||||
|
||||
// Try to extract from content if parsing failed
|
||||
try {
|
||||
// Try to find issues array
|
||||
const issuesMatch = content.match(/"issues"\s*:\s*\[([\s\S]*?)\]/);
|
||||
let issues = [];
|
||||
|
||||
if (issuesMatch) {
|
||||
// Try to parse the array content
|
||||
try {
|
||||
const arrayContent = `[${issuesMatch[1]}]`;
|
||||
const parsedIssues = JSON.parse(arrayContent);
|
||||
issues = parsedIssues.map(issue => ({
|
||||
productIndex: issue.productIndex ?? issue.index ?? 0,
|
||||
field: issue.field || 'unknown',
|
||||
issue: issue.issue || issue.message || '',
|
||||
suggestion: issue.suggestion || null
|
||||
}));
|
||||
} catch {
|
||||
// Couldn't parse the array
|
||||
}
|
||||
}
|
||||
|
||||
// Try to find summary
|
||||
const summaryMatch = content.match(/"summary"\s*:\s*"([^"]+)"/);
|
||||
const summary = summaryMatch ? summaryMatch[1] : 'Review complete';
|
||||
|
||||
return { issues, summary };
|
||||
} catch {
|
||||
return { issues: [], summary: 'Could not parse review results' };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildSanityCheckUserPrompt,
|
||||
parseSanityCheckResponse
|
||||
};
|
||||
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Groq Provider - Handles chat completions via Groq's OpenAI-compatible API
|
||||
*
|
||||
* Uses Groq's fast inference for real-time AI validation tasks.
|
||||
* Supports models like openai/gpt-oss-120b (complex) and openai/gpt-oss-20b (simple).
|
||||
*/
|
||||
|
||||
const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
|
||||
|
||||
// Default models
|
||||
const MODELS = {
|
||||
LARGE: 'openai/gpt-oss-120b', // For complex tasks (descriptions, sanity checks)
|
||||
SMALL: 'openai/gpt-oss-20b' // For simple tasks (name validation)
|
||||
};
|
||||
|
||||
class GroqProvider {
|
||||
/**
|
||||
* @param {Object} options
|
||||
* @param {string} options.apiKey - Groq API key
|
||||
* @param {string} [options.baseUrl] - Override base URL
|
||||
* @param {number} [options.timeoutMs=30000] - Default timeout
|
||||
*/
|
||||
constructor({ apiKey, baseUrl = GROQ_BASE_URL, timeoutMs = 30000 }) {
|
||||
if (!apiKey) {
|
||||
throw new Error('Groq API key is required');
|
||||
}
|
||||
this.apiKey = apiKey;
|
||||
this.baseUrl = baseUrl;
|
||||
this.timeoutMs = timeoutMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a chat completion request
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {Array<{role: string, content: string}>} params.messages - Conversation messages
|
||||
* @param {string} [params.model] - Model to use (defaults to LARGE)
|
||||
* @param {number} [params.temperature=0.3] - Response randomness (0-2)
|
||||
* @param {number} [params.maxTokens=500] - Max tokens in response
|
||||
* @param {Object} [params.responseFormat] - For JSON mode: { type: 'json_object' }
|
||||
* @param {number} [params.timeoutMs] - Request timeout override
|
||||
* @returns {Promise<{content: string, parsed: Object|null, usage: Object, latencyMs: number, model: string}>}
|
||||
*/
|
||||
async chatCompletion({
|
||||
messages,
|
||||
model = MODELS.LARGE,
|
||||
temperature = 0.3,
|
||||
maxTokens = 500,
|
||||
responseFormat = null,
|
||||
timeoutMs = this.timeoutMs
|
||||
}) {
|
||||
const started = Date.now();
|
||||
|
||||
const body = {
|
||||
model,
|
||||
messages,
|
||||
temperature,
|
||||
max_completion_tokens: maxTokens
|
||||
};
|
||||
|
||||
// Enable JSON mode if requested
|
||||
if (responseFormat?.type === 'json_object') {
|
||||
body.response_format = { type: 'json_object' };
|
||||
}
|
||||
|
||||
// Debug: Log request being sent
|
||||
console.log('[Groq] Request:', {
|
||||
model: body.model,
|
||||
temperature: body.temperature,
|
||||
maxTokens: body.max_completion_tokens,
|
||||
hasResponseFormat: !!body.response_format,
|
||||
messageCount: body.messages?.length,
|
||||
systemPromptLength: body.messages?.[0]?.content?.length,
|
||||
userPromptLength: body.messages?.[1]?.content?.length
|
||||
});
|
||||
|
||||
const response = await this._makeRequest('chat/completions', body, timeoutMs);
|
||||
|
||||
// Debug: Log raw response structure
|
||||
console.log('[Groq] Raw response:', {
|
||||
hasChoices: !!response.choices,
|
||||
choicesLength: response.choices?.length,
|
||||
firstChoice: response.choices?.[0] ? {
|
||||
finishReason: response.choices[0].finish_reason,
|
||||
hasMessage: !!response.choices[0].message,
|
||||
contentLength: response.choices[0].message?.content?.length,
|
||||
contentPreview: response.choices[0].message?.content?.substring(0, 200)
|
||||
} : null,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
});
|
||||
|
||||
const content = response.choices?.[0]?.message?.content || '';
|
||||
const usage = response.usage || {};
|
||||
|
||||
// Attempt to parse JSON if response format was requested
|
||||
let parsed = null;
|
||||
if (responseFormat && content) {
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch {
|
||||
// Content isn't valid JSON - try to extract JSON from markdown
|
||||
parsed = this._extractJson(content);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
parsed,
|
||||
usage: {
|
||||
promptTokens: usage.prompt_tokens || 0,
|
||||
completionTokens: usage.completion_tokens || 0,
|
||||
totalTokens: usage.total_tokens || 0
|
||||
},
|
||||
latencyMs: Date.now() - started,
|
||||
model: response.model || model
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON from content that might be wrapped in markdown code blocks
|
||||
* @private
|
||||
*/
|
||||
_extractJson(content) {
|
||||
// Try to find JSON in code blocks
|
||||
const codeBlockMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
||||
if (codeBlockMatch) {
|
||||
try {
|
||||
return JSON.parse(codeBlockMatch[1].trim());
|
||||
} catch {
|
||||
// Fall through
|
||||
}
|
||||
}
|
||||
|
||||
// Try to find JSON object/array directly
|
||||
const jsonMatch = content.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
||||
if (jsonMatch) {
|
||||
try {
|
||||
return JSON.parse(jsonMatch[1]);
|
||||
} catch {
|
||||
// Fall through
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an HTTP request to Groq API
|
||||
* @private
|
||||
*/
|
||||
async _makeRequest(endpoint, body, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({}));
|
||||
const message = error.error?.message || `Groq API error: ${response.status}`;
|
||||
const err = new Error(message);
|
||||
err.status = response.status;
|
||||
err.code = error.error?.code;
|
||||
// Include failed_generation if available (for JSON mode failures)
|
||||
if (error.error?.failed_generation) {
|
||||
err.failedGeneration = error.error.failed_generation;
|
||||
console.error('[Groq] JSON validation failed. Model output:', error.error.failed_generation);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
const err = new Error(`Groq request timed out after ${timeoutMs}ms`);
|
||||
err.code = 'TIMEOUT';
|
||||
throw err;
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the provider is properly configured
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isConfigured() {
|
||||
return !!this.apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { GroqProvider, MODELS, GROQ_BASE_URL };
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* OpenAI Provider - Handles embedding generation
|
||||
*/
|
||||
|
||||
const EMBEDDING_MODEL = 'text-embedding-3-small';
|
||||
const EMBEDDING_DIMENSIONS = 1536;
|
||||
const MAX_BATCH_SIZE = 2048;
|
||||
|
||||
class OpenAIProvider {
|
||||
constructor({ apiKey, baseUrl = 'https://api.openai.com/v1', timeoutMs = 60000 }) {
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key is required');
|
||||
}
|
||||
this.apiKey = apiKey;
|
||||
this.baseUrl = baseUrl;
|
||||
this.timeoutMs = timeoutMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for one or more texts
|
||||
* @param {string|string[]} input - Text or array of texts
|
||||
* @param {Object} options
|
||||
* @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>}
|
||||
*/
|
||||
async embed(input, options = {}) {
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
const model = options.model || EMBEDDING_MODEL;
|
||||
const dimensions = options.dimensions || EMBEDDING_DIMENSIONS;
|
||||
const timeoutMs = options.timeoutMs || this.timeoutMs;
|
||||
|
||||
if (texts.length > MAX_BATCH_SIZE) {
|
||||
throw new Error(`Batch size ${texts.length} exceeds max of ${MAX_BATCH_SIZE}`);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
|
||||
// Clean and truncate input texts
|
||||
const cleanedTexts = texts.map(t =>
|
||||
(t || '').replace(/\n+/g, ' ').trim().substring(0, 8000)
|
||||
);
|
||||
|
||||
const body = {
|
||||
input: cleanedTexts,
|
||||
model,
|
||||
encoding_format: 'float'
|
||||
};
|
||||
|
||||
// Only embedding-3 models support dimensions parameter
|
||||
if (model.includes('embedding-3')) {
|
||||
body.dimensions = dimensions;
|
||||
}
|
||||
|
||||
const response = await this._makeRequest('embeddings', body, timeoutMs);
|
||||
|
||||
// Sort by index to ensure order matches input
|
||||
const sortedData = response.data.sort((a, b) => a.index - b.index);
|
||||
|
||||
return {
|
||||
embeddings: sortedData.map(item => item.embedding),
|
||||
usage: {
|
||||
promptTokens: response.usage?.prompt_tokens || 0,
|
||||
totalTokens: response.usage?.total_tokens || 0
|
||||
},
|
||||
model: response.model || model,
|
||||
latencyMs: Date.now() - started
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generator for processing large batches in chunks
|
||||
*/
|
||||
async *embedBatchChunked(texts, options = {}) {
|
||||
const batchSize = Math.min(options.batchSize || 100, MAX_BATCH_SIZE);
|
||||
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
const chunk = texts.slice(i, i + batchSize);
|
||||
const result = await this.embed(chunk, options);
|
||||
|
||||
yield {
|
||||
embeddings: result.embeddings,
|
||||
startIndex: i,
|
||||
endIndex: i + chunk.length,
|
||||
usage: result.usage,
|
||||
model: result.model,
|
||||
latencyMs: result.latencyMs
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async _makeRequest(endpoint, body, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({}));
|
||||
throw new Error(error.error?.message || `OpenAI API error: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { OpenAIProvider, EMBEDDING_MODEL, EMBEDDING_DIMENSIONS };
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Description Validation Task
|
||||
*
|
||||
* Validates a product description for quality, accuracy, and guideline compliance.
|
||||
* Uses Groq with the larger model for better reasoning about content quality.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
loadDescriptionValidationPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
buildDescriptionUserPrompt,
|
||||
parseDescriptionResponse
|
||||
} = require('../prompts/descriptionPrompts');
|
||||
|
||||
const TASK_ID = 'validate.description';
|
||||
|
||||
/**
|
||||
* Create the description validation task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createDescriptionValidationTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Validate product description for quality and guideline compliance',
|
||||
|
||||
/**
|
||||
* Run the description validation
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object} payload.product - Product data
|
||||
* @param {string} payload.product.name - Product name (for context)
|
||||
* @param {string} payload.product.description - Description to validate
|
||||
* @param {string} [payload.product.company_name] - Company name
|
||||
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
|
||||
* @param {string} [payload.product.categories] - Product categories
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { product, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!product?.name && !product?.description) {
|
||||
return {
|
||||
isValid: true,
|
||||
suggestion: null,
|
||||
issues: [],
|
||||
skipped: true,
|
||||
reason: 'No name or description provided'
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
}
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const companyKey = product.company_id || product.company_name || product.company;
|
||||
const prompts = await loadDescriptionValidationPrompts(pool, companyKey);
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'description_validation');
|
||||
|
||||
// Build the user prompt with database-loaded prompts
|
||||
const userPrompt = buildDescriptionUserPrompt(product, prompts);
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - better for content analysis
|
||||
temperature: 0.3, // Slightly higher for creative suggestions
|
||||
maxTokens: 2000, // Reasoning models need extra tokens for thinking
|
||||
responseFormat: { type: 'json_object' }
|
||||
});
|
||||
|
||||
// Log full raw response for debugging
|
||||
log.info('[DescriptionValidation] Raw AI response:', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
});
|
||||
|
||||
// Parse the response
|
||||
result = parseDescriptionResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[DescriptionValidation] JSON mode failed, attempting to parse failed_generation:', {
|
||||
failedGeneration: jsonError.failedGeneration
|
||||
});
|
||||
result = parseDescriptionResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[DescriptionValidation] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.3,
|
||||
maxTokens: 2000 // Reasoning models need extra tokens for thinking
|
||||
// No responseFormat - let the model respond freely
|
||||
});
|
||||
log.info('[DescriptionValidation] Raw AI response (no JSON mode):', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
});
|
||||
result = parseDescriptionResponse(response.parsed, response.content);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[DescriptionValidation] Validated description for "${product.name}" in ${response.latencyMs}ms`, {
|
||||
isValid: result.isValid,
|
||||
hasSuggestion: !!result.suggestion,
|
||||
issueCount: result.issues.length
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[DescriptionValidation] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createDescriptionValidationTask
|
||||
};
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* AI Task Registry
|
||||
*
|
||||
* Simple registry pattern for AI tasks. Each task has:
|
||||
* - id: Unique identifier
|
||||
* - run: Async function that executes the task
|
||||
*
|
||||
* This allows adding new AI capabilities without modifying core code.
|
||||
*/
|
||||
|
||||
const { createNameValidationTask, TASK_ID: NAME_TASK_ID } = require('./nameValidationTask');
|
||||
const { createDescriptionValidationTask, TASK_ID: DESC_TASK_ID } = require('./descriptionValidationTask');
|
||||
const { createSanityCheckTask, TASK_ID: SANITY_TASK_ID } = require('./sanityCheckTask');
|
||||
|
||||
/**
|
||||
* Task IDs - frozen constants for type safety
|
||||
*/
|
||||
const TASK_IDS = Object.freeze({
|
||||
// Inline validation (triggered on field blur)
|
||||
VALIDATE_NAME: NAME_TASK_ID,
|
||||
VALIDATE_DESCRIPTION: DESC_TASK_ID,
|
||||
|
||||
// Batch operations (triggered on user action)
|
||||
SANITY_CHECK: SANITY_TASK_ID
|
||||
});
|
||||
|
||||
/**
|
||||
* Task Registry
|
||||
*/
|
||||
class TaskRegistry {
|
||||
constructor() {
|
||||
this.tasks = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a task
|
||||
* @param {Object} task
|
||||
* @param {string} task.id - Unique task identifier
|
||||
* @param {Function} task.run - Async function: (payload) => result
|
||||
* @param {string} [task.description] - Human-readable description
|
||||
*/
|
||||
register(task) {
|
||||
if (!task?.id) {
|
||||
throw new Error('Task must have an id');
|
||||
}
|
||||
if (typeof task.run !== 'function') {
|
||||
throw new Error(`Task ${task.id} must have a run function`);
|
||||
}
|
||||
if (this.tasks.has(task.id)) {
|
||||
throw new Error(`Task ${task.id} is already registered`);
|
||||
}
|
||||
|
||||
this.tasks.set(task.id, task);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a task by ID
|
||||
* @param {string} taskId
|
||||
* @returns {Object|null}
|
||||
*/
|
||||
get(taskId) {
|
||||
return this.tasks.get(taskId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a task exists
|
||||
* @param {string} taskId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
has(taskId) {
|
||||
return this.tasks.has(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a task by ID
|
||||
* @param {string} taskId
|
||||
* @param {Object} payload - Task-specific input
|
||||
* @returns {Promise<Object>} Task result
|
||||
*/
|
||||
async runTask(taskId, payload = {}) {
|
||||
const task = this.get(taskId);
|
||||
if (!task) {
|
||||
throw new Error(`Unknown task: ${taskId}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await task.run(payload);
|
||||
return {
|
||||
success: true,
|
||||
taskId,
|
||||
...result
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
taskId,
|
||||
error: error.message,
|
||||
code: error.code
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all registered task IDs
|
||||
* @returns {string[]}
|
||||
*/
|
||||
list() {
|
||||
return Array.from(this.tasks.keys());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of registered tasks
|
||||
* @returns {number}
|
||||
*/
|
||||
size() {
|
||||
return this.tasks.size;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
let registry = null;
|
||||
|
||||
/**
|
||||
* Get or create the task registry
|
||||
* @returns {TaskRegistry}
|
||||
*/
|
||||
function getRegistry() {
|
||||
if (!registry) {
|
||||
registry = new TaskRegistry();
|
||||
}
|
||||
return registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the registry (mainly for testing)
|
||||
*/
|
||||
function resetRegistry() {
|
||||
registry = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all validation tasks with the registry
|
||||
* Call this during initialization after the registry is created
|
||||
*
|
||||
* @param {Object} [logger] - Optional logger
|
||||
*/
|
||||
function registerAllTasks(logger = console) {
|
||||
const reg = getRegistry();
|
||||
|
||||
// Register name validation
|
||||
if (!reg.has(TASK_IDS.VALIDATE_NAME)) {
|
||||
reg.register(createNameValidationTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_NAME}`);
|
||||
}
|
||||
|
||||
// Register description validation
|
||||
if (!reg.has(TASK_IDS.VALIDATE_DESCRIPTION)) {
|
||||
reg.register(createDescriptionValidationTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.VALIDATE_DESCRIPTION}`);
|
||||
}
|
||||
|
||||
// Register sanity check
|
||||
if (!reg.has(TASK_IDS.SANITY_CHECK)) {
|
||||
reg.register(createSanityCheckTask());
|
||||
logger.info(`[Tasks] Registered: ${TASK_IDS.SANITY_CHECK}`);
|
||||
}
|
||||
|
||||
return reg;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
// Constants
|
||||
TASK_IDS,
|
||||
|
||||
// Registry
|
||||
TaskRegistry,
|
||||
getRegistry,
|
||||
resetRegistry,
|
||||
registerAllTasks,
|
||||
|
||||
// Task factories (for custom registration)
|
||||
createNameValidationTask,
|
||||
createDescriptionValidationTask,
|
||||
createSanityCheckTask
|
||||
};
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Name Validation Task
|
||||
*
|
||||
* Validates a product name for spelling, grammar, and naming conventions.
|
||||
* Uses Groq with the smaller model for fast response times.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
loadNameValidationPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
buildNameUserPrompt,
|
||||
parseNameResponse
|
||||
} = require('../prompts/namePrompts');
|
||||
|
||||
const TASK_ID = 'validate.name';
|
||||
|
||||
/**
|
||||
* Create the name validation task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createNameValidationTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Validate product name for spelling, grammar, and conventions',
|
||||
|
||||
/**
|
||||
* Run the name validation
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object} payload.product - Product data
|
||||
* @param {string} payload.product.name - Product name to validate
|
||||
* @param {string} [payload.product.company_name] - Company name
|
||||
* @param {string} [payload.product.company_id] - Company ID for loading specific rules
|
||||
* @param {string} [payload.product.line_name] - Product line
|
||||
* @param {string} [payload.product.description] - Description for context
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { product, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!product?.name) {
|
||||
return {
|
||||
isValid: true,
|
||||
suggestion: null,
|
||||
issues: [],
|
||||
skipped: true,
|
||||
reason: 'No name provided'
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
}
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const companyKey = product.company_id || product.company_name || product.company;
|
||||
const prompts = await loadNameValidationPrompts(pool, companyKey);
|
||||
|
||||
// Debug: Log loaded prompts
|
||||
log.info('[NameValidation] Loaded prompts:', {
|
||||
hasSystem: !!prompts.system,
|
||||
systemLength: prompts.system?.length || 0,
|
||||
hasGeneral: !!prompts.general,
|
||||
generalLength: prompts.general?.length || 0,
|
||||
generalPreview: prompts.general?.substring(0, 100) || '(empty)',
|
||||
hasCompanySpecific: !!prompts.companySpecific,
|
||||
companyKey
|
||||
});
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'name_validation');
|
||||
|
||||
// Build the user prompt with database-loaded prompts
|
||||
const userPrompt = buildNameUserPrompt(product, prompts);
|
||||
|
||||
// Debug: Log the full user prompt being sent
|
||||
log.info('[NameValidation] User prompt:', userPrompt.substring(0, 500));
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - reasoning model
|
||||
temperature: 0.2, // Low temperature for consistent results
|
||||
maxTokens: 3000, // Reasoning models need extra tokens for thinking
|
||||
responseFormat: { type: 'json_object' }
|
||||
});
|
||||
|
||||
// Log full raw response for debugging
|
||||
log.info('[NameValidation] Raw AI response:', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
});
|
||||
|
||||
// Parse the response
|
||||
result = parseNameResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[NameValidation] JSON mode failed, attempting to parse failed_generation:', {
|
||||
failedGeneration: jsonError.failedGeneration
|
||||
});
|
||||
result = parseNameResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.SMALL };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[NameValidation] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.SMALL,
|
||||
temperature: 0.2,
|
||||
maxTokens: 1500 // Reasoning models need extra tokens for thinking
|
||||
// No responseFormat - let the model respond freely
|
||||
});
|
||||
log.info('[NameValidation] Raw AI response (no JSON mode):', {
|
||||
parsed: response.parsed,
|
||||
content: response.content,
|
||||
contentLength: response.content?.length
|
||||
});
|
||||
result = parseNameResponse(response.parsed, response.content);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[NameValidation] Validated "${product.name}" in ${response.latencyMs}ms`, {
|
||||
isValid: result.isValid,
|
||||
hassuggestion: !!result.suggestion,
|
||||
issueCount: result.issues.length
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[NameValidation] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createNameValidationTask
|
||||
};
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Sanity Check Task
|
||||
*
|
||||
* Reviews a batch of products for consistency and appropriateness.
|
||||
* Uses Groq with the larger model for complex batch analysis.
|
||||
* Loads all prompts from the database (no hardcoded prompts).
|
||||
*/
|
||||
|
||||
const { MODELS } = require('../providers/groqProvider');
|
||||
const {
|
||||
loadSanityCheckPrompts,
|
||||
validateRequiredPrompts
|
||||
} = require('../prompts/promptLoader');
|
||||
const {
|
||||
buildSanityCheckUserPrompt,
|
||||
parseSanityCheckResponse
|
||||
} = require('../prompts/sanityCheckPrompts');
|
||||
|
||||
const TASK_ID = 'sanity.check';
|
||||
|
||||
// Maximum products to send in a single request (to avoid token limits)
|
||||
const MAX_PRODUCTS_PER_REQUEST = 50;
|
||||
|
||||
/**
|
||||
* Create the sanity check task
|
||||
*
|
||||
* @returns {Object} Task definition
|
||||
*/
|
||||
function createSanityCheckTask() {
|
||||
return {
|
||||
id: TASK_ID,
|
||||
description: 'Review batch of products for consistency and appropriateness',
|
||||
|
||||
/**
|
||||
* Run the sanity check
|
||||
*
|
||||
* @param {Object} payload
|
||||
* @param {Object[]} payload.products - Array of products to check
|
||||
* @param {Object} payload.provider - Groq provider instance
|
||||
* @param {Object} payload.pool - PostgreSQL pool
|
||||
* @param {Object} [payload.logger] - Logger instance
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async run(payload) {
|
||||
const { products, provider, pool, logger } = payload;
|
||||
const log = logger || console;
|
||||
|
||||
// Validate required input
|
||||
if (!Array.isArray(products) || products.length === 0) {
|
||||
return {
|
||||
issues: [],
|
||||
summary: 'No products to check',
|
||||
skipped: true
|
||||
};
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
throw new Error('Groq provider not available');
|
||||
}
|
||||
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not available');
|
||||
}
|
||||
|
||||
try {
|
||||
// Load prompts from database
|
||||
const prompts = await loadSanityCheckPrompts(pool);
|
||||
|
||||
// Validate required prompts exist
|
||||
validateRequiredPrompts(prompts, 'sanity_check');
|
||||
|
||||
// If batch is small enough, process in one request
|
||||
if (products.length <= MAX_PRODUCTS_PER_REQUEST) {
|
||||
return await checkBatch(products, prompts, provider, log);
|
||||
}
|
||||
|
||||
// Otherwise, process in chunks and combine results
|
||||
log.info(`[SanityCheck] Processing ${products.length} products in chunks`);
|
||||
const allIssues = [];
|
||||
const summaries = [];
|
||||
|
||||
for (let i = 0; i < products.length; i += MAX_PRODUCTS_PER_REQUEST) {
|
||||
const chunk = products.slice(i, i + MAX_PRODUCTS_PER_REQUEST);
|
||||
const chunkOffset = i; // To adjust product indices in results
|
||||
|
||||
const result = await checkBatch(chunk, prompts, provider, log);
|
||||
|
||||
// Adjust product indices to match original array
|
||||
const adjustedIssues = result.issues.map(issue => ({
|
||||
...issue,
|
||||
productIndex: issue.productIndex + chunkOffset
|
||||
}));
|
||||
|
||||
allIssues.push(...adjustedIssues);
|
||||
summaries.push(result.summary);
|
||||
}
|
||||
|
||||
return {
|
||||
issues: allIssues,
|
||||
summary: summaries.length > 1
|
||||
? `Reviewed ${products.length} products in ${summaries.length} batches. ${allIssues.length} issues found.`
|
||||
: summaries[0],
|
||||
totalProducts: products.length,
|
||||
issueCount: allIssues.length
|
||||
};
|
||||
} catch (error) {
|
||||
log.error('[SanityCheck] Error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a single batch of products
|
||||
*
|
||||
* @param {Object[]} products - Products to check
|
||||
* @param {Object} prompts - Loaded prompts from database
|
||||
* @param {Object} provider - Groq provider
|
||||
* @param {Object} log - Logger
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async function checkBatch(products, prompts, provider, log) {
|
||||
const userPrompt = buildSanityCheckUserPrompt(products, prompts);
|
||||
|
||||
let response;
|
||||
let result;
|
||||
|
||||
try {
|
||||
// Try with JSON mode first
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.LARGE, // openai/gpt-oss-120b - needed for complex batch analysis
|
||||
temperature: 0.2, // Low temperature for consistent analysis
|
||||
maxTokens: 2000, // More tokens for batch results
|
||||
responseFormat: { type: 'json_object' }
|
||||
});
|
||||
|
||||
result = parseSanityCheckResponse(response.parsed, response.content);
|
||||
} catch (jsonError) {
|
||||
// If JSON mode failed, check if we have failedGeneration to parse
|
||||
if (jsonError.failedGeneration) {
|
||||
log.warn('[SanityCheck] JSON mode failed, attempting to parse failed_generation');
|
||||
result = parseSanityCheckResponse(null, jsonError.failedGeneration);
|
||||
response = { latencyMs: 0, usage: {}, model: MODELS.LARGE };
|
||||
} else {
|
||||
// Retry without JSON mode
|
||||
log.warn('[SanityCheck] JSON mode failed, retrying without JSON mode');
|
||||
response = await provider.chatCompletion({
|
||||
messages: [
|
||||
{ role: 'system', content: prompts.system },
|
||||
{ role: 'user', content: userPrompt }
|
||||
],
|
||||
model: MODELS.LARGE,
|
||||
temperature: 0.2,
|
||||
maxTokens: 2000
|
||||
// No responseFormat - let the model respond freely
|
||||
});
|
||||
result = parseSanityCheckResponse(response.parsed, response.content);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[SanityCheck] Checked ${products.length} products in ${response.latencyMs}ms`, {
|
||||
issueCount: result.issues.length
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
latencyMs: response.latencyMs,
|
||||
usage: response.usage,
|
||||
model: response.model
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TASK_ID,
|
||||
createSanityCheckTask,
|
||||
MAX_PRODUCTS_PER_REQUEST
|
||||
};
|
||||
Reference in New Issue
Block a user