diff --git a/docs/ai-validation-redesign.md b/docs/ai-validation-redesign.md new file mode 100644 index 0000000..00aca4e --- /dev/null +++ b/docs/ai-validation-redesign.md @@ -0,0 +1,2846 @@ +# AI Validation System Redesign + +> **Status:** Design Document (not yet implemented) +> **Date:** January 2025 +> **Goal:** Replace expensive batch AI validation with a multi-tier, real-time validation system + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Current State Analysis](#current-state-analysis) +3. [Proposed Architecture](#proposed-architecture) +4. [Backend Implementation](#backend-implementation) + - [File Structure](#file-structure) + - [Provider System](#provider-system) + - [Task Registry](#task-registry) + - [Embedding System](#embedding-system) + - [Micro-Prompt Tasks](#micro-prompt-tasks) +5. [Frontend Implementation](#frontend-implementation) + - [Suggestion Hooks](#suggestion-hooks) + - [UI Components](#ui-components) + - [Integration with ValidationStep](#integration-with-validationstep) +6. [Database Schema](#database-schema) +7. [Migration Strategy](#migration-strategy) +8. [Cost Analysis](#cost-analysis) + +--- + +## Executive Summary + +### Problem +The current AI validation system: +- Costs **$0.30-0.50 per run** (GPT-5.2 with reasoning) +- Takes **3-4 minutes** to process +- Sends a **giant prompt** (18KB+ instructions + full taxonomy + all products) +- Processes everything in **one batch at the end** of the workflow + +### Solution +A multi-tier validation system that: +- Performs **deterministic validation in code** (no AI needed for formatting) +- Uses **Groq (Llama 3.3)** for real-time field suggestions (~200ms, ~$0.001/product) +- Pre-computes **category/theme embeddings** for fast similarity search +- Reserves **expensive models** only for complex description generation +- Validates **incrementally** as user works, not all at once + +### Expected Outcomes +| Metric | Current | Proposed | +|--------|---------|----------| +| Cost per 50 products | $0.30-0.50 | $0.02-0.05 | +| Processing time | 3-4 minutes | Real-time + 10-30s batch | +| User experience | Wait at end | Inline suggestions | + +--- + +## Current State Analysis + +### What the Current System Does + +Based on the general prompt in `ai_prompts` table, validation performs these tasks: + +| Task | AI Required? | Proposed Approach | +|------|--------------|-------------------| +| Price formatting (`$5.00` → `5.00`) | ❌ No | JavaScript normalizer | +| UPC/SKU trimming | ❌ No | JavaScript normalizer | +| Country code conversion (`USA` → `US`) | ❌ No | Lookup table | +| Date formatting (ETA field) | ❌ No | Date parser | +| Category assignment | ⚠️ Partial | Embeddings + small model | +| Theme detection | ⚠️ Partial | Embeddings + small model | +| Color extraction | ⚠️ Partial | Small model | +| Name standardization | ✅ Yes | Medium model | +| Description enhancement | ✅ Yes | Medium model (streaming) | +| Weight/dimension consistency | ✅ Yes | Batch comparison | +| Tax code assignment | ⚠️ Partial | Rules + small model | + +### Current Prompt Structure + +``` +[System Instructions] ~200 chars +[General Prompt] ~18,000 chars (field guidelines, naming rules, etc.) +[Company-Specific] ~variable +[Taxonomy Data] ~50,000-100,000 chars (categories, themes, colors, etc.) +[Product Data] ~variable (all products as JSON) +``` + +**Total prompt size:** Often 100,000+ characters = ~25,000+ tokens input + +### Current Model Usage + +- **Model:** GPT-5.2 (reasoning model) +- **Reasoning effort:** Medium +- **Max output tokens:** 50,000 +- **Response format:** Strict JSON schema + +--- + +## Proposed Architecture + +### System Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ VALIDATION STEP UI │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ Field Edit │ │ Row Navigation │ │ Batch Action │ │ +│ │ (on blur) │ │ (on row change)│ │ (user clicks) │ │ +│ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │ +│ │ │ │ │ +└───────────┼─────────────────────┼─────────────────────┼─────────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌───────────────────────────────────────────────────────────────────────────┐ +│ TIER 1: CLIENT-SIDE │ +│ (Deterministic - No AI) │ +├───────────────────────────────────────────────────────────────────────────┤ +│ • Price formatting • UPC trimming • Country codes │ +│ • Numeric validation • Required fields • Date parsing │ +└───────────────────────────────────────────────────────────────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌───────────────────────────────────────────────────────────────────────────┐ +│ TIER 2: REAL-TIME AI SUGGESTIONS │ +│ (Groq - Llama 3.3 70B) │ +├───────────────────────────────────────────────────────────────────────────┤ +│ │ +│ POST /api/ai/suggest/:field │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ Name │ │ Category │ │ Theme │ │ Color │ │ Tax Code │ │ +│ │ Task │ │ Task │ │ Task │ │ Task │ │ Task │ │ +│ └──────────┘ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ +│ │ │ │ │ │ │ +│ │ ▼ │ │ │ │ +│ │ ┌────────────────┐ │ │ │ │ +│ │ │ Embedding │ │ │ │ │ +│ │ │ Pre-Filter │ │ │ │ │ +│ │ └────────────────┘ │ │ │ │ +│ │ │ │ │ │ │ +└───────┼──────────────┼─────────────┼─────────────┼─────────────┼──────────┘ + │ │ │ │ │ + ▼ ▼ ▼ ▼ ▼ +┌───────────────────────────────────────────────────────────────────────────┐ +│ TIER 3: BATCH AI ENHANCEMENT │ +│ (Claude Haiku / GPT-4o-mini) │ +├───────────────────────────────────────────────────────────────────────────┤ +│ │ +│ POST /api/ai/enhance/descriptions │ +│ POST /api/ai/validate/consistency │ +│ │ +│ ┌────────────────────┐ ┌────────────────────┐ │ +│ │ Description │ │ Cross-Product │ │ +│ │ Generation │ │ Consistency │ │ +│ │ (10-20 at once) │ │ (weights/dims) │ │ +│ └────────────────────┘ └────────────────────┘ │ +│ │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +### Validation Flow + +``` +User enters/maps data + │ + ▼ +┌───────────────────┐ +│ Tier 1: Code │ ◄── Instant (0ms) +│ - Format prices │ +│ - Validate UPCs │ +│ - Convert country │ +└─────────┬─────────┘ + │ + ▼ +┌───────────────────┐ +│ Tier 2: Groq │ ◄── Fast (200-500ms per field) +│ - Suggest category│ Triggered on focus/blur +│ - Detect themes │ +│ - Format name │ +└─────────┬─────────┘ + │ + ▼ +┌───────────────────┐ +│ Tier 3: Batch │ ◄── On-demand (5-30s) +│ - Descriptions │ User clicks "Enhance" +│ - Consistency │ +└───────────────────┘ +``` + +--- + +## Backend Implementation + +### File Structure + +``` +inventory-server/src/ +├── services/ +│ └── ai/ +│ ├── index.js # Main entry, initialization, exports +│ ├── config.js # AI configuration management +│ ├── taskRegistry.js # Task registration system +│ ├── workQueue.js # Concurrency control +│ │ +│ ├── providers/ +│ │ ├── index.js # Provider factory +│ │ ├── groqProvider.js # Groq API client (chat) +│ │ ├── openaiProvider.js # OpenAI API client (embeddings + chat) +│ │ └── anthropicProvider.js # Claude API client (batch tasks) +│ │ +│ ├── embeddings/ +│ │ ├── index.js # Embedding service entry +│ │ ├── categoryEmbeddings.js # Category embedding management +│ │ ├── themeEmbeddings.js # Theme embedding management +│ │ ├── vectorStore.js # In-memory vector storage +│ │ └── similarity.js # Cosine similarity utilities +│ │ +│ ├── tasks/ +│ │ ├── index.js # Task exports +│ │ ├── nameSuggestionTask.js +│ │ ├── categorySuggestionTask.js +│ │ ├── themeSuggestionTask.js +│ │ ├── colorSuggestionTask.js +│ │ ├── taxCodeSuggestionTask.js +│ │ ├── descriptionEnhanceTask.js +│ │ ├── consistencyCheckTask.js +│ │ └── utils/ +│ │ ├── productUtils.js # Product data helpers +│ │ └── responseParser.js # AI response parsing +│ │ +│ ├── prompts/ +│ │ ├── index.js # Prompt exports +│ │ ├── namePrompts.js +│ │ ├── categoryPrompts.js +│ │ ├── themePrompts.js +│ │ ├── colorPrompts.js +│ │ ├── descriptionPrompts.js +│ │ └── consistencyPrompts.js +│ │ +│ └── normalizers/ +│ ├── index.js # Normalizer exports +│ ├── priceNormalizer.js +│ ├── upcNormalizer.js +│ ├── countryCodeNormalizer.js +│ ├── dateNormalizer.js +│ └── numericNormalizer.js +│ +└── routes/ + └── ai.js # New AI routes (replaces ai-validation.js) +``` + +### Provider System + +#### Provider Interface + +```javascript +// services/ai/providers/index.js + +/** + * All providers must implement this interface + */ +class AIProvider { + /** + * Chat completion + * @param {Object} params + * @param {Array<{role: string, content: string}>} params.messages + * @param {string} params.model + * @param {number} [params.temperature=0.3] + * @param {number} [params.maxTokens=500] + * @param {Object} [params.responseFormat] - JSON schema for structured output + * @param {number} [params.timeoutMs=30000] + * @returns {Promise<{content: string, parsed: Object|null, usage: Object, latencyMs: number, model: string}>} + */ + async chatCompletion(params) { + throw new Error('Not implemented'); + } + + /** + * Generate embeddings + * @param {string|string[]} input - Text or array of texts + * @param {Object} [options] + * @param {string} [options.model] + * @param {number} [options.dimensions] + * @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>} + */ + async embed(input, options) { + throw new Error('Not implemented'); + } +} + +/** + * Provider factory + */ +function createProvider(providerName, config) { + switch (providerName.toLowerCase()) { + case 'groq': + return new GroqProvider(config.providers.groq); + case 'openai': + return new OpenAIProvider(config.providers.openai); + case 'anthropic': + return new AnthropicProvider(config.providers.anthropic); + default: + throw new Error(`Unknown provider: ${providerName}`); + } +} + +module.exports = { AIProvider, createProvider }; +``` + +#### Groq Provider (Primary for Real-Time) + +```javascript +// services/ai/providers/groqProvider.js + +const Groq = require('groq-sdk'); + +class GroqProvider { + constructor({ apiKey, baseUrl, timeoutMs = 30000 }) { + if (!apiKey) { + throw new Error('Groq API key is required'); + } + this.client = new Groq({ apiKey, baseURL: baseUrl }); + this.timeoutMs = timeoutMs; + } + + async chatCompletion({ + messages, + model = 'llama-3.3-70b-versatile', + temperature = 0.3, + maxTokens = 500, + responseFormat = null, + timeoutMs = this.timeoutMs + }) { + const started = Date.now(); + + const params = { + messages, + model, + temperature, + max_tokens: maxTokens + }; + + // Add JSON mode if requested + if (responseFormat) { + params.response_format = { type: 'json_object' }; + } + + const response = await this.client.chat.completions.create(params, { + timeout: timeoutMs + }); + + const content = response.choices[0]?.message?.content || ''; + const usage = response.usage || {}; + + // Try to parse JSON if response format was requested + let parsed = null; + if (responseFormat && content) { + try { + parsed = JSON.parse(content); + } catch { + // Will return raw content + } + } + + return { + content, + parsed, + usage: { + promptTokens: usage.prompt_tokens || 0, + completionTokens: usage.completion_tokens || 0, + totalTokens: usage.total_tokens || 0 + }, + latencyMs: Date.now() - started, + model: response.model || model + }; + } + + // Groq doesn't support embeddings, so this throws + async embed() { + throw new Error('Groq does not support embeddings. Use OpenAI provider.'); + } +} + +module.exports = { GroqProvider }; +``` + +#### OpenAI Provider (Embeddings + Fallback Chat) + +```javascript +// services/ai/providers/openaiProvider.js + +const MAX_EMBEDDING_BATCH_SIZE = 2048; + +class OpenAIProvider { + constructor({ + apiKey, + baseUrl = 'https://api.openai.com/v1', + embeddingModel = 'text-embedding-3-small', + embeddingDimensions = 1536, + chatModel = 'gpt-4o-mini', + timeoutMs = 60000 + }) { + if (!apiKey) { + throw new Error('OpenAI API key is required'); + } + this.apiKey = apiKey; + this.baseUrl = baseUrl; + this.embeddingModel = embeddingModel; + this.embeddingDimensions = embeddingDimensions; + this.chatModel = chatModel; + this.timeoutMs = timeoutMs; + } + + async chatCompletion({ + messages, + model = this.chatModel, + temperature = 0.3, + maxTokens = 500, + responseFormat = null, + timeoutMs = this.timeoutMs + }) { + const started = Date.now(); + + const body = { + model, + messages, + temperature, + max_tokens: maxTokens + }; + + if (responseFormat) { + body.response_format = { type: 'json_object' }; + } + + const response = await this._makeRequest('chat/completions', body, timeoutMs); + const content = response.choices[0]?.message?.content || ''; + const usage = response.usage || {}; + + let parsed = null; + if (responseFormat && content) { + try { + parsed = JSON.parse(content); + } catch { + // Will return raw content + } + } + + return { + content, + parsed, + usage: { + promptTokens: usage.prompt_tokens || 0, + completionTokens: usage.completion_tokens || 0, + totalTokens: usage.total_tokens || 0 + }, + latencyMs: Date.now() - started, + model: response.model || model + }; + } + + /** + * Generate embeddings for a single text or batch + */ + async embed(input, options = {}) { + const texts = Array.isArray(input) ? input : [input]; + const model = options.model || this.embeddingModel; + const dimensions = options.dimensions || this.embeddingDimensions; + const timeoutMs = options.timeoutMs || this.timeoutMs; + + if (texts.length > MAX_EMBEDDING_BATCH_SIZE) { + throw new Error(`Batch size ${texts.length} exceeds max of ${MAX_EMBEDDING_BATCH_SIZE}`); + } + + const started = Date.now(); + + // Clean input texts + const cleanedTexts = texts.map(t => + (t || '').replace(/\n+/g, ' ').trim().substring(0, 8000) + ); + + const body = { + input: cleanedTexts, + model, + encoding_format: 'float' + }; + + // Only embedding-3 models support dimensions parameter + if (model.includes('embedding-3')) { + body.dimensions = dimensions; + } + + const response = await this._makeRequest('embeddings', body, timeoutMs); + + // Sort by index to ensure order matches input + const sortedData = response.data.sort((a, b) => a.index - b.index); + const embeddings = sortedData.map(item => item.embedding); + + return { + embeddings, + usage: { + promptTokens: response.usage?.prompt_tokens || 0, + totalTokens: response.usage?.total_tokens || 0 + }, + model: response.model || model, + latencyMs: Date.now() - started + }; + } + + /** + * Generator for processing large batches in chunks + */ + async *embedBatchChunked(texts, options = {}) { + const batchSize = Math.min(options.batchSize || 100, MAX_EMBEDDING_BATCH_SIZE); + + for (let i = 0; i < texts.length; i += batchSize) { + const chunk = texts.slice(i, i + batchSize); + const result = await this.embed(chunk, options); + + yield { + embeddings: result.embeddings, + startIndex: i, + endIndex: i + chunk.length, + usage: result.usage, + model: result.model, + latencyMs: result.latencyMs + }; + } + } + + async _makeRequest(endpoint, body, timeoutMs) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(`${this.baseUrl}/${endpoint}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${this.apiKey}` + }, + body: JSON.stringify(body), + signal: controller.signal + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + throw new Error(error.error?.message || `OpenAI API error: ${response.status}`); + } + + return response.json(); + } finally { + clearTimeout(timeout); + } + } +} + +module.exports = { OpenAIProvider, MAX_EMBEDDING_BATCH_SIZE }; +``` + +#### Anthropic Provider (Batch Enhancement) + +```javascript +// services/ai/providers/anthropicProvider.js + +const Anthropic = require('@anthropic-ai/sdk'); + +class AnthropicProvider { + constructor({ + apiKey, + defaultModel = 'claude-3-5-haiku-20241022', + timeoutMs = 120000 + }) { + if (!apiKey) { + throw new Error('Anthropic API key is required'); + } + this.client = new Anthropic({ apiKey }); + this.defaultModel = defaultModel; + this.timeoutMs = timeoutMs; + } + + async chatCompletion({ + messages, + model = this.defaultModel, + temperature = 0.3, + maxTokens = 1000, + system = null, + timeoutMs = this.timeoutMs + }) { + const started = Date.now(); + + // Anthropic uses separate system parameter + const params = { + model, + max_tokens: maxTokens, + temperature, + messages: messages.filter(m => m.role !== 'system') + }; + + // Extract system message if present + const systemMessage = system || messages.find(m => m.role === 'system')?.content; + if (systemMessage) { + params.system = systemMessage; + } + + const response = await this.client.messages.create(params); + + const content = response.content + .filter(block => block.type === 'text') + .map(block => block.text) + .join(''); + + let parsed = null; + try { + parsed = JSON.parse(content); + } catch { + // Not JSON + } + + return { + content, + parsed, + usage: { + promptTokens: response.usage?.input_tokens || 0, + completionTokens: response.usage?.output_tokens || 0, + totalTokens: (response.usage?.input_tokens || 0) + (response.usage?.output_tokens || 0) + }, + latencyMs: Date.now() - started, + model: response.model || model + }; + } + + async embed() { + throw new Error('Anthropic does not support embeddings. Use OpenAI provider.'); + } +} + +module.exports = { AnthropicProvider }; +``` + +### Task Registry + +```javascript +// services/ai/taskRegistry.js + +/** + * Registry for AI tasks + * Manages task registration, lookup, and execution + */ +class AiTaskRegistry { + constructor() { + this.tasks = new Map(); + } + + /** + * Register a task + * @param {Object} taskDefinition + * @param {string} taskDefinition.id - Unique task identifier + * @param {string} taskDefinition.description - Human-readable description + * @param {Function} taskDefinition.run - Async function to execute + * @param {Object} [taskDefinition.config] - Task-specific configuration + */ + register(taskDefinition) { + if (!taskDefinition?.id) { + throw new Error('Task must have an id'); + } + if (typeof taskDefinition.run !== 'function') { + throw new Error(`Task ${taskDefinition.id} must have a run function`); + } + if (this.tasks.has(taskDefinition.id)) { + throw new Error(`Task ${taskDefinition.id} is already registered`); + } + this.tasks.set(taskDefinition.id, taskDefinition); + return this; + } + + /** + * Get a task by ID + */ + get(taskId) { + return this.tasks.get(taskId) || null; + } + + /** + * Check if a task exists + */ + has(taskId) { + return this.tasks.has(taskId); + } + + /** + * List all registered task IDs + */ + list() { + return Array.from(this.tasks.keys()); + } +} + +/** + * Task IDs as frozen constants + */ +const TASK_IDS = Object.freeze({ + // Tier 2: Real-time suggestions + SUGGEST_NAME: 'suggest.name', + SUGGEST_CATEGORIES: 'suggest.categories', + SUGGEST_THEMES: 'suggest.themes', + SUGGEST_COLORS: 'suggest.colors', + SUGGEST_TAX_CODE: 'suggest.taxCode', + SUGGEST_SIZE_CATEGORY: 'suggest.sizeCategory', + + // Tier 3: Batch enhancement + ENHANCE_DESCRIPTIONS: 'enhance.descriptions', + CHECK_CONSISTENCY: 'check.consistency', + + // Utility tasks + COMPUTE_EMBEDDINGS: 'util.computeEmbeddings' +}); + +module.exports = { AiTaskRegistry, TASK_IDS }; +``` + +### Work Queue (Concurrency Control) + +```javascript +// services/ai/workQueue.js + +/** + * Simple concurrent work queue + * Prevents overwhelming AI providers with too many parallel requests + */ +class AiWorkQueue { + constructor(concurrency = 3) { + this.concurrency = Math.max(1, concurrency); + this.active = 0; + this.queue = []; + } + + /** + * Enqueue a task for execution + * @param {Function} taskFactory - Async function to execute + * @returns {Promise} - Resolves with task result + */ + enqueue(taskFactory) { + return new Promise((resolve, reject) => { + const execute = async () => { + this.active += 1; + try { + const result = await taskFactory(); + resolve(result); + } catch (error) { + reject(error); + } finally { + this.active -= 1; + this._processNext(); + } + }; + + if (this.active < this.concurrency) { + execute(); + } else { + this.queue.push(execute); + } + }); + } + + _processNext() { + if (this.queue.length === 0 || this.active >= this.concurrency) { + return; + } + const next = this.queue.shift(); + if (next) { + next(); + } + } + + /** + * Get current queue statistics + */ + getStats() { + return { + active: this.active, + queued: this.queue.length, + concurrency: this.concurrency + }; + } +} + +module.exports = { AiWorkQueue }; +``` + +### Embedding System + +#### Embedding Service Entry + +```javascript +// services/ai/embeddings/index.js + +const { CategoryEmbeddings } = require('./categoryEmbeddings'); +const { ThemeEmbeddings } = require('./themeEmbeddings'); +const { VectorStore } = require('./vectorStore'); + +let categoryEmbeddings = null; +let themeEmbeddings = null; +let initialized = false; + +/** + * Initialize the embedding system + * Should be called once at server startup + */ +async function initializeEmbeddings({ openaiProvider, mysqlConnection, logger }) { + if (initialized) { + return { categoryEmbeddings, themeEmbeddings }; + } + + logger?.info('[Embeddings] Initializing embedding system...'); + + // Initialize category embeddings + categoryEmbeddings = new CategoryEmbeddings({ + provider: openaiProvider, + connection: mysqlConnection, + logger + }); + + // Initialize theme embeddings + themeEmbeddings = new ThemeEmbeddings({ + provider: openaiProvider, + connection: mysqlConnection, + logger + }); + + // Load or compute embeddings + await Promise.all([ + categoryEmbeddings.initialize(), + themeEmbeddings.initialize() + ]); + + initialized = true; + logger?.info('[Embeddings] Embedding system initialized'); + + return { categoryEmbeddings, themeEmbeddings }; +} + +/** + * Get category suggestions for a product + */ +async function suggestCategories(productText, topK = 10) { + if (!categoryEmbeddings) { + throw new Error('Embeddings not initialized'); + } + return categoryEmbeddings.findSimilar(productText, topK); +} + +/** + * Get theme suggestions for a product + */ +async function suggestThemes(productText, topK = 5) { + if (!themeEmbeddings) { + throw new Error('Embeddings not initialized'); + } + return themeEmbeddings.findSimilar(productText, topK); +} + +module.exports = { + initializeEmbeddings, + suggestCategories, + suggestThemes +}; +``` + +#### Category Embeddings + +```javascript +// services/ai/embeddings/categoryEmbeddings.js + +const { VectorStore } = require('./vectorStore'); +const { cosineSimilarity } = require('./similarity'); + +class CategoryEmbeddings { + constructor({ provider, connection, logger }) { + this.provider = provider; + this.connection = connection; + this.logger = logger; + this.vectorStore = new VectorStore(); + this.categories = []; // Raw category data + } + + /** + * Initialize embeddings - load from cache or compute + */ + async initialize() { + this.logger?.info('[CategoryEmbeddings] Loading categories from database...'); + + // Fetch hierarchical categories + const [rows] = await this.connection.query(` + SELECT + cat_id, + name, + master_cat_id, + type + FROM product_categories + WHERE type IN (10, 11, 12, 13) + ORDER BY type, name + `); + + // Build category paths (e.g., "Paper > Patterned Paper > 12x12 Single Sheets") + this.categories = this._buildCategoryPaths(rows); + this.logger?.info(`[CategoryEmbeddings] Built ${this.categories.length} category paths`); + + // Check if we have cached embeddings + const cached = await this._loadCachedEmbeddings(); + if (cached && cached.length === this.categories.length) { + this.logger?.info('[CategoryEmbeddings] Using cached embeddings'); + this.vectorStore.load(cached); + return; + } + + // Compute new embeddings + await this._computeAndCacheEmbeddings(); + } + + /** + * Find similar categories for a product + */ + async findSimilar(productText, topK = 10) { + // Get embedding for product + const { embeddings } = await this.provider.embed(productText); + const productEmbedding = embeddings[0]; + + // Find most similar categories + const results = this.vectorStore.search(productEmbedding, topK); + + // Enrich with category data + return results.map(result => { + const category = this.categories.find(c => c.id === result.id); + return { + id: category.id, + name: category.name, + fullPath: category.fullPath, + parentId: category.parentId, + similarity: result.similarity + }; + }); + } + + /** + * Build full paths for categories + */ + _buildCategoryPaths(rows) { + const byId = new Map(rows.map(r => [r.cat_id, r])); + const categories = []; + + for (const row of rows) { + const path = []; + let current = row; + + // Walk up the tree to build full path + while (current) { + path.unshift(current.name); + current = current.master_cat_id ? byId.get(current.master_cat_id) : null; + } + + categories.push({ + id: row.cat_id, + name: row.name, + parentId: row.master_cat_id, + type: row.type, + fullPath: path.join(' > '), + // Text for embedding includes path for context + embeddingText: path.join(' ') + }); + } + + return categories; + } + + /** + * Compute embeddings for all categories + */ + async _computeAndCacheEmbeddings() { + this.logger?.info('[CategoryEmbeddings] Computing embeddings...'); + + const texts = this.categories.map(c => c.embeddingText); + const allEmbeddings = []; + + // Process in chunks (OpenAI has batch limits) + for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) { + for (let i = 0; i < chunk.embeddings.length; i++) { + const globalIndex = chunk.startIndex + i; + allEmbeddings.push({ + id: this.categories[globalIndex].id, + embedding: chunk.embeddings[i] + }); + } + this.logger?.info(`[CategoryEmbeddings] Processed ${chunk.endIndex}/${texts.length}`); + } + + // Store in vector store + this.vectorStore.load(allEmbeddings); + + // Cache to database for faster startup next time + await this._cacheEmbeddings(allEmbeddings); + + this.logger?.info('[CategoryEmbeddings] Embeddings computed and cached'); + } + + async _loadCachedEmbeddings() { + // TODO: Load from ai_embeddings_cache table + return null; + } + + async _cacheEmbeddings(embeddings) { + // TODO: Save to ai_embeddings_cache table + } +} + +module.exports = { CategoryEmbeddings }; +``` + +#### Vector Store + +```javascript +// services/ai/embeddings/vectorStore.js + +const { cosineSimilarity } = require('./similarity'); + +/** + * In-memory vector store for fast similarity search + */ +class VectorStore { + constructor() { + this.vectors = []; // Array of { id, embedding } + } + + /** + * Load vectors into the store + */ + load(vectors) { + this.vectors = vectors; + } + + /** + * Add a single vector + */ + add(id, embedding) { + this.vectors.push({ id, embedding }); + } + + /** + * Search for most similar vectors + */ + search(queryEmbedding, topK = 10) { + const scored = this.vectors.map(item => ({ + id: item.id, + similarity: cosineSimilarity(queryEmbedding, item.embedding) + })); + + // Sort by similarity descending + scored.sort((a, b) => b.similarity - a.similarity); + + return scored.slice(0, topK); + } + + /** + * Get store size + */ + size() { + return this.vectors.length; + } + + /** + * Clear the store + */ + clear() { + this.vectors = []; + } +} + +module.exports = { VectorStore }; +``` + +#### Similarity Utilities + +```javascript +// services/ai/embeddings/similarity.js + +/** + * Compute cosine similarity between two vectors + */ +function cosineSimilarity(a, b) { + if (a.length !== b.length) { + throw new Error('Vectors must have same length'); + } + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + const denominator = Math.sqrt(normA) * Math.sqrt(normB); + if (denominator === 0) return 0; + + return dotProduct / denominator; +} + +/** + * Compute Euclidean distance between two vectors + */ +function euclideanDistance(a, b) { + if (a.length !== b.length) { + throw new Error('Vectors must have same length'); + } + + let sum = 0; + for (let i = 0; i < a.length; i++) { + const diff = a[i] - b[i]; + sum += diff * diff; + } + + return Math.sqrt(sum); +} + +module.exports = { cosineSimilarity, euclideanDistance }; +``` + +### Micro-Prompt Tasks + +#### Name Suggestion Task + +```javascript +// services/ai/tasks/nameSuggestionTask.js + +const { buildNamePrompt } = require('../prompts/namePrompts'); + +function createNameSuggestionTask({ provider, logger, config }) { + const taskConfig = config.tasks?.nameSuggestion || {}; + + async function run({ product }) { + if (!product?.name && !product?.description) { + return { suggestion: null, reason: 'No name or description provided' }; + } + + const prompt = buildNamePrompt(product); + + const response = await provider.chatCompletion({ + messages: [{ role: 'user', content: prompt }], + model: taskConfig.model || 'llama-3.3-70b-versatile', + temperature: taskConfig.temperature || 0.2, + maxTokens: taskConfig.maxTokens || 150 + }); + + const suggestion = response.content.trim(); + + // Only return if different from original + if (suggestion === product.name) { + return { suggestion: null, unchanged: true }; + } + + return { + suggestion, + original: product.name, + usage: response.usage, + latencyMs: response.latencyMs + }; + } + + return { + id: 'suggest.name', + description: 'Suggest formatted product name', + run + }; +} + +module.exports = { createNameSuggestionTask }; +``` + +#### Category Suggestion Task + +```javascript +// services/ai/tasks/categorySuggestionTask.js + +const { suggestCategories } = require('../embeddings'); +const { buildCategoryPrompt } = require('../prompts/categoryPrompts'); + +function createCategorySuggestionTask({ provider, logger, config }) { + const taskConfig = config.tasks?.categorySuggestion || {}; + + async function run({ product }) { + const productText = `${product.name || ''} ${product.description || ''}`.trim(); + + if (!productText) { + return { suggestions: [], reason: 'No product text provided' }; + } + + // Step 1: Get top candidates via embedding similarity + const embeddingMatches = await suggestCategories(productText, 10); + + // Step 2: Use AI to pick best matches from candidates + const prompt = buildCategoryPrompt(product, embeddingMatches); + + const response = await provider.chatCompletion({ + messages: [{ role: 'user', content: prompt }], + model: taskConfig.model || 'llama-3.1-8b-instant', + temperature: taskConfig.temperature || 0.1, + maxTokens: taskConfig.maxTokens || 100, + responseFormat: { type: 'json_object' } + }); + + let selectedIds = []; + try { + const parsed = JSON.parse(response.content); + selectedIds = Array.isArray(parsed.categories) ? parsed.categories : parsed; + } catch { + logger?.warn('[CategorySuggestion] Failed to parse response', { content: response.content }); + } + + // Filter matches to only selected IDs + const suggestions = embeddingMatches + .filter(m => selectedIds.includes(m.id)) + .map(m => ({ + id: m.id, + name: m.name, + fullPath: m.fullPath, + similarity: m.similarity + })); + + return { + suggestions, + allMatches: embeddingMatches, // Include all for fallback + usage: response.usage, + latencyMs: response.latencyMs + }; + } + + return { + id: 'suggest.categories', + description: 'Suggest product categories', + run + }; +} + +module.exports = { createCategorySuggestionTask }; +``` + +#### Description Enhancement Task + +```javascript +// services/ai/tasks/descriptionEnhanceTask.js + +const { buildDescriptionPrompt } = require('../prompts/descriptionPrompts'); + +function createDescriptionEnhanceTask({ provider, logger, config }) { + const taskConfig = config.tasks?.descriptionEnhance || {}; + + /** + * Enhance descriptions for a batch of products + */ + async function run({ products, mode = 'enhance' }) { + if (!Array.isArray(products) || products.length === 0) { + return { results: [], reason: 'No products provided' }; + } + + // Process in smaller batches for reliability + const batchSize = taskConfig.batchSize || 10; + const results = []; + + for (let i = 0; i < products.length; i += batchSize) { + const batch = products.slice(i, i + batchSize); + const batchResults = await processBatch(batch, mode); + results.push(...batchResults); + } + + return { results }; + } + + async function processBatch(products, mode) { + const prompt = buildDescriptionPrompt(products, mode); + + const response = await provider.chatCompletion({ + messages: [ + { role: 'system', content: getSystemPrompt() }, + { role: 'user', content: prompt } + ], + model: taskConfig.model || 'claude-3-5-haiku-20241022', + temperature: taskConfig.temperature || 0.7, + maxTokens: taskConfig.maxTokens || 2000, + responseFormat: { type: 'json_object' } + }); + + let parsed = []; + try { + const result = JSON.parse(response.content); + parsed = result.descriptions || result; + } catch (error) { + logger?.warn('[DescriptionEnhance] Failed to parse response', { error: error.message }); + } + + // Match results back to products + return products.map((product, index) => { + const enhanced = parsed[index] || {}; + return { + productId: product._index || product.upc || index, + original: product.description, + enhanced: enhanced.description || null, + changed: enhanced.description && enhanced.description !== product.description + }; + }); + } + + function getSystemPrompt() { + return `You are a product copywriter for a craft supplies ecommerce store. +Write SEO-friendly, accurate descriptions that help customers understand what they're buying. +Always state what's included. Never use "our" - use "this" or the company name. +Keep descriptions to 2-4 sentences unless the product is complex.`; + } + + return { + id: 'enhance.descriptions', + description: 'Enhance product descriptions in batch', + run + }; +} + +module.exports = { createDescriptionEnhanceTask }; +``` + +### Prompts + +#### Name Prompts + +```javascript +// services/ai/prompts/namePrompts.js + +function buildNamePrompt(product) { + return `Format this product name for a craft supplies store. + +CURRENT NAME: "${product.name || ''}" +COMPANY: ${product.company_name || product.company || 'Unknown'} +LINE: ${product.line_name || product.line || 'None'} +PRODUCT TYPE: ${inferProductType(product)} + +NAMING RULES: +- Single product in line: [Line Name] [Product Name] - [Company] +- Multiple similar products: [Differentiator] [Product Type] - [Line Name] - [Company] +- Standalone products: [Product Name] - [Company] +- Always capitalize every word (including "The", "And", etc.) +- Paper sizes: Use "12x12", "6x6" (no spaces or units) +- All stamps → "Stamp Set" (not "Clear Stamps") +- All dies → "Dies" (not "Die Set") + +SPECIAL RULES: +- Tim Holtz from Ranger: "[Color] [Product] - Tim Holtz Distress - Ranger" +- Tim Holtz from Sizzix: "[Product] by Tim Holtz - Sizzix" +- Dylusions from Ranger: "[Product] - Dylusions - Ranger" + +Return ONLY the corrected name, nothing else.`; +} + +function inferProductType(product) { + const name = (product.name || '').toLowerCase(); + const desc = (product.description || '').toLowerCase(); + const text = `${name} ${desc}`; + + if (text.includes('stamp')) return 'Stamps'; + if (text.includes('die') || text.includes('thinlit')) return 'Dies'; + if (text.includes('paper') || text.includes('cardstock')) return 'Paper'; + if (text.includes('ink')) return 'Ink'; + if (text.includes('sticker')) return 'Stickers'; + if (text.includes('washi')) return 'Washi Tape'; + return 'Unknown'; +} + +module.exports = { buildNamePrompt }; +``` + +#### Category Prompts + +```javascript +// services/ai/prompts/categoryPrompts.js + +function buildCategoryPrompt(product, categoryMatches) { + const matchList = categoryMatches + .map(c => `${c.id}: ${c.fullPath}`) + .join('\n'); + + return `Select the best categories for this craft product. + +PRODUCT: +Name: "${product.name || ''}" +Description: "${(product.description || '').substring(0, 200)}" +Company: ${product.company_name || 'Unknown'} + +CATEGORY OPTIONS: +${matchList} + +RULES: +- Select 1-3 most specific categories +- Prefer deeper subcategories over parents +- If selecting a subcategory, don't also select its parent +- Never select "Deals" or "Black Friday" categories + +Return JSON: {"categories": [id1, id2]}`; +} + +module.exports = { buildCategoryPrompt }; +``` + +#### Description Prompts + +```javascript +// services/ai/prompts/descriptionPrompts.js + +function buildDescriptionPrompt(products, mode = 'enhance') { + const productList = products.map((p, i) => { + return `[${i}] +Name: ${p.name || 'Unknown'} +Company: ${p.company_name || 'Unknown'} +Current Description: ${p.description || '(none)'} +Categories: ${p.category_names?.join(', ') || 'Unknown'} +Dimensions: ${p.length || '?'}x${p.width || '?'} inches +Weight: ${p.weight || '?'} oz`; + }).join('\n\n'); + + const instruction = mode === 'generate' + ? 'Write new descriptions for these products.' + : 'Improve these product descriptions. Fix grammar, add missing details, make SEO-friendly.'; + + return `${instruction} + +PRODUCTS: +${productList} + +RULES: +- 2-4 sentences each, professional but friendly +- Always state what's included (quantity, size) +- Don't use "our" - use "this" or company name +- Don't add generic filler ("perfect for all your crafts") +- State facts: dimensions, compatibility, materials +- Don't make up information you're not sure about + +Return JSON: {"descriptions": [{"description": "..."}, ...]}`; +} + +module.exports = { buildDescriptionPrompt }; +``` + +### Normalizers (Tier 1 - No AI) + +```javascript +// services/ai/normalizers/index.js + +const priceNormalizer = require('./priceNormalizer'); +const upcNormalizer = require('./upcNormalizer'); +const countryCodeNormalizer = require('./countryCodeNormalizer'); +const dateNormalizer = require('./dateNormalizer'); +const numericNormalizer = require('./numericNormalizer'); + +/** + * Apply all relevant normalizers to a product + */ +function normalizeProduct(product, fieldMappings) { + const normalized = { ...product }; + const changes = []; + + // Price fields + for (const field of ['msrp', 'cost_each', 'price']) { + if (normalized[field] !== undefined) { + const result = priceNormalizer.normalize(normalized[field]); + if (result.value !== normalized[field]) { + changes.push({ field, from: normalized[field], to: result.value }); + normalized[field] = result.value; + } + } + } + + // UPC/SKU fields + for (const field of ['upc', 'supplier_no', 'notions_no', 'item_number']) { + if (normalized[field] !== undefined) { + const result = upcNormalizer.normalize(normalized[field]); + if (result.value !== normalized[field]) { + changes.push({ field, from: normalized[field], to: result.value }); + normalized[field] = result.value; + } + } + } + + // Country of origin + if (normalized.coo !== undefined) { + const result = countryCodeNormalizer.normalize(normalized.coo); + if (result.value !== normalized.coo) { + changes.push({ field: 'coo', from: normalized.coo, to: result.value }); + normalized.coo = result.value; + } + } + + // ETA date + if (normalized.eta !== undefined) { + const result = dateNormalizer.normalizeEta(normalized.eta); + if (result.value !== normalized.eta) { + changes.push({ field: 'eta', from: normalized.eta, to: result.value }); + normalized.eta = result.value; + } + } + + // Numeric fields + for (const field of ['qty_per_unit', 'case_qty']) { + if (normalized[field] !== undefined) { + const result = numericNormalizer.normalize(normalized[field]); + if (result.value !== normalized[field]) { + changes.push({ field, from: normalized[field], to: result.value }); + normalized[field] = result.value; + } + } + } + + return { normalized, changes }; +} + +module.exports = { + normalizeProduct, + priceNormalizer, + upcNormalizer, + countryCodeNormalizer, + dateNormalizer, + numericNormalizer +}; +``` + +```javascript +// services/ai/normalizers/priceNormalizer.js + +/** + * Normalize price values + * Input: "$5.00", "5", "5.5", "$1,234.56" + * Output: "5.00", "5.00", "5.50", "1234.56" + */ +function normalize(value) { + if (value === null || value === undefined || value === '') { + return { value, changed: false }; + } + + const original = String(value); + + // Remove currency symbols and commas + let cleaned = original.replace(/[$,]/g, '').trim(); + + // Try to parse as number + const num = parseFloat(cleaned); + if (isNaN(num)) { + return { value: original, changed: false, error: 'Not a valid number' }; + } + + // Format to 2 decimal places + const formatted = num.toFixed(2); + + return { + value: formatted, + changed: formatted !== original + }; +} + +module.exports = { normalize }; +``` + +```javascript +// services/ai/normalizers/countryCodeNormalizer.js + +const COUNTRY_MAP = { + // Full names + 'united states': 'US', + 'united states of america': 'US', + 'china': 'CN', + 'peoples republic of china': 'CN', + "people's republic of china": 'CN', + 'taiwan': 'TW', + 'japan': 'JP', + 'south korea': 'KR', + 'korea': 'KR', + 'india': 'IN', + 'germany': 'DE', + 'united kingdom': 'GB', + 'great britain': 'GB', + 'france': 'FR', + 'italy': 'IT', + 'spain': 'ES', + 'canada': 'CA', + 'mexico': 'MX', + 'brazil': 'BR', + 'australia': 'AU', + 'vietnam': 'VN', + 'thailand': 'TH', + 'indonesia': 'ID', + 'philippines': 'PH', + 'malaysia': 'MY', + + // Common abbreviations + 'usa': 'US', + 'u.s.a.': 'US', + 'u.s.': 'US', + 'prc': 'CN', + 'uk': 'GB', + 'u.k.': 'GB', + 'rok': 'KR' +}; + +function normalize(value) { + if (value === null || value === undefined || value === '') { + return { value, changed: false }; + } + + const original = String(value).trim(); + const lookup = original.toLowerCase(); + + // Check if it's already a valid 2-letter code + if (/^[A-Z]{2}$/.test(original)) { + return { value: original, changed: false }; + } + + // Look up in map + if (COUNTRY_MAP[lookup]) { + return { + value: COUNTRY_MAP[lookup], + changed: true + }; + } + + // If 2 chars, uppercase and return + if (original.length === 2) { + const upper = original.toUpperCase(); + return { + value: upper, + changed: upper !== original + }; + } + + // Take first 2 chars as fallback + const fallback = original.substring(0, 2).toUpperCase(); + return { + value: fallback, + changed: true, + warning: `Unknown country "${original}", using "${fallback}"` + }; +} + +module.exports = { normalize }; +``` + +### Main AI Service Entry + +```javascript +// services/ai/index.js + +const { AiTaskRegistry, TASK_IDS } = require('./taskRegistry'); +const { AiWorkQueue } = require('./workQueue'); +const { createProvider } = require('./providers'); +const { initializeEmbeddings } = require('./embeddings'); +const { normalizeProduct } = require('./normalizers'); + +// Task factories +const { createNameSuggestionTask } = require('./tasks/nameSuggestionTask'); +const { createCategorySuggestionTask } = require('./tasks/categorySuggestionTask'); +const { createThemeSuggestionTask } = require('./tasks/themeSuggestionTask'); +const { createColorSuggestionTask } = require('./tasks/colorSuggestionTask'); +const { createDescriptionEnhanceTask } = require('./tasks/descriptionEnhanceTask'); +const { createConsistencyCheckTask } = require('./tasks/consistencyCheckTask'); + +let initialized = false; +let aiEnabled = false; +let registry = null; +let workQueue = null; +let providers = {}; + +/** + * Initialize the AI system + */ +async function initialize({ config, mysqlConnection, logger }) { + if (initialized) { + return { enabled: aiEnabled }; + } + + if (!config?.ai?.enabled) { + logger?.info('[AI] AI features disabled by configuration'); + initialized = true; + aiEnabled = false; + return { enabled: false }; + } + + try { + // Initialize providers + providers.groq = createProvider('groq', config.ai); + providers.openai = createProvider('openai', config.ai); + + if (config.ai.providers?.anthropic?.apiKey) { + providers.anthropic = createProvider('anthropic', config.ai); + } + + // Initialize embeddings (requires OpenAI for embedding generation) + await initializeEmbeddings({ + openaiProvider: providers.openai, + mysqlConnection, + logger + }); + + // Initialize work queue + workQueue = new AiWorkQueue(config.ai.maxConcurrentTasks || 5); + + // Initialize task registry + registry = new AiTaskRegistry(); + + // Register Tier 2 tasks (real-time, Groq) + registry.register(createNameSuggestionTask({ + provider: providers.groq, + logger, + config: config.ai + })); + + registry.register(createCategorySuggestionTask({ + provider: providers.groq, + logger, + config: config.ai + })); + + registry.register(createThemeSuggestionTask({ + provider: providers.groq, + logger, + config: config.ai + })); + + registry.register(createColorSuggestionTask({ + provider: providers.groq, + logger, + config: config.ai + })); + + // Register Tier 3 tasks (batch, Anthropic/OpenAI) + const batchProvider = providers.anthropic || providers.openai; + + registry.register(createDescriptionEnhanceTask({ + provider: batchProvider, + logger, + config: config.ai + })); + + registry.register(createConsistencyCheckTask({ + provider: batchProvider, + logger, + config: config.ai + })); + + initialized = true; + aiEnabled = true; + logger?.info('[AI] AI system initialized successfully'); + + return { enabled: true }; + } catch (error) { + logger?.error('[AI] Failed to initialize AI system', { error: error.message }); + initialized = true; + aiEnabled = false; + return { enabled: false, error: error.message }; + } +} + +/** + * Run a task by ID + */ +async function runTask(taskId, payload = {}) { + if (!aiEnabled) { + throw new Error('AI features are disabled'); + } + + const task = registry?.get(taskId); + if (!task) { + throw new Error(`Unknown task: ${taskId}`); + } + + // Execute through work queue for concurrency control + return workQueue.enqueue(() => task.run(payload)); +} + +/** + * Normalize a product using Tier 1 (code-based) rules + */ +function normalize(product, fieldMappings) { + return normalizeProduct(product, fieldMappings); +} + +/** + * Get system status + */ +function getStatus() { + return { + enabled: aiEnabled, + initialized, + tasks: registry?.list() || [], + queue: workQueue?.getStats() || null + }; +} + +module.exports = { + initialize, + runTask, + normalize, + getStatus, + TASK_IDS +}; +``` + +### API Routes + +```javascript +// routes/ai.js + +const express = require('express'); +const router = express.Router(); +const ai = require('../services/ai'); +const { TASK_IDS } = require('../services/ai/taskRegistry'); + +/** + * Get AI system status + */ +router.get('/status', (req, res) => { + res.json(ai.getStatus()); +}); + +/** + * Normalize a product (Tier 1 - no AI) + */ +router.post('/normalize', (req, res) => { + try { + const { product, fieldMappings } = req.body; + const result = ai.normalize(product, fieldMappings); + res.json(result); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +/** + * Normalize multiple products (Tier 1 - no AI) + */ +router.post('/normalize/batch', (req, res) => { + try { + const { products, fieldMappings } = req.body; + const results = products.map(product => ai.normalize(product, fieldMappings)); + res.json({ results }); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +/** + * Get name suggestion (Tier 2 - Groq) + */ +router.post('/suggest/name', async (req, res) => { + try { + const { product } = req.body; + const result = await ai.runTask(TASK_IDS.SUGGEST_NAME, { product }); + res.json(result); + } catch (error) { + console.error('[AI] Name suggestion error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Get category suggestions (Tier 2 - Embeddings + Groq) + */ +router.post('/suggest/categories', async (req, res) => { + try { + const { product } = req.body; + const result = await ai.runTask(TASK_IDS.SUGGEST_CATEGORIES, { product }); + res.json(result); + } catch (error) { + console.error('[AI] Category suggestion error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Get theme suggestions (Tier 2 - Embeddings + Groq) + */ +router.post('/suggest/themes', async (req, res) => { + try { + const { product } = req.body; + const result = await ai.runTask(TASK_IDS.SUGGEST_THEMES, { product }); + res.json(result); + } catch (error) { + console.error('[AI] Theme suggestion error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Get color suggestions (Tier 2 - Groq) + */ +router.post('/suggest/colors', async (req, res) => { + try { + const { product } = req.body; + const result = await ai.runTask(TASK_IDS.SUGGEST_COLORS, { product }); + res.json(result); + } catch (error) { + console.error('[AI] Color suggestion error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Enhance descriptions (Tier 3 - Batch) + */ +router.post('/enhance/descriptions', async (req, res) => { + try { + const { products, mode = 'enhance' } = req.body; + const result = await ai.runTask(TASK_IDS.ENHANCE_DESCRIPTIONS, { products, mode }); + res.json(result); + } catch (error) { + console.error('[AI] Description enhancement error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Check consistency across products (Tier 3 - Batch) + */ +router.post('/check/consistency', async (req, res) => { + try { + const { products } = req.body; + const result = await ai.runTask(TASK_IDS.CHECK_CONSISTENCY, { products }); + res.json(result); + } catch (error) { + console.error('[AI] Consistency check error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * Legacy endpoint - full validation (redirects to new system) + * Kept for backwards compatibility during migration + */ +router.post('/validate', async (req, res) => { + // TODO: Implement migration path + // For now, combine Tier 1 normalization with Tier 3 batch processing + res.status(501).json({ + error: 'Legacy validation endpoint deprecated. Use /normalize, /suggest/*, and /enhance/* endpoints.' + }); +}); + +module.exports = router; +``` + +--- + +## Frontend Implementation + +### Suggestion Hooks + +#### Base Hook + +```typescript +// hooks/ai/useAiSuggestion.ts + +import { useState, useCallback } from 'react'; +import { useDebouncedCallback } from 'use-debounce'; + +interface SuggestionResult { + suggestion: T | null; + isLoading: boolean; + error: string | null; + latencyMs: number | null; +} + +interface UseAiSuggestionOptions { + debounceMs?: number; + enabled?: boolean; +} + +export function useAiSuggestion( + endpoint: string, + options: UseAiSuggestionOptions = {} +) { + const { debounceMs = 300, enabled = true } = options; + + const [result, setResult] = useState>({ + suggestion: null, + isLoading: false, + error: null, + latencyMs: null + }); + + const fetchSuggestion = useCallback(async (payload: unknown) => { + if (!enabled) return; + + setResult(prev => ({ ...prev, isLoading: true, error: null })); + + try { + const response = await fetch(`/api/ai${endpoint}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + + const data = await response.json(); + + setResult({ + suggestion: data.suggestion ?? data.suggestions ?? data, + isLoading: false, + error: null, + latencyMs: data.latencyMs ?? null + }); + } catch (error) { + setResult(prev => ({ + ...prev, + isLoading: false, + error: error instanceof Error ? error.message : 'Unknown error' + })); + } + }, [endpoint, enabled]); + + const debouncedFetch = useDebouncedCallback(fetchSuggestion, debounceMs); + + const clear = useCallback(() => { + setResult({ + suggestion: null, + isLoading: false, + error: null, + latencyMs: null + }); + }, []); + + return { + ...result, + fetch: fetchSuggestion, + fetchDebounced: debouncedFetch, + clear + }; +} +``` + +#### Name Suggestion Hook + +```typescript +// hooks/ai/useNameSuggestion.ts + +import { useAiSuggestion } from './useAiSuggestion'; +import { ProductRow } from '@/types/import'; + +interface NameSuggestionResult { + suggestion: string | null; + original: string; + unchanged?: boolean; +} + +export function useNameSuggestion() { + const { + suggestion, + isLoading, + error, + fetch, + fetchDebounced, + clear + } = useAiSuggestion('/suggest/name'); + + const suggest = (product: Partial) => { + if (!product.name && !product.description) return; + fetchDebounced({ product }); + }; + + const suggestImmediate = (product: Partial) => { + if (!product.name && !product.description) return; + fetch({ product }); + }; + + return { + suggestion: suggestion?.suggestion ?? null, + original: suggestion?.original ?? null, + unchanged: suggestion?.unchanged ?? false, + isLoading, + error, + suggest, + suggestImmediate, + clear + }; +} +``` + +#### Category Suggestion Hook + +```typescript +// hooks/ai/useCategorySuggestion.ts + +import { useAiSuggestion } from './useAiSuggestion'; +import { ProductRow } from '@/types/import'; + +interface CategoryMatch { + id: number; + name: string; + fullPath: string; + similarity: number; +} + +interface CategorySuggestionResult { + suggestions: CategoryMatch[]; + allMatches: CategoryMatch[]; +} + +export function useCategorySuggestion() { + const { + suggestion, + isLoading, + error, + fetch, + clear + } = useAiSuggestion('/suggest/categories', { + debounceMs: 500 // Longer debounce for embedding lookup + }); + + const suggest = (product: Partial) => { + const hasText = product.name || product.description; + if (!hasText) return; + fetch({ product }); + }; + + return { + suggestions: suggestion?.suggestions ?? [], + allMatches: suggestion?.allMatches ?? [], + isLoading, + error, + suggest, + clear + }; +} +``` + +#### Batch Enhancement Hook + +```typescript +// hooks/ai/useDescriptionEnhancement.ts + +import { useState, useCallback } from 'react'; +import { ProductRow } from '@/types/import'; + +interface EnhancementResult { + productId: string | number; + original: string | null; + enhanced: string | null; + changed: boolean; +} + +interface UseDescriptionEnhancementResult { + results: EnhancementResult[]; + isProcessing: boolean; + progress: { current: number; total: number }; + error: string | null; + enhance: (products: ProductRow[], mode?: 'enhance' | 'generate') => Promise; + cancel: () => void; +} + +export function useDescriptionEnhancement(): UseDescriptionEnhancementResult { + const [results, setResults] = useState([]); + const [isProcessing, setIsProcessing] = useState(false); + const [progress, setProgress] = useState({ current: 0, total: 0 }); + const [error, setError] = useState(null); + const [abortController, setAbortController] = useState(null); + + const enhance = useCallback(async ( + products: ProductRow[], + mode: 'enhance' | 'generate' = 'enhance' + ) => { + const controller = new AbortController(); + setAbortController(controller); + setIsProcessing(true); + setProgress({ current: 0, total: products.length }); + setResults([]); + setError(null); + + try { + const response = await fetch('/api/ai/enhance/descriptions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ products, mode }), + signal: controller.signal + }); + + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + + const data = await response.json(); + setResults(data.results || []); + setProgress({ current: products.length, total: products.length }); + } catch (err) { + if (err instanceof Error && err.name === 'AbortError') { + // Cancelled by user + return; + } + setError(err instanceof Error ? err.message : 'Unknown error'); + } finally { + setIsProcessing(false); + setAbortController(null); + } + }, []); + + const cancel = useCallback(() => { + abortController?.abort(); + }, [abortController]); + + return { + results, + isProcessing, + progress, + error, + enhance, + cancel + }; +} +``` + +### UI Components + +#### Suggestion Badge + +```tsx +// components/ai/SuggestionBadge.tsx + +import { Check, X, Sparkles } from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { cn } from '@/lib/utils'; + +interface SuggestionBadgeProps { + suggestion: string; + onAccept: () => void; + onDismiss: () => void; + className?: string; +} + +export function SuggestionBadge({ + suggestion, + onAccept, + onDismiss, + className +}: SuggestionBadgeProps) { + return ( +
+ + + {suggestion} + +
+ + +
+
+ ); +} +``` + +#### Category Suggestion Dropdown + +```tsx +// components/ai/CategorySuggestionDropdown.tsx + +import { useState, useEffect } from 'react'; +import { Sparkles, ChevronDown } from 'lucide-react'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger +} from '@/components/ui/dropdown-menu'; +import { Button } from '@/components/ui/button'; +import { Badge } from '@/components/ui/badge'; +import { useCategorySuggestion } from '@/hooks/ai/useCategorySuggestion'; +import { ProductRow } from '@/types/import'; + +interface CategorySuggestionDropdownProps { + product: ProductRow; + currentCategories: number[]; + onSelect: (categoryId: number) => void; + allCategories: Array<{ id: number; name: string; fullPath: string }>; +} + +export function CategorySuggestionDropdown({ + product, + currentCategories, + onSelect, + allCategories +}: CategorySuggestionDropdownProps) { + const { suggestions, allMatches, isLoading, suggest, clear } = useCategorySuggestion(); + const [isOpen, setIsOpen] = useState(false); + + // Fetch suggestions when dropdown opens + useEffect(() => { + if (isOpen && suggestions.length === 0) { + suggest(product); + } + }, [isOpen, product, suggest, suggestions.length]); + + // Clear when dropdown closes + useEffect(() => { + if (!isOpen) { + clear(); + } + }, [isOpen, clear]); + + return ( + + + + + + + {/* AI Suggestions Section */} + {(isLoading || suggestions.length > 0) && ( + <> +
+ + AI Suggested + {isLoading && (loading...)} +
+ + {suggestions.map(cat => ( + onSelect(cat.id)} + className="flex items-center justify-between" + > + {cat.fullPath} + + {Math.round(cat.similarity * 100)}% + + + ))} + + + + )} + + {/* All Categories Section */} +
+ All Categories +
+ +
+ {allCategories.slice(0, 50).map(cat => ( + onSelect(cat.id)} + disabled={currentCategories.includes(cat.id)} + > + {cat.fullPath} + + ))} +
+ + + ); +} +``` + +#### AI Validation Cell + +```tsx +// components/ai/AiValidationCell.tsx + +import { useState, useEffect } from 'react'; +import { Loader2 } from 'lucide-react'; +import { Input } from '@/components/ui/input'; +import { SuggestionBadge } from './SuggestionBadge'; +import { useNameSuggestion } from '@/hooks/ai/useNameSuggestion'; +import { ProductRow } from '@/types/import'; + +interface AiValidationCellProps { + field: 'name' | 'description'; + value: string; + product: ProductRow; + onChange: (value: string) => void; + onBlur?: () => void; +} + +export function AiValidationCell({ + field, + value, + product, + onChange, + onBlur +}: AiValidationCellProps) { + const { + suggestion, + isLoading, + suggest, + clear + } = useNameSuggestion(); + + const [localValue, setLocalValue] = useState(value); + const [showSuggestion, setShowSuggestion] = useState(false); + + // Sync external value changes + useEffect(() => { + setLocalValue(value); + }, [value]); + + // Show suggestion when it arrives and differs from current value + useEffect(() => { + if (suggestion && suggestion !== localValue) { + setShowSuggestion(true); + } + }, [suggestion, localValue]); + + const handleBlur = () => { + // Trigger AI suggestion on blur + if (field === 'name') { + suggest({ ...product, name: localValue }); + } + onBlur?.(); + }; + + const handleAccept = () => { + if (suggestion) { + setLocalValue(suggestion); + onChange(suggestion); + setShowSuggestion(false); + clear(); + } + }; + + const handleDismiss = () => { + setShowSuggestion(false); + clear(); + }; + + return ( +
+
+ { + setLocalValue(e.target.value); + onChange(e.target.value); + }} + onBlur={handleBlur} + className="pr-8" + /> + {isLoading && ( +
+ +
+ )} +
+ + {showSuggestion && suggestion && ( + + )} +
+ ); +} +``` + +#### Batch Enhancement Button + +```tsx +// components/ai/EnhanceDescriptionsButton.tsx + +import { useState } from 'react'; +import { Sparkles, Loader2, CheckCircle } from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle +} from '@/components/ui/dialog'; +import { Progress } from '@/components/ui/progress'; +import { useDescriptionEnhancement } from '@/hooks/ai/useDescriptionEnhancement'; +import { useValidationStore } from '../store/validationStore'; + +export function EnhanceDescriptionsButton() { + const [showDialog, setShowDialog] = useState(false); + const rows = useValidationStore(state => state.rows); + const updateRow = useValidationStore(state => state.updateRow); + + const { + results, + isProcessing, + progress, + error, + enhance, + cancel + } = useDescriptionEnhancement(); + + const handleEnhance = async () => { + setShowDialog(true); + await enhance(rows.map(r => r.data)); + }; + + const handleApply = () => { + // Apply enhanced descriptions to store + for (const result of results) { + if (result.changed && result.enhanced) { + const rowIndex = rows.findIndex( + r => r.data._index === result.productId || r.data.upc === result.productId + ); + if (rowIndex !== -1) { + updateRow(rowIndex, { description: result.enhanced }); + } + } + } + setShowDialog(false); + }; + + const changedCount = results.filter(r => r.changed).length; + const progressPercent = progress.total > 0 + ? Math.round((progress.current / progress.total) * 100) + : 0; + + return ( + <> + + + + + + Enhance Descriptions + + AI will improve product descriptions for SEO and clarity. + + + +
+ {isProcessing ? ( +
+
+ + Processing {progress.current} of {progress.total} products... +
+ +
+ ) : error ? ( +
{error}
+ ) : results.length > 0 ? ( +
+
+ + Enhanced {changedCount} descriptions +
+

+ {results.length - changedCount} descriptions were already good or unchanged. +

+
+ ) : null} +
+ + + {isProcessing ? ( + + ) : results.length > 0 ? ( + <> + + + + ) : null} + + + + + ); +} +``` + +### Integration with ValidationStep + +```tsx +// Example integration in ValidationContainer.tsx + +import { AiValidationCell } from '@/components/ai/AiValidationCell'; +import { CategorySuggestionDropdown } from '@/components/ai/CategorySuggestionDropdown'; +import { EnhanceDescriptionsButton } from '@/components/ai/EnhanceDescriptionsButton'; + +// In the toolbar +
+ + {/* Other toolbar items */} +
+ +// In the data grid column definitions +const columns = [ + { + key: 'name', + name: 'Name', + renderCell: ({ row, onRowChange }) => ( + onRowChange({ ...row, name: value })} + /> + ) + }, + { + key: 'categories', + name: 'Categories', + renderCell: ({ row, onRowChange }) => ( + { + const current = parseCategories(row.categories); + onRowChange({ + ...row, + categories: [...current, catId].join(',') + }); + }} + allCategories={allCategories} + /> + ) + } +]; +``` + +--- + +## Database Schema + +### New Tables + +```sql +-- Embedding cache for faster startup +CREATE TABLE IF NOT EXISTS ai_embedding_cache ( + id SERIAL PRIMARY KEY, + entity_type VARCHAR(50) NOT NULL, -- 'category', 'theme', 'color' + entity_id INTEGER NOT NULL, + embedding_model VARCHAR(100) NOT NULL, + embedding VECTOR(1536), -- Using pgvector extension + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(entity_type, entity_id, embedding_model) +); + +-- Index for fast lookups +CREATE INDEX idx_embedding_cache_lookup +ON ai_embedding_cache(entity_type, embedding_model); + +-- AI suggestion history (for analytics and improvement) +CREATE TABLE IF NOT EXISTS ai_suggestion_log ( + id SERIAL PRIMARY KEY, + task_id VARCHAR(100) NOT NULL, + product_identifier VARCHAR(255), + suggestion JSONB, + accepted BOOLEAN DEFAULT NULL, + latency_ms INTEGER, + token_usage JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Index for analytics queries +CREATE INDEX idx_suggestion_log_task +ON ai_suggestion_log(task_id, created_at); +``` + +### Configuration Table Updates + +```sql +-- Add AI configuration to existing config or create new +CREATE TABLE IF NOT EXISTS ai_config ( + id SERIAL PRIMARY KEY, + key VARCHAR(100) UNIQUE NOT NULL, + value JSONB NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Insert default config +INSERT INTO ai_config (key, value) VALUES +('providers', '{ + "groq": { + "enabled": true, + "model": "llama-3.3-70b-versatile" + }, + "openai": { + "enabled": true, + "embeddingModel": "text-embedding-3-small" + }, + "anthropic": { + "enabled": true, + "model": "claude-3-5-haiku-20241022" + } +}'), +('tasks', '{ + "nameSuggestion": { + "model": "llama-3.3-70b-versatile", + "temperature": 0.2, + "maxTokens": 150 + }, + "categorySuggestion": { + "model": "llama-3.1-8b-instant", + "temperature": 0.1, + "maxTokens": 100 + }, + "descriptionEnhance": { + "model": "claude-3-5-haiku-20241022", + "temperature": 0.7, + "maxTokens": 2000, + "batchSize": 10 + } +}'); +``` + +--- + +## Migration Strategy + +### Phase 1: Add New System Alongside Old (Week 1-2) + +1. Implement `services/ai/` structure +2. Add new `/api/ai/*` routes +3. Keep old `/api/ai-validation/*` routes working +4. Add feature flag to enable new system per-user + +### Phase 2: Frontend Integration (Week 2-3) + +1. Add suggestion hooks +2. Integrate `AiValidationCell` for name field first +3. Add `CategorySuggestionDropdown` +4. Add `EnhanceDescriptionsButton` for batch descriptions + +### Phase 3: Replace Batch Validation (Week 3-4) + +1. Update "AI Validate" button to use new tiered system +2. Remove old giant-prompt approach +3. Keep old endpoint as deprecated fallback +4. Monitor costs and latency + +### Phase 4: Cleanup (Week 4+) + +1. Remove old `ai-validation.js` route +2. Remove old frontend components +3. Archive old prompts +4. Document new system + +--- + +## Cost Analysis + +### Current Costs (GPT-5.2 Reasoning) + +| Metric | Value | +|--------|-------| +| Input tokens (est.) | ~25,000 | +| Output tokens (est.) | ~5,000 | +| Cost per run | $0.30-0.50 | +| Runs per day (est.) | 20 | +| **Monthly cost** | **~$200-300** | + +### Projected Costs (New System) + +| Tier | Model | Cost per call | Calls per product | Cost per product | +|------|-------|---------------|-------------------|------------------| +| Tier 1 | Code | $0 | N/A | $0 | +| Tier 2 | Groq Llama 3.3 70B | ~$0.0003 | 3-5 | ~$0.001 | +| Tier 2 | OpenAI Embeddings | ~$0.00002 | 1 | ~$0.00002 | +| Tier 3 | Claude Haiku | ~$0.001 | 0.1 (batch) | ~$0.0001 | + +**Per 50 products:** +- Tier 1: $0 +- Tier 2: ~$0.05 (if all fields suggested) +- Tier 3: ~$0.01 (description batch) +- **Total: ~$0.06** + +**Monthly projection (20 runs/day × 50 products):** +- Current: $200-300 +- New: ~$36 +- **Savings: 80-90%** + +--- + +## Next Steps + +1. **Review this document** - Confirm approach aligns with expectations +2. **Set up Groq account** - Get API key for real-time inference +3. **Implement providers** - Start with Groq + OpenAI +4. **Build embedding system** - Pre-compute category embeddings +5. **Create first task** - Start with name suggestion +6. **Integrate frontend** - Add suggestion badge to name field +7. **Iterate** - Add more tasks based on feedback + +--- + +## Appendix: Key Files Reference + +### Email App AI System (Reference) +- `/Users/matt/Dev/email/email-server/services/ai/index.js` - Main entry +- `/Users/matt/Dev/email/email-server/services/ai/providers/groqProvider.js` - Groq implementation +- `/Users/matt/Dev/email/email-server/services/ai/taskRegistry.js` - Task system +- `/Users/matt/Dev/email/email-server/services/ai/workQueue.js` - Concurrency + +### Current Inventory AI System +- `/inventory-server/src/routes/ai-validation.js` - Current monolithic implementation +- `/inventory/src/components/product-import/steps/ValidationStep/hooks/useAiValidation/` - Current frontend hooks diff --git a/inventory-server/scripts/embedding-poc.js b/inventory-server/scripts/embedding-poc.js new file mode 100644 index 0000000..0235dd1 --- /dev/null +++ b/inventory-server/scripts/embedding-poc.js @@ -0,0 +1,283 @@ +#!/usr/bin/env node +/** + * Embedding Proof-of-Concept Script + * + * Demonstrates how category embeddings work for product matching. + * Uses OpenAI text-embedding-3-small model. + * + * Usage: node scripts/embedding-poc.js + */ + +const path = require('path'); +require('dotenv').config({ path: path.join(__dirname, '../.env') }); + +const { getDbConnection, closeAllConnections } = require('../src/utils/dbConnection'); + +// ============================================================================ +// Configuration +// ============================================================================ + +const OPENAI_API_KEY = process.env.OPENAI_API_KEY; +const EMBEDDING_MODEL = 'text-embedding-3-small'; +const EMBEDDING_DIMENSIONS = 1536; + +// Sample products to test (you can modify these) +const TEST_PRODUCTS = [ + { + name: "Cosmos Infinity Chipboard - Stamperia", + description: "Laser-cut chipboard shapes featuring celestial designs for mixed media projects" + }, + { + name: "Distress Oxide Ink Pad - Mermaid Lagoon", + description: "Water-reactive dye ink that creates an oxidized effect" + }, + { + name: "Hedwig Puffy Stickers - Paper House Productions", + description: "3D puffy stickers featuring Harry Potter's owl Hedwig" + }, + { + name: "Black Velvet Watercolor Brush Size 6", + description: "Round brush for watercolor painting with synthetic bristles" + }, + { + name: "Floral Washi Tape Set", + description: "Decorative paper tape with flower patterns, pack of 6 rolls" + } +]; + +// ============================================================================ +// OpenAI Embedding Functions +// ============================================================================ + +async function getEmbeddings(texts) { + const response = await fetch('https://api.openai.com/v1/embeddings', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${OPENAI_API_KEY}` + }, + body: JSON.stringify({ + input: texts.map(t => t.substring(0, 8000)), // Max 8k chars per text + model: EMBEDDING_MODEL, + dimensions: EMBEDDING_DIMENSIONS + }) + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(`OpenAI API error: ${error.error?.message || response.status}`); + } + + const data = await response.json(); + + // Sort by index to ensure order matches input + const sorted = data.data.sort((a, b) => a.index - b.index); + + return { + embeddings: sorted.map(item => item.embedding), + usage: data.usage, + model: data.model + }; +} + +// ============================================================================ +// Vector Math +// ============================================================================ + +function cosineSimilarity(a, b) { + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +function findTopMatches(queryEmbedding, categoryEmbeddings, topK = 10) { + const scored = categoryEmbeddings.map(cat => ({ + ...cat, + similarity: cosineSimilarity(queryEmbedding, cat.embedding) + })); + + scored.sort((a, b) => b.similarity - a.similarity); + + return scored.slice(0, topK); +} + +// ============================================================================ +// Database Functions +// ============================================================================ + +async function fetchCategories(connection) { + console.log('\n📂 Fetching categories from database...'); + + // Fetch hierarchical categories (types 10-13) + const [rows] = await connection.query(` + SELECT + cat_id, + name, + master_cat_id, + type + FROM product_categories + WHERE type IN (10, 11, 12, 13) + ORDER BY type, name + `); + + console.log(` Found ${rows.length} category records`); + + // Build category paths + const byId = new Map(rows.map(r => [r.cat_id, r])); + const categories = []; + + for (const row of rows) { + const path = []; + let current = row; + + // Walk up the tree to build full path + while (current) { + path.unshift(current.name); + current = current.master_cat_id ? byId.get(current.master_cat_id) : null; + } + + categories.push({ + id: row.cat_id, + name: row.name, + type: row.type, + fullPath: path.join(' > '), + embeddingText: path.join(' ') // For embedding generation + }); + } + + // Count by level + const levels = { + 10: categories.filter(c => c.type === 10).length, + 11: categories.filter(c => c.type === 11).length, + 12: categories.filter(c => c.type === 12).length, + 13: categories.filter(c => c.type === 13).length, + }; + + console.log(` Level breakdown: ${levels[10]} top-level, ${levels[11]} L2, ${levels[12]} L3, ${levels[13]} L4`); + + return categories; +} + +// ============================================================================ +// Main Script +// ============================================================================ + +async function main() { + console.log('═══════════════════════════════════════════════════════════════'); + console.log(' EMBEDDING PROOF-OF-CONCEPT'); + console.log(' Model: ' + EMBEDDING_MODEL); + console.log('═══════════════════════════════════════════════════════════════'); + + if (!OPENAI_API_KEY) { + console.error('❌ OPENAI_API_KEY not found in environment'); + process.exit(1); + } + + let connection; + + try { + // Step 1: Connect to database + console.log('\n🔌 Connecting to database via SSH tunnel...'); + const { connection: conn } = await getDbConnection(); + connection = conn; + console.log(' ✅ Connected'); + + // Step 2: Fetch categories + const categories = await fetchCategories(connection); + + // Step 3: Generate embeddings for categories + console.log('\n🧮 Generating embeddings for categories...'); + console.log(' This will cost approximately $' + (categories.length * 0.00002).toFixed(4)); + + const startTime = Date.now(); + + // Process in batches of 100 (OpenAI limit is 2048) + const BATCH_SIZE = 100; + let totalTokens = 0; + + for (let i = 0; i < categories.length; i += BATCH_SIZE) { + const batch = categories.slice(i, i + BATCH_SIZE); + const texts = batch.map(c => c.embeddingText); + + const result = await getEmbeddings(texts); + + // Attach embeddings to categories + for (let j = 0; j < batch.length; j++) { + batch[j].embedding = result.embeddings[j]; + } + + totalTokens += result.usage.total_tokens; + console.log(` Batch ${Math.floor(i / BATCH_SIZE) + 1}/${Math.ceil(categories.length / BATCH_SIZE)}: ${batch.length} categories embedded`); + } + + const embeddingTime = Date.now() - startTime; + console.log(` ✅ Generated ${categories.length} embeddings in ${embeddingTime}ms`); + console.log(` 📊 Total tokens used: ${totalTokens} (~$${(totalTokens * 0.00002).toFixed(4)})`); + + // Step 4: Test with sample products + console.log('\n═══════════════════════════════════════════════════════════════'); + console.log(' TESTING WITH SAMPLE PRODUCTS'); + console.log('═══════════════════════════════════════════════════════════════'); + + for (const product of TEST_PRODUCTS) { + console.log('\n┌─────────────────────────────────────────────────────────────'); + console.log(`│ Product: "${product.name}"`); + console.log(`│ Description: "${product.description.substring(0, 60)}..."`); + console.log('├─────────────────────────────────────────────────────────────'); + + // Generate embedding for product + const productText = `${product.name} ${product.description}`; + const { embeddings: [productEmbedding] } = await getEmbeddings([productText]); + + // Find top matches + const matches = findTopMatches(productEmbedding, categories, 10); + + console.log('│ Top 10 Category Matches:'); + matches.forEach((match, i) => { + const similarity = (match.similarity * 100).toFixed(1); + const bar = '█'.repeat(Math.round(match.similarity * 20)); + const marker = i < 3 ? ' ✅' : ''; + console.log(`│ ${(i + 1).toString().padStart(2)}. [${similarity.padStart(5)}%] ${bar.padEnd(20)} ${match.fullPath}${marker}`); + }); + console.log('└─────────────────────────────────────────────────────────────'); + } + + // Step 5: Summary + console.log('\n═══════════════════════════════════════════════════════════════'); + console.log(' SUMMARY'); + console.log('═══════════════════════════════════════════════════════════════'); + console.log(` Categories embedded: ${categories.length}`); + console.log(` Embedding time: ${embeddingTime}ms (one-time cost)`); + console.log(` Per-product lookup: ~${(Date.now() - startTime) / TEST_PRODUCTS.length}ms`); + console.log(` Vector dimensions: ${EMBEDDING_DIMENSIONS}`); + console.log(` Memory usage: ~${(categories.length * EMBEDDING_DIMENSIONS * 4 / 1024 / 1024).toFixed(2)} MB (in-memory vectors)`); + console.log(''); + console.log(' 💡 In production:'); + console.log(' - Category embeddings are computed once and cached'); + console.log(' - Only product embedding is computed per-request (~$0.00002)'); + console.log(' - Vector search is instant (in-memory cosine similarity)'); + console.log(' - Top 10 results go to AI for final selection (~$0.0001)'); + console.log('═══════════════════════════════════════════════════════════════\n'); + + } catch (error) { + console.error('\n❌ Error:', error.message); + if (error.stack) { + console.error(error.stack); + } + process.exit(1); + } finally { + await closeAllConnections(); + console.log('🔌 Database connections closed'); + } +} + +// Run the script +main(); diff --git a/inventory-server/src/routes/ai.js b/inventory-server/src/routes/ai.js new file mode 100644 index 0000000..f3a5ebe --- /dev/null +++ b/inventory-server/src/routes/ai.js @@ -0,0 +1,281 @@ +/** + * AI Routes + * + * API endpoints for AI-powered product validation features. + * Provides embedding generation and similarity-based suggestions. + */ + +const express = require('express'); +const router = express.Router(); +const aiService = require('../services/ai'); +const { getDbConnection, closeAllConnections } = require('../utils/dbConnection'); + +// Track initialization state +let initializationPromise = null; + +/** + * Ensure AI service is initialized + * Uses lazy initialization on first request + */ +async function ensureInitialized() { + if (aiService.isReady()) { + return true; + } + + if (initializationPromise) { + await initializationPromise; + return aiService.isReady(); + } + + initializationPromise = (async () => { + try { + console.log('[AI Routes] Initializing AI service...'); + + // Get database connection for taxonomy + const { connection } = await getDbConnection(); + + const result = await aiService.initialize({ + openaiApiKey: process.env.OPENAI_API_KEY, + mysqlConnection: connection, + logger: console + }); + + if (!result.success) { + console.error('[AI Routes] AI service initialization failed:', result.message); + return false; + } + + console.log('[AI Routes] AI service initialized:', result.stats); + return true; + } catch (error) { + console.error('[AI Routes] Failed to initialize AI service:', error); + return false; + } + })(); + + await initializationPromise; + return aiService.isReady(); +} + +/** + * GET /api/ai/status + * Get AI service status + */ +router.get('/status', async (req, res) => { + try { + const status = aiService.getStatus(); + res.json(status); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/initialize + * Manually trigger initialization (also happens automatically on first use) + */ +router.post('/initialize', async (req, res) => { + try { + const ready = await ensureInitialized(); + const status = aiService.getStatus(); + + res.json({ + success: ready, + ...status + }); + } catch (error) { + console.error('[AI Routes] Initialize error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * GET /api/ai/taxonomy + * Get all taxonomy data (categories, themes, colors) without embeddings + */ +router.get('/taxonomy', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const taxonomy = aiService.getTaxonomyData(); + res.json(taxonomy); + } catch (error) { + console.error('[AI Routes] Taxonomy error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/embedding + * Generate embedding for a single product + * + * Body: { product: { name, description, company_name, line_name } } + * Returns: { embedding: number[], latencyMs: number } + */ +router.post('/embedding', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const { product } = req.body; + + if (!product) { + return res.status(400).json({ error: 'Product is required' }); + } + + const result = await aiService.getProductEmbedding(product); + res.json(result); + } catch (error) { + console.error('[AI Routes] Embedding error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/embeddings + * Generate embeddings for multiple products + * + * Body: { products: Array<{ name, description, company_name, line_name }> } + * Returns: { embeddings: Array<{ index, embedding }>, latencyMs } + */ +router.post('/embeddings', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const { products } = req.body; + + if (!Array.isArray(products)) { + return res.status(400).json({ error: 'Products array is required' }); + } + + const result = await aiService.getProductEmbeddings(products); + res.json(result); + } catch (error) { + console.error('[AI Routes] Embeddings error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/suggestions + * Get category/theme/color suggestions for a single product + * Generates embedding and finds similar taxonomy items + * + * Body: { product: { name, description, company_name, line_name }, options?: { topCategories, topThemes, topColors } } + * Returns: { categories: Array, themes: Array, colors: Array, latencyMs } + */ +router.post('/suggestions', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const { product, options } = req.body; + + if (!product) { + return res.status(400).json({ error: 'Product is required' }); + } + + const suggestions = await aiService.getSuggestionsForProduct(product, options); + res.json(suggestions); + } catch (error) { + console.error('[AI Routes] Suggestions error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/suggestions/batch + * Get suggestions for multiple products + * More efficient than calling /suggestions multiple times + * + * Body: { products: Array, options?: { topCategories, topThemes, topColors } } + * Returns: { results: Array<{ index, categories, themes, colors }>, latencyMs } + */ +router.post('/suggestions/batch', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const { products, options } = req.body; + + if (!Array.isArray(products)) { + return res.status(400).json({ error: 'Products array is required' }); + } + + const startTime = Date.now(); + + // Generate all embeddings at once + const { embeddings, latencyMs: embeddingLatency } = await aiService.getProductEmbeddings(products); + + // Find suggestions for each embedding + const results = embeddings.map(({ index, embedding }) => { + const suggestions = aiService.findSimilarTaxonomy(embedding, options); + return { + index, + ...suggestions + }; + }); + + const totalLatency = Date.now() - startTime; + + res.json({ + results, + latencyMs: totalLatency, + embeddingLatencyMs: embeddingLatency, + searchLatencyMs: totalLatency - embeddingLatency, + productCount: products.length, + embeddingCount: embeddings.length + }); + } catch (error) { + console.error('[AI Routes] Batch suggestions error:', error); + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /api/ai/similar + * Find similar taxonomy items given a pre-computed embedding + * Useful when frontend has cached the embedding + * + * Body: { embedding: number[], options?: { topCategories, topThemes, topColors } } + * Returns: { categories, themes, colors } + */ +router.post('/similar', async (req, res) => { + try { + const ready = await ensureInitialized(); + if (!ready) { + return res.status(503).json({ error: 'AI service not available' }); + } + + const { embedding, options } = req.body; + + if (!embedding || !Array.isArray(embedding)) { + return res.status(400).json({ error: 'Embedding array is required' }); + } + + const startTime = Date.now(); + const suggestions = aiService.findSimilarTaxonomy(embedding, options); + + res.json({ + ...suggestions, + latencyMs: Date.now() - startTime + }); + } catch (error) { + console.error('[AI Routes] Similar error:', error); + res.status(500).json({ error: error.message }); + } +}); + +module.exports = router; diff --git a/inventory-server/src/server.js b/inventory-server/src/server.js index ac8a4d5..bb7e8a8 100644 --- a/inventory-server/src/server.js +++ b/inventory-server/src/server.js @@ -15,6 +15,7 @@ const configRouter = require('./routes/config'); const metricsRouter = require('./routes/metrics'); const importRouter = require('./routes/import'); const aiValidationRouter = require('./routes/ai-validation'); +const aiRouter = require('./routes/ai'); const templatesRouter = require('./routes/templates'); const aiPromptsRouter = require('./routes/ai-prompts'); const reusableImagesRouter = require('./routes/reusable-images'); @@ -124,6 +125,7 @@ async function startServer() { app.use('/api/brands-aggregate', brandsAggregateRouter); app.use('/api/import', importRouter); app.use('/api/ai-validation', aiValidationRouter); + app.use('/api/ai', aiRouter); app.use('/api/templates', templatesRouter); app.use('/api/ai-prompts', aiPromptsRouter); app.use('/api/reusable-images', reusableImagesRouter); diff --git a/inventory-server/src/services/ai/embeddings/similarity.js b/inventory-server/src/services/ai/embeddings/similarity.js new file mode 100644 index 0000000..8803b36 --- /dev/null +++ b/inventory-server/src/services/ai/embeddings/similarity.js @@ -0,0 +1,82 @@ +/** + * Vector similarity utilities + */ + +/** + * Compute cosine similarity between two vectors + * @param {number[]} a + * @param {number[]} b + * @returns {number} Similarity score between -1 and 1 + */ +function cosineSimilarity(a, b) { + if (!a || !b || a.length !== b.length) { + return 0; + } + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + const denominator = Math.sqrt(normA) * Math.sqrt(normB); + if (denominator === 0) return 0; + + return dotProduct / denominator; +} + +/** + * Find top K most similar items from a collection + * @param {number[]} queryEmbedding - The embedding to search for + * @param {Array<{id: any, embedding: number[]}>} items - Items with embeddings + * @param {number} topK - Number of results to return + * @returns {Array<{id: any, similarity: number}>} + */ +function findTopMatches(queryEmbedding, items, topK = 10) { + if (!queryEmbedding || !items || items.length === 0) { + return []; + } + + const scored = items.map(item => ({ + id: item.id, + similarity: cosineSimilarity(queryEmbedding, item.embedding) + })); + + scored.sort((a, b) => b.similarity - a.similarity); + + return scored.slice(0, topK); +} + +/** + * Find matches above a similarity threshold + * @param {number[]} queryEmbedding + * @param {Array<{id: any, embedding: number[]}>} items + * @param {number} threshold - Minimum similarity (0-1) + * @returns {Array<{id: any, similarity: number}>} + */ +function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) { + if (!queryEmbedding || !items || items.length === 0) { + return []; + } + + const scored = items + .map(item => ({ + id: item.id, + similarity: cosineSimilarity(queryEmbedding, item.embedding) + })) + .filter(item => item.similarity >= threshold); + + scored.sort((a, b) => b.similarity - a.similarity); + + return scored; +} + +module.exports = { + cosineSimilarity, + findTopMatches, + findMatchesAboveThreshold +}; diff --git a/inventory-server/src/services/ai/embeddings/taxonomyEmbeddings.js b/inventory-server/src/services/ai/embeddings/taxonomyEmbeddings.js new file mode 100644 index 0000000..bacf55a --- /dev/null +++ b/inventory-server/src/services/ai/embeddings/taxonomyEmbeddings.js @@ -0,0 +1,323 @@ +/** + * Taxonomy Embedding Service + * + * Generates and caches embeddings for categories, themes, and colors. + * Excludes "Black Friday", "Gifts", "Deals" categories and their children. + */ + +const { findTopMatches } = require('./similarity'); + +// Categories to exclude (and all their children) +const EXCLUDED_CATEGORY_NAMES = ['black friday', 'gifts', 'deals']; + +class TaxonomyEmbeddings { + constructor({ provider, logger }) { + this.provider = provider; + this.logger = logger || console; + + // Cached taxonomy with embeddings + this.categories = []; + this.themes = []; + this.colors = []; + + // Raw data without embeddings (for lookup) + this.categoryMap = new Map(); + this.themeMap = new Map(); + this.colorMap = new Map(); + + this.initialized = false; + this.initializing = false; + } + + /** + * Initialize embeddings - fetch taxonomy and generate embeddings + */ + async initialize(connection) { + if (this.initialized) { + return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length }; + } + + if (this.initializing) { + // Wait for existing initialization + while (this.initializing) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length }; + } + + this.initializing = true; + + try { + this.logger.info('[TaxonomyEmbeddings] Starting initialization...'); + + // Fetch raw taxonomy data + const [categories, themes, colors] = await Promise.all([ + this._fetchCategories(connection), + this._fetchThemes(connection), + this._fetchColors(connection) + ]); + + this.logger.info(`[TaxonomyEmbeddings] Fetched ${categories.length} categories, ${themes.length} themes, ${colors.length} colors`); + + // Generate embeddings in parallel + const [catEmbeddings, themeEmbeddings, colorEmbeddings] = await Promise.all([ + this._generateEmbeddings(categories, 'categories'), + this._generateEmbeddings(themes, 'themes'), + this._generateEmbeddings(colors, 'colors') + ]); + + // Store with embeddings + this.categories = catEmbeddings; + this.themes = themeEmbeddings; + this.colors = colorEmbeddings; + + // Build lookup maps + this.categoryMap = new Map(this.categories.map(c => [c.id, c])); + this.themeMap = new Map(this.themes.map(t => [t.id, t])); + this.colorMap = new Map(this.colors.map(c => [c.id, c])); + + this.initialized = true; + this.logger.info('[TaxonomyEmbeddings] Initialization complete'); + + return { + categories: this.categories.length, + themes: this.themes.length, + colors: this.colors.length + }; + } catch (error) { + this.logger.error('[TaxonomyEmbeddings] Initialization failed:', error); + throw error; + } finally { + this.initializing = false; + } + } + + /** + * Find similar categories for a product embedding + */ + findSimilarCategories(productEmbedding, topK = 10) { + if (!this.initialized || !productEmbedding) { + return []; + } + + const matches = findTopMatches(productEmbedding, this.categories, topK); + + return matches.map(match => { + const cat = this.categoryMap.get(match.id); + return { + id: match.id, + name: cat?.name || '', + fullPath: cat?.fullPath || '', + similarity: match.similarity + }; + }); + } + + /** + * Find similar themes for a product embedding + */ + findSimilarThemes(productEmbedding, topK = 5) { + if (!this.initialized || !productEmbedding) { + return []; + } + + const matches = findTopMatches(productEmbedding, this.themes, topK); + + return matches.map(match => { + const theme = this.themeMap.get(match.id); + return { + id: match.id, + name: theme?.name || '', + fullPath: theme?.fullPath || '', + similarity: match.similarity + }; + }); + } + + /** + * Find similar colors for a product embedding + */ + findSimilarColors(productEmbedding, topK = 5) { + if (!this.initialized || !productEmbedding) { + return []; + } + + const matches = findTopMatches(productEmbedding, this.colors, topK); + + return matches.map(match => { + const color = this.colorMap.get(match.id); + return { + id: match.id, + name: color?.name || '', + similarity: match.similarity + }; + }); + } + + /** + * Get all taxonomy data (without embeddings) for frontend + */ + getTaxonomyData() { + return { + categories: this.categories.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })), + themes: this.themes.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })), + colors: this.colors.map(({ id, name }) => ({ id, name })) + }; + } + + /** + * Check if service is ready + */ + isReady() { + return this.initialized; + } + + // ============================================================================ + // Private Methods + // ============================================================================ + + async _fetchCategories(connection) { + // Fetch hierarchical categories (types 10-13) + const [rows] = await connection.query(` + SELECT cat_id, name, master_cat_id, type + FROM product_categories + WHERE type IN (10, 11, 12, 13) + ORDER BY type, name + `); + + // Build lookup for hierarchy + const byId = new Map(rows.map(r => [r.cat_id, r])); + + // Find IDs of excluded top-level categories and all their descendants + const excludedIds = new Set(); + + // First pass: find excluded top-level categories + for (const row of rows) { + if (row.type === 10 && EXCLUDED_CATEGORY_NAMES.includes(row.name.toLowerCase())) { + excludedIds.add(row.cat_id); + } + } + + // Multiple passes to find all descendants + let foundNew = true; + while (foundNew) { + foundNew = false; + for (const row of rows) { + if (!excludedIds.has(row.cat_id) && excludedIds.has(row.master_cat_id)) { + excludedIds.add(row.cat_id); + foundNew = true; + } + } + } + + this.logger.info(`[TaxonomyEmbeddings] Excluding ${excludedIds.size} categories (Black Friday, Gifts, Deals and children)`); + + // Build category objects with full paths, excluding filtered ones + const categories = []; + + for (const row of rows) { + if (excludedIds.has(row.cat_id)) { + continue; + } + + const path = []; + let current = row; + + // Walk up the tree to build full path + while (current) { + path.unshift(current.name); + current = current.master_cat_id ? byId.get(current.master_cat_id) : null; + } + + categories.push({ + id: row.cat_id, + name: row.name, + parentId: row.master_cat_id, + type: row.type, + fullPath: path.join(' > '), + embeddingText: path.join(' ') + }); + } + + return categories; + } + + async _fetchThemes(connection) { + // Fetch themes (types 20-21) + const [rows] = await connection.query(` + SELECT cat_id, name, master_cat_id, type + FROM product_categories + WHERE type IN (20, 21) + ORDER BY type, name + `); + + const byId = new Map(rows.map(r => [r.cat_id, r])); + const themes = []; + + for (const row of rows) { + const path = []; + let current = row; + + while (current) { + path.unshift(current.name); + current = current.master_cat_id ? byId.get(current.master_cat_id) : null; + } + + themes.push({ + id: row.cat_id, + name: row.name, + parentId: row.master_cat_id, + type: row.type, + fullPath: path.join(' > '), + embeddingText: path.join(' ') + }); + } + + return themes; + } + + async _fetchColors(connection) { + const [rows] = await connection.query(` + SELECT color, name, hex_color + FROM product_color_list + ORDER BY \`order\` + `); + + return rows.map(row => ({ + id: row.color, + name: row.name, + hexColor: row.hex_color, + embeddingText: row.name + })); + } + + async _generateEmbeddings(items, label) { + if (items.length === 0) { + return items; + } + + const startTime = Date.now(); + const texts = items.map(item => item.embeddingText); + const results = [...items]; + + // Process in batches + let batchNum = 0; + for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) { + batchNum++; + for (let i = 0; i < chunk.embeddings.length; i++) { + const globalIndex = chunk.startIndex + i; + results[globalIndex] = { + ...results[globalIndex], + embedding: chunk.embeddings[i] + }; + } + } + + const elapsed = Date.now() - startTime; + this.logger.info(`[TaxonomyEmbeddings] Generated ${items.length} ${label} embeddings in ${elapsed}ms`); + + return results; + } +} + +module.exports = { TaxonomyEmbeddings }; diff --git a/inventory-server/src/services/ai/index.js b/inventory-server/src/services/ai/index.js new file mode 100644 index 0000000..628d599 --- /dev/null +++ b/inventory-server/src/services/ai/index.js @@ -0,0 +1,273 @@ +/** + * AI Service + * + * Main entry point for AI functionality including embeddings. + * Provides embedding generation and similarity search for product validation. + */ + +const { OpenAIProvider } = require('./providers/openaiProvider'); +const { TaxonomyEmbeddings } = require('./embeddings/taxonomyEmbeddings'); +const { cosineSimilarity, findTopMatches } = require('./embeddings/similarity'); + +let initialized = false; +let initializing = false; +let openaiProvider = null; +let taxonomyEmbeddings = null; +let logger = console; + +/** + * Initialize the AI service + * @param {Object} options + * @param {string} options.openaiApiKey - OpenAI API key + * @param {Object} options.mysqlConnection - MySQL connection for taxonomy data + * @param {Object} [options.logger] - Logger instance + */ +async function initialize({ openaiApiKey, mysqlConnection, logger: customLogger }) { + if (initialized) { + return { success: true, message: 'Already initialized' }; + } + + if (initializing) { + // Wait for existing initialization + while (initializing) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + return { success: initialized, message: initialized ? 'Initialized' : 'Initialization failed' }; + } + + initializing = true; + + try { + if (customLogger) { + logger = customLogger; + } + + if (!openaiApiKey) { + throw new Error('OpenAI API key is required'); + } + + logger.info('[AI] Initializing AI service...'); + + // Create OpenAI provider + openaiProvider = new OpenAIProvider({ apiKey: openaiApiKey }); + + // Create and initialize taxonomy embeddings + taxonomyEmbeddings = new TaxonomyEmbeddings({ + provider: openaiProvider, + logger + }); + + const stats = await taxonomyEmbeddings.initialize(mysqlConnection); + + initialized = true; + logger.info('[AI] AI service initialized', stats); + + return { + success: true, + message: 'Initialized', + stats + }; + } catch (error) { + logger.error('[AI] Initialization failed:', error); + return { success: false, message: error.message }; + } finally { + initializing = false; + } +} + +/** + * Check if service is ready + */ +function isReady() { + return initialized && taxonomyEmbeddings?.isReady(); +} + +/** + * Build weighted product text for embedding. + * Weights the product name heavily by repeating it, and truncates long descriptions + * to prevent verbose marketing copy from drowning out the product signal. + * + * @param {Object} product - Product with name, description, company, line + * @returns {string} - Combined text for embedding + */ +function buildProductText(product) { + const parts = []; + const name = product.name?.trim(); + const description = product.description?.trim(); + const company = (product.company_name || product.company)?.trim(); + const line = (product.line_name || product.line)?.trim(); + + // Name is most important - repeat 3x to weight it heavily in the embedding + if (name) { + parts.push(name, name, name); + } + + // Company and line provide context + if (company) { + parts.push(company); + } + if (line) { + parts.push(line); + } + + // Truncate description to prevent it from overwhelming the signal + if (description) { + const truncated = description.length > 500 + ? description.substring(0, 500) + '...' + : description; + parts.push(truncated); + } + + return parts.join(' ').trim(); +} + +/** + * Generate embedding for a product + * @param {Object} product - Product with name, description, company, line + * @returns {Promise<{embedding: number[], latencyMs: number}>} + */ +async function getProductEmbedding(product) { + if (!initialized || !openaiProvider) { + throw new Error('AI service not initialized'); + } + + const text = buildProductText(product); + + if (!text) { + return { embedding: null, latencyMs: 0 }; + } + + const result = await openaiProvider.embed(text); + + return { + embedding: result.embeddings[0], + latencyMs: result.latencyMs + }; +} + +/** + * Generate embeddings for multiple products + * @param {Object[]} products - Array of products + * @returns {Promise<{embeddings: Array<{index: number, embedding: number[]}>, latencyMs: number}>} + */ +async function getProductEmbeddings(products) { + if (!initialized || !openaiProvider) { + throw new Error('AI service not initialized'); + } + + const texts = products.map(buildProductText); + + // Track which products have empty text + const validIndices = texts.map((t, i) => t ? i : -1).filter(i => i >= 0); + const validTexts = texts.filter(t => t); + + if (validTexts.length === 0) { + return { embeddings: [], latencyMs: 0 }; + } + + const result = await openaiProvider.embed(validTexts); + + // Map embeddings back to original indices + const embeddings = validIndices.map((originalIndex, resultIndex) => ({ + index: originalIndex, + embedding: result.embeddings[resultIndex] + })); + + return { + embeddings, + latencyMs: result.latencyMs + }; +} + +/** + * Find similar taxonomy items for a product embedding + * @param {number[]} productEmbedding + * @param {Object} options + * @returns {{categories: Array, themes: Array, colors: Array}} + */ +function findSimilarTaxonomy(productEmbedding, options = {}) { + if (!initialized || !taxonomyEmbeddings) { + throw new Error('AI service not initialized'); + } + + const topCategories = options.topCategories ?? 10; + const topThemes = options.topThemes ?? 5; + const topColors = options.topColors ?? 5; + + return { + categories: taxonomyEmbeddings.findSimilarCategories(productEmbedding, topCategories), + themes: taxonomyEmbeddings.findSimilarThemes(productEmbedding, topThemes), + colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors) + }; +} + +/** + * Get product embedding and find similar taxonomy in one call + * @param {Object} product + * @param {Object} options + */ +async function getSuggestionsForProduct(product, options = {}) { + const { embedding, latencyMs: embeddingLatency } = await getProductEmbedding(product); + + if (!embedding) { + return { + categories: [], + themes: [], + colors: [], + latencyMs: embeddingLatency + }; + } + + const startSearch = Date.now(); + const suggestions = findSimilarTaxonomy(embedding, options); + const searchLatency = Date.now() - startSearch; + + return { + ...suggestions, + latencyMs: embeddingLatency + searchLatency, + embeddingLatencyMs: embeddingLatency, + searchLatencyMs: searchLatency + }; +} + +/** + * Get all taxonomy data (without embeddings) for frontend + */ +function getTaxonomyData() { + if (!initialized || !taxonomyEmbeddings) { + throw new Error('AI service not initialized'); + } + + return taxonomyEmbeddings.getTaxonomyData(); +} + +/** + * Get service status + */ +function getStatus() { + return { + initialized, + ready: isReady(), + hasProvider: !!openaiProvider, + hasTaxonomy: !!taxonomyEmbeddings, + taxonomyStats: taxonomyEmbeddings ? { + categories: taxonomyEmbeddings.categories?.length || 0, + themes: taxonomyEmbeddings.themes?.length || 0, + colors: taxonomyEmbeddings.colors?.length || 0 + } : null + }; +} + +module.exports = { + initialize, + isReady, + getProductEmbedding, + getProductEmbeddings, + findSimilarTaxonomy, + getSuggestionsForProduct, + getTaxonomyData, + getStatus, + // Re-export utilities + cosineSimilarity, + findTopMatches +}; diff --git a/inventory-server/src/services/ai/providers/openaiProvider.js b/inventory-server/src/services/ai/providers/openaiProvider.js new file mode 100644 index 0000000..c12902a --- /dev/null +++ b/inventory-server/src/services/ai/providers/openaiProvider.js @@ -0,0 +1,117 @@ +/** + * OpenAI Provider - Handles embedding generation + */ + +const EMBEDDING_MODEL = 'text-embedding-3-small'; +const EMBEDDING_DIMENSIONS = 1536; +const MAX_BATCH_SIZE = 2048; + +class OpenAIProvider { + constructor({ apiKey, baseUrl = 'https://api.openai.com/v1', timeoutMs = 60000 }) { + if (!apiKey) { + throw new Error('OpenAI API key is required'); + } + this.apiKey = apiKey; + this.baseUrl = baseUrl; + this.timeoutMs = timeoutMs; + } + + /** + * Generate embeddings for one or more texts + * @param {string|string[]} input - Text or array of texts + * @param {Object} options + * @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>} + */ + async embed(input, options = {}) { + const texts = Array.isArray(input) ? input : [input]; + const model = options.model || EMBEDDING_MODEL; + const dimensions = options.dimensions || EMBEDDING_DIMENSIONS; + const timeoutMs = options.timeoutMs || this.timeoutMs; + + if (texts.length > MAX_BATCH_SIZE) { + throw new Error(`Batch size ${texts.length} exceeds max of ${MAX_BATCH_SIZE}`); + } + + const started = Date.now(); + + // Clean and truncate input texts + const cleanedTexts = texts.map(t => + (t || '').replace(/\n+/g, ' ').trim().substring(0, 8000) + ); + + const body = { + input: cleanedTexts, + model, + encoding_format: 'float' + }; + + // Only embedding-3 models support dimensions parameter + if (model.includes('embedding-3')) { + body.dimensions = dimensions; + } + + const response = await this._makeRequest('embeddings', body, timeoutMs); + + // Sort by index to ensure order matches input + const sortedData = response.data.sort((a, b) => a.index - b.index); + + return { + embeddings: sortedData.map(item => item.embedding), + usage: { + promptTokens: response.usage?.prompt_tokens || 0, + totalTokens: response.usage?.total_tokens || 0 + }, + model: response.model || model, + latencyMs: Date.now() - started + }; + } + + /** + * Generator for processing large batches in chunks + */ + async *embedBatchChunked(texts, options = {}) { + const batchSize = Math.min(options.batchSize || 100, MAX_BATCH_SIZE); + + for (let i = 0; i < texts.length; i += batchSize) { + const chunk = texts.slice(i, i + batchSize); + const result = await this.embed(chunk, options); + + yield { + embeddings: result.embeddings, + startIndex: i, + endIndex: i + chunk.length, + usage: result.usage, + model: result.model, + latencyMs: result.latencyMs + }; + } + } + + async _makeRequest(endpoint, body, timeoutMs) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(`${this.baseUrl}/${endpoint}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${this.apiKey}` + }, + body: JSON.stringify(body), + signal: controller.signal + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + throw new Error(error.error?.message || `OpenAI API error: ${response.status}`); + } + + return response.json(); + } finally { + clearTimeout(timeout); + } + } +} + +module.exports = { OpenAIProvider, EMBEDDING_MODEL, EMBEDDING_DIMENSIONS }; diff --git a/inventory/src/components/product-import/config.ts b/inventory/src/components/product-import/config.ts index f76433c..21e668f 100644 --- a/inventory/src/components/product-import/config.ts +++ b/inventory/src/components/product-import/config.ts @@ -93,7 +93,7 @@ export const BASE_IMPORT_FIELDS = [ description: "Internal notions number", alternateMatches: ["notions #","nmc"], fieldType: { type: "input" }, - width: 100, + width: 110, validations: [ { rule: "required", errorMessage: "Required", level: "error" }, { rule: "unique", errorMessage: "Must be unique", level: "error" }, @@ -106,7 +106,7 @@ export const BASE_IMPORT_FIELDS = [ description: "Product name/title", alternateMatches: ["sku description","product name"], fieldType: { type: "input" }, - width: 500, + width: 400, validations: [ { rule: "required", errorMessage: "Required", level: "error" }, { rule: "unique", errorMessage: "Must be unique", level: "error" }, @@ -133,7 +133,7 @@ export const BASE_IMPORT_FIELDS = [ description: "Quantity of items per individual unit", alternateMatches: ["inner pack", "inner", "min qty", "unit qty", "min. order qty", "supplier qty/unit"], fieldType: { type: "input" }, - width: 80, + width: 100, validations: [ { rule: "required", errorMessage: "Required", level: "error" }, { rule: "regex", value: "^[0-9]+$", errorMessage: "Must be a number", level: "error" }, @@ -148,7 +148,7 @@ export const BASE_IMPORT_FIELDS = [ type: "input", price: true }, - width: 100, + width: 110, validations: [ { rule: "required", errorMessage: "Required", level: "error" }, { rule: "regex", value: "^[0-9]*.?[0-9]+$", errorMessage: "Must be a number", level: "error" }, @@ -312,7 +312,7 @@ export const BASE_IMPORT_FIELDS = [ type: "multi-select", options: [], // Will be populated from API }, - width: 350, + width: 400, validations: [{ rule: "required", errorMessage: "Required", level: "error" }], }, { diff --git a/inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx b/inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx index c8e0d8d..59dbe88 100644 --- a/inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx +++ b/inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx @@ -1917,18 +1917,11 @@ const MatchColumnsStepComponent = ({ )}
-
diff --git a/inventory/src/components/product-import/steps/UploadFlow.tsx b/inventory/src/components/product-import/steps/UploadFlow.tsx index 30cd251..59fd34b 100644 --- a/inventory/src/components/product-import/steps/UploadFlow.tsx +++ b/inventory/src/components/product-import/steps/UploadFlow.tsx @@ -4,7 +4,6 @@ import { UploadStep } from "./UploadStep/UploadStep" import { SelectHeaderStep } from "./SelectHeaderStep/SelectHeaderStep" import { SelectSheetStep } from "./SelectSheetStep/SelectSheetStep" import { mapWorkbook } from "../utils/mapWorkbook" -import { ValidationStepNew } from "./ValidationStepNew" import { ValidationStep } from "./ValidationStep" import { ImageUploadStep } from "./ImageUploadStep/ImageUploadStep" import { MatchColumnsStep } from "./MatchColumnsStep/MatchColumnsStep" @@ -220,36 +219,8 @@ export const UploadFlow = ({ state, onNext, onBack }: Props) => { /> ) case StepType.validateData: - // Always use the new ValidationStepNew component - return ( - { - // If we started from scratch, we need to go back to the upload step - if (state.isFromScratch) { - onNext({ - type: StepType.upload - }); - } else if (onBack) { - // Use the provided onBack function - onBack(); - } - }} - onNext={(validatedData: any[]) => { - // Go to image upload step with the validated data - onNext({ - type: StepType.imageUpload, - data: validatedData, - file: uploadedFile || new File([], "empty.xlsx"), - globalSelections: state.globalSelections - }); - }} - isFromScratch={state.isFromScratch} - /> - ) case StepType.validateDataNew: - // New Zustand-based ValidationStep component + // Zustand-based ValidationStep component (both cases now use this) return ( { { + // Go back to the validation step with the current data + onNext({ + type: StepType.validateDataNew, + data: state.data, + file: state.file, + globalSelections: state.globalSelections + }); + }} onSubmit={(data, file, options) => { // Create a Result object from the array data const result = { diff --git a/inventory/src/components/product-import/steps/ValidationStep/components/SuggestionBadges.tsx b/inventory/src/components/product-import/steps/ValidationStep/components/SuggestionBadges.tsx new file mode 100644 index 0000000..6a7ada2 --- /dev/null +++ b/inventory/src/components/product-import/steps/ValidationStep/components/SuggestionBadges.tsx @@ -0,0 +1,192 @@ +/** + * Suggestion Badges Component + * + * Displays AI-suggested options inline for categories, themes, and colors. + * Shows similarity scores and allows one-click selection. + */ + +import { Sparkles, Loader2, Plus, Check } from 'lucide-react'; +import { cn } from '@/lib/utils'; +import type { TaxonomySuggestion } from '../store/types'; + +interface SuggestionBadgesProps { + /** Suggestions to display */ + suggestions: TaxonomySuggestion[]; + /** Currently selected values (IDs) */ + selectedValues: (string | number)[]; + /** Callback when a suggestion is clicked */ + onSelect: (id: number) => void; + /** Whether suggestions are loading */ + isLoading?: boolean; + /** Maximum suggestions to show */ + maxSuggestions?: number; + /** Minimum similarity to show (0-1) */ + minSimilarity?: number; + /** Label for the section */ + label?: string; + /** Compact mode for smaller displays */ + compact?: boolean; + /** Show similarity scores */ + showScores?: boolean; + /** Custom class name */ + className?: string; +} + +export function SuggestionBadges({ + suggestions, + selectedValues, + onSelect, + isLoading = false, + maxSuggestions = 5, + minSimilarity = 0, + label = 'Suggested', + compact = false, + showScores = true, + className, +}: SuggestionBadgesProps) { + // Filter and limit suggestions + const filteredSuggestions = suggestions + .filter(s => s.similarity >= minSimilarity) + .slice(0, maxSuggestions); + + // Don't render if no suggestions and not loading + if (!isLoading && filteredSuggestions.length === 0) { + return null; + } + + const isSelected = (id: number) => { + return selectedValues.some(v => String(v) === String(id)); + }; + + return ( +
+ {/* Label */} +
+ + {!compact && {label}:} +
+ + {/* Loading state */} + {isLoading && ( +
+ + {!compact && Loading...} +
+ )} + + {/* Suggestion badges */} + {filteredSuggestions.map((suggestion) => { + const selected = isSelected(suggestion.id); + const similarityPercent = Math.round(suggestion.similarity * 100); + + return ( + + ); + })} +
+ ); +} + +/** + * Inline suggestion for a single field (used inside dropdowns) + */ +interface InlineSuggestionProps { + suggestion: TaxonomySuggestion; + isSelected: boolean; + onSelect: () => void; + showScore?: boolean; +} + +export function InlineSuggestion({ + suggestion, + isSelected, + onSelect, + showScore = true, +}: InlineSuggestionProps) { + const similarityPercent = Math.round(suggestion.similarity * 100); + + return ( +
+
+ + + {suggestion.fullPath || suggestion.name} + +
+
+ {showScore && ( + + {similarityPercent}% + + )} + {isSelected ? ( + + ) : ( + + )} +
+
+ ); +} + +/** + * Suggestion section header for dropdowns + */ +interface SuggestionSectionHeaderProps { + isLoading?: boolean; + count?: number; +} + +export function SuggestionSectionHeader({ isLoading, count }: SuggestionSectionHeaderProps) { + return ( +
+ + AI Suggested + {isLoading && } + {!isLoading && count !== undefined && ( + ({count}) + )} +
+ ); +} + +export default SuggestionBadges; diff --git a/inventory/src/components/product-import/steps/ValidationStep/components/ValidationContainer.tsx b/inventory/src/components/product-import/steps/ValidationStep/components/ValidationContainer.tsx index 76ca3ae..ce48c01 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/components/ValidationContainer.tsx +++ b/inventory/src/components/product-import/steps/ValidationStep/components/ValidationContainer.tsx @@ -6,7 +6,7 @@ * Note: Initialization effects are in index.tsx so they run before this mounts. */ -import { useCallback, useMemo } from 'react'; +import { useCallback, useMemo, useRef } from 'react'; import { useValidationStore } from '../store/validationStore'; import { useTotalErrorCount, @@ -21,11 +21,13 @@ import { FloatingSelectionBar } from './FloatingSelectionBar'; import { useAiValidationFlow } from '../hooks/useAiValidation'; import { useFieldOptions } from '../hooks/useFieldOptions'; import { useTemplateManagement } from '../hooks/useTemplateManagement'; +import { useCopyDownValidation } from '../hooks/useCopyDownValidation'; import { AiValidationProgressDialog } from '../dialogs/AiValidationProgress'; import { AiValidationResultsDialog } from '../dialogs/AiValidationResults'; import { AiDebugDialog } from '../dialogs/AiDebugDialog'; import { TemplateForm } from '@/components/templates/TemplateForm'; -import type { CleanRowData } from '../store/types'; +import { AiSuggestionsProvider } from '../contexts/AiSuggestionsContext'; +import type { CleanRowData, RowData } from '../store/types'; interface ValidationContainerProps { onBack?: () => void; @@ -57,6 +59,32 @@ export const ValidationContainer = ({ const { data: fieldOptionsData } = useFieldOptions(); const { loadTemplates } = useTemplateManagement(); + // Handle UPC validation after copy-down operations on supplier/upc fields + useCopyDownValidation(); + + // Get initial products for AI suggestions (read once via ref to avoid re-fetching) + const initialProductsRef = useRef(null); + if (initialProductsRef.current === null) { + initialProductsRef.current = useValidationStore.getState().rows; + } + + // Create stable lookup functions for company/line names + const getCompanyName = useCallback((id: string): string | undefined => { + const companies = fieldOptionsData?.companies || []; + const company = companies.find(c => c.value === id); + return company?.label; + }, [fieldOptionsData?.companies]); + + const getLineName = useCallback((id: string): string | undefined => { + // Lines are fetched dynamically per company, check the cache + const cache = useValidationStore.getState().productLinesCache; + for (const lines of cache.values()) { + const line = lines.find(l => l.value === id); + if (line) return line.label; + } + return undefined; + }, []); + // Convert field options to TemplateForm format const templateFormFieldOptions = useMemo(() => { if (!fieldOptionsData) return null; @@ -94,69 +122,76 @@ export const ValidationContainer = ({ }, [onBack]); return ( -
- {/* Toolbar */} - + +
+ {/* Toolbar */} + - {/* Main table area */} -
- + {/* Main table area */} +
+ +
+ + {/* Footer with navigation */} + + + {/* Floating selection bar - appears when rows selected */} + + + {/* AI Validation dialogs */} + {aiValidation.isValidating && aiValidation.progress && ( + + )} + + {aiValidation.results && !aiValidation.isValidating && ( + + )} + + {/* AI Debug Dialog - for viewing prompt */} + + + {/* Template form dialog - for saving row as template */} + [0]['initialData']} + mode="create" + fieldOptions={templateFormFieldOptions} + />
- - {/* Footer with navigation */} - - - {/* Floating selection bar - appears when rows selected */} - - - {/* AI Validation dialogs */} - {aiValidation.isValidating && aiValidation.progress && ( - - )} - - {aiValidation.results && !aiValidation.isValidating && ( - - )} - - {/* AI Debug Dialog - for viewing prompt */} - - - {/* Template form dialog - for saving row as template */} - [0]['initialData']} - mode="create" - fieldOptions={templateFormFieldOptions} - /> -
+ ); }; diff --git a/inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx b/inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx index 508bc98..337d24a 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx +++ b/inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx @@ -47,6 +47,12 @@ import { ComboboxCell } from './cells/ComboboxCell'; import { MultiSelectCell } from './cells/MultiSelectCell'; import { MultilineInput } from './cells/MultilineInput'; +// AI Suggestions context +import { useAiSuggestionsContext } from '../contexts/AiSuggestionsContext'; + +// Fields that trigger AI suggestion refresh when changed +const AI_EMBEDDING_FIELDS = ['company', 'line', 'name', 'description'] as const; + // Threshold for switching to ComboboxCell (with search) instead of SelectCell const COMBOBOX_OPTION_THRESHOLD = 50; @@ -96,6 +102,8 @@ const EMPTY_ROW_ERRORS: Record = {}; interface CellWrapperProps { field: Field; rowIndex: number; + /** Product's unique __index for AI suggestions */ + productIndex: string; value: unknown; errors: ValidationError[]; isValidating: boolean; @@ -124,6 +132,7 @@ interface CellWrapperProps { const CellWrapper = memo(({ field, rowIndex, + productIndex, value, errors, isValidating, @@ -142,6 +151,11 @@ const CellWrapper = memo(({ const needsCompany = field.key === 'line'; const needsLine = field.key === 'subline'; + // AI suggestions context - for notifying when embedding fields change + // This uses stable callbacks so it won't cause re-renders + const aiSuggestions = useAiSuggestionsContext(); + const isEmbeddingField = AI_EMBEDDING_FIELDS.includes(field.key as typeof AI_EMBEDDING_FIELDS[number]); + // Check if cell has a value (for showing copy-down button) const hasValue = value !== undefined && value !== null && value !== ''; @@ -390,8 +404,17 @@ const CellWrapper = memo(({ } } } + + // Notify AI suggestions system when embedding fields change + // This triggers a refresh of category/theme/color suggestions + if (isEmbeddingField && aiSuggestions) { + const currentRow = useValidationStore.getState().rows[rowIndex]; + if (currentRow) { + aiSuggestions.handleFieldBlur(currentRow, field.key); + } + } }, 0); - }, [rowIndex, field.key]); + }, [rowIndex, field.key, isEmbeddingField, aiSuggestions]); // Stable callback for fetching options (for line/subline dropdowns) const handleFetchOptions = useCallback(async () => { @@ -481,6 +504,7 @@ const CellWrapper = memo(({ field={field} options={options} rowIndex={rowIndex} + productIndex={productIndex} isValidating={isValidating} errors={errors} onChange={handleChange} @@ -553,6 +577,7 @@ CellWrapper.displayName = 'CellWrapper'; * Template column width */ const TEMPLATE_COLUMN_WIDTH = 200; +const NAME_COLUMN_STICKY_LEFT = 0; /** * TemplateCell Component @@ -777,9 +802,12 @@ const VirtualRow = memo(({ useCallback((state) => state.errors.get(rowIndex) ?? EMPTY_ROW_ERRORS, [rowIndex]) ); - // DON'T subscribe to validatingCells - check it during render instead + // DON'T subscribe to validatingCells for most fields - check it during render instead // This avoids creating new objects in selectors which causes infinite loops - // Validation status changes are rare, so reading via getState() is fine + // EXCEPTION: Subscribe specifically for item_number so it shows loading state during UPC validation + const isItemNumberValidating = useValidationStore( + useCallback((state) => state.validatingCells.has(`${rowIndex}-item_number`), [rowIndex]) + ); // Subscribe to selection status const isSelected = useValidationStore( @@ -860,7 +888,10 @@ const VirtualRow = memo(({ const columnWidth = columns[fieldIndex + 2]?.size || field.width || 150; const fieldErrors = rowErrors[field.key] || EMPTY_ERRORS; // Check validating status via getState() - not subscribed to avoid object creation - const isValidating = useValidationStore.getState().validatingCells.has(`${rowIndex}-${field.key}`); + // EXCEPTION: item_number uses the subscribed isItemNumberValidating for reactive loading state + const isValidating = field.key === 'item_number' + ? isItemNumberValidating + : useValidationStore.getState().validatingCells.has(`${rowIndex}-${field.key}`); // CRITICAL: Only pass company/line to cells that need them! // Passing to all cells breaks memoization - when company changes, ALL cells re-render @@ -889,20 +920,27 @@ const VirtualRow = memo(({ copyDownMode.targetRowIndex !== null && rowIndex <= copyDownMode.targetRowIndex; + const isNameColumn = field.key === 'name'; + return (
{ className="flex h-full" style={{ minWidth: totalTableWidth }} > - {columns.map((column, index) => ( -
- {typeof column.header === 'function' - ? column.header({} as any) - : column.header} -
- ))} + {columns.map((column, index) => { + const isNameColumn = column.id === 'name'; + return ( +
+ {typeof column.header === 'function' + ? column.header({} as any) + : column.header} +
+ ); + })}
diff --git a/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultiSelectCell.tsx b/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultiSelectCell.tsx index 5e1a86e..6701f47 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultiSelectCell.tsx +++ b/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultiSelectCell.tsx @@ -6,10 +6,14 @@ * * PERFORMANCE: Uses uncontrolled open state for Popover. * Controlled open state can cause delays due to React state processing. + * + * AI SUGGESTIONS: For categories, themes, and colors fields, this component + * displays AI-powered suggestions based on product embeddings. Suggestions + * appear at the top of the dropdown with similarity scores. */ import { useCallback, useMemo, memo, useState } from 'react'; -import { Check, ChevronsUpDown, AlertCircle } from 'lucide-react'; +import { Check, ChevronsUpDown, AlertCircle, Sparkles, Loader2 } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Command, @@ -34,8 +38,9 @@ import { Badge } from '@/components/ui/badge'; import { Skeleton } from '@/components/ui/skeleton'; import { cn } from '@/lib/utils'; import type { Field, SelectOption } from '../../../../types'; -import type { ValidationError } from '../../store/types'; +import type { ValidationError, TaxonomySuggestion } from '../../store/types'; import { ErrorType } from '../../store/types'; +import { useCellSuggestions } from '../../contexts/AiSuggestionsContext'; // Extended option type to include hex color values interface MultiSelectOption extends SelectOption { @@ -49,6 +54,8 @@ interface MultiSelectCellProps { field: Field; options?: SelectOption[]; rowIndex: number; + /** Product's unique __index for AI suggestions */ + productIndex?: string; isValidating: boolean; errors: ValidationError[]; onChange: (value: unknown) => void; @@ -56,6 +63,10 @@ interface MultiSelectCellProps { onFetchOptions?: () => void; } +// Fields that support AI suggestions +const SUGGESTION_FIELDS = ['categories', 'themes', 'colors'] as const; +type SuggestionField = typeof SUGGESTION_FIELDS[number]; + /** * Helper to extract hex color from option * Supports hex, hexColor, and hex_color field names @@ -79,6 +90,7 @@ const MultiSelectCellComponent = ({ value, field, options = [], + productIndex, isValidating, errors, onChange: _onChange, // Unused - onBlur handles both update and validation @@ -86,6 +98,21 @@ const MultiSelectCellComponent = ({ }: MultiSelectCellProps) => { const [open, setOpen] = useState(false); + // Get AI suggestions for categories, themes, and colors + const supportsSuggestions = SUGGESTION_FIELDS.includes(field.key as SuggestionField); + const suggestions = useCellSuggestions(productIndex || ''); + + // Get the right suggestions based on field type + const fieldSuggestions: TaxonomySuggestion[] = useMemo(() => { + if (!supportsSuggestions || !productIndex) return []; + switch (field.key) { + case 'categories': return suggestions.categories; + case 'themes': return suggestions.themes; + case 'colors': return suggestions.colors; + default: return []; + } + }, [supportsSuggestions, productIndex, field.key, suggestions]); + // Handle wheel scroll in dropdown - stop propagation to prevent table scroll const handleWheel = useCallback((e: React.WheelEvent) => { e.stopPropagation(); @@ -216,48 +243,132 @@ const MultiSelectCellComponent = ({ - + No options found.
- - {options.map((option) => { - const hexColor = field.key === 'colors' ? getOptionHex(option as MultiSelectOption) : undefined; - const isWhite = hexColor ? isWhiteColor(hexColor) : false; + {/* Selected items section - floats to top of dropdown */} + {selectedValues.length > 0 && ( + +
+ + Selected ({selectedValues.length}) +
+ {selectedValues.map((selectedVal) => { + const option = options.find((opt) => opt.value === selectedVal) as MultiSelectOption | undefined; + const hexColor = field.key === 'colors' && option ? getOptionHex(option) : undefined; + const isWhite = hexColor ? isWhiteColor(hexColor) : false; + const label = option?.label || selectedVal; - return ( - handleSelect(option.value)} - > - handleSelect(selectedVal)} + className="bg-green-50/50 dark:bg-green-950/30" + > + + {field.key === 'colors' && hexColor && ( + )} - /> - {/* Color circle for colors field */} - {field.key === 'colors' && hexColor && ( - + ); + })} + + )} + + {/* AI Suggestions section - shown below selected items */} + {supportsSuggestions && (fieldSuggestions.length > 0 || suggestions.isLoading) && ( + +
+ + Suggested + {suggestions.isLoading && } +
+ {fieldSuggestions.slice(0, 5).map((suggestion) => { + const isSelected = selectedValues.includes(String(suggestion.id)); + // Skip suggestions that are already in the Selected section + if (isSelected) return null; + const similarityPercent = Math.round(suggestion.similarity * 100); + const hexColor = field.key === 'colors' + ? options.find(o => o.value === String(suggestion.id)) as MultiSelectOption | undefined + : undefined; + const suggestionHex = hexColor ? getOptionHex(hexColor) : undefined; + + return ( + handleSelect(String(suggestion.id))} + className="bg-purple-50/30 dark:bg-purple-950/20" + > +
+ + {/* Color circle for colors */} + {field.key === 'colors' && suggestionHex && ( + )} - style={{ backgroundColor: hexColor }} - /> - )} - {option.label} - - ); - })} + {/* Show full path for categories/themes, just name for colors */} + + {field.key === 'colors' ? suggestion.name : (suggestion.fullPath || suggestion.name)} + +
+ + {similarityPercent}% + + + ); + })} + + )} + + {/* Regular options - excludes selected items (shown in Selected section above) */} + 0 || (supportsSuggestions && fieldSuggestions.length > 0) ? "All Options" : undefined}> + {options + .filter((option) => !selectedValues.includes(option.value)) + .map((option) => { + const hexColor = field.key === 'colors' ? getOptionHex(option as MultiSelectOption) : undefined; + const isWhite = hexColor ? isWhiteColor(hexColor) : false; + + return ( + handleSelect(option.value)} + > + + {/* Color circle for colors field */} + {field.key === 'colors' && hexColor && ( + + )} + {option.label} + + ); + })}
diff --git a/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultilineInput.tsx b/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultilineInput.tsx index 7584b64..5950c6b 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultilineInput.tsx +++ b/inventory/src/components/product-import/steps/ValidationStep/components/cells/MultilineInput.tsx @@ -9,6 +9,12 @@ import { useState, useCallback, useRef, useEffect, memo } from 'react'; import { Textarea } from '@/components/ui/textarea'; import { cn } from '@/lib/utils'; import { Popover, PopoverTrigger, PopoverContent } from '@/components/ui/popover'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; import { X, Loader2 } from 'lucide-react'; import { Button } from '@/components/ui/button'; import type { Field, SelectOption } from '../../../../types'; @@ -114,24 +120,42 @@ const MultilineInputComponent = ({ // Calculate display value const displayValue = localDisplayValue !== null ? localDisplayValue : String(value ?? ''); + // Tooltip content - show full description or error message + const tooltipContent = errorMessage || displayValue; + const showTooltip = tooltipContent && tooltipContent.length > 30; + return (
- -
+ + + +
+ {displayValue} +
+ + + {showTooltip && !popoverOpen && ( + +

{tooltipContent}

+ )} - title={errorMessage || displayValue} - > - {displayValue} -
- + + ProductSuggestions | undefined; + /** Check if suggestions are loading for a product */ + isLoading: (productIndex: string) => boolean; + /** Trigger suggestion fetch for a product */ + fetchSuggestions: (product: RowData) => void; + /** Handle field blur - refreshes suggestions if relevant field changed */ + handleFieldBlur: (product: RowData, fieldKey: string) => void; + /** Get category suggestions for a product */ + getCategorySuggestions: (productIndex: string) => TaxonomySuggestion[]; + /** Get theme suggestions for a product */ + getThemeSuggestions: (productIndex: string) => TaxonomySuggestion[]; + /** Get color suggestions for a product */ + getColorSuggestions: (productIndex: string) => TaxonomySuggestion[]; + /** Subscribe to suggestion changes for a product (returns unsubscribe fn) */ + subscribe: (productIndex: string, callback: () => void) => () => void; + /** Force refresh suggestions for all products */ + refreshAll: () => void; +} + +interface AiSuggestionsProviderProps { + children: React.ReactNode; + /** Get company name by ID */ + getCompanyName?: (id: string) => string | undefined; + /** Get line name by ID */ + getLineName?: (id: string) => string | undefined; + /** Initial products to fetch suggestions for */ + initialProducts?: RowData[]; + /** Whether to auto-initialize (default: true) */ + autoInitialize?: boolean; +} + +// ============================================================================ +// Context +// ============================================================================ + +const AiSuggestionsContext = createContext(null); + +// Fields that affect embeddings +const EMBEDDING_FIELDS = ['company', 'line', 'name', 'description']; + +const API_BASE = '/api/ai'; + +// ============================================================================ +// Provider +// ============================================================================ + +export function AiSuggestionsProvider({ + children, + getCompanyName, + getLineName, + initialProducts, + autoInitialize = true, +}: AiSuggestionsProviderProps) { + // State for initialization status (this can cause re-render, but it's rare) + const [isInitialized, setIsInitialized] = useState(false); + + // Refs for data that shouldn't trigger re-renders + const suggestionsRef = useRef>(new Map()); + const loadingRef = useRef>(new Set()); + const fieldValuesRef = useRef>>(new Map()); + const subscribersRef = useRef void>>>(new Map()); + + // Ref for lookup functions (updated on each render) + const lookupFnsRef = useRef({ getCompanyName, getLineName }); + lookupFnsRef.current = { getCompanyName, getLineName }; + + /** + * Notify subscribers when suggestions change + */ + const notifySubscribers = useCallback((productIndex: string) => { + const callbacks = subscribersRef.current.get(productIndex); + if (callbacks) { + callbacks.forEach(cb => cb()); + } + }, []); + + /** + * Initialize the AI service + */ + const initialize = useCallback(async (): Promise => { + if (isInitialized) return true; + + try { + const response = await fetch(`${API_BASE}/initialize`, { method: 'POST' }); + const data = await response.json(); + + if (!response.ok || !data.success) { + console.error('[AiSuggestions] Initialization failed:', data.error); + return false; + } + + setIsInitialized(true); + return true; + } catch (error) { + console.error('[AiSuggestions] Initialization error:', error); + return false; + } + }, [isInitialized]); + + /** + * Build product data for API request + */ + const buildProductRequest = useCallback((product: RowData) => { + const { getCompanyName: getCompany, getLineName: getLine } = lookupFnsRef.current; + return { + name: product.name, + description: product.description, + company_name: product.company ? getCompany?.(String(product.company)) : undefined, + line_name: product.line ? getLine?.(String(product.line)) : undefined, + }; + }, []); + + /** + * Fetch suggestions for a single product + */ + const fetchSuggestions = useCallback(async (product: RowData) => { + const productIndex = product.__index; + if (!productIndex) return; + + // Skip if already loading + if (loadingRef.current.has(productIndex)) return; + + // Ensure initialized + const ready = await initialize(); + if (!ready) return; + + // Check if product has enough data for meaningful suggestions + const productData = buildProductRequest(product); + const hasText = productData.name || productData.description || productData.company_name; + if (!hasText) return; + + // Mark as loading + loadingRef.current.add(productIndex); + notifySubscribers(productIndex); + + try { + const response = await fetch(`${API_BASE}/suggestions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ product: productData }), + }); + + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + + const suggestions: ProductSuggestions = await response.json(); + + // Store suggestions + suggestionsRef.current.set(productIndex, suggestions); + + // Store field values for change detection + fieldValuesRef.current.set(productIndex, { + company: product.company, + line: product.line, + name: product.name, + description: product.description, + }); + } catch (error) { + console.error('[AiSuggestions] Fetch error:', error); + } finally { + loadingRef.current.delete(productIndex); + notifySubscribers(productIndex); + } + }, [initialize, buildProductRequest, notifySubscribers]); + + /** + * Fetch suggestions for multiple products in batch + */ + const fetchBatchSuggestions = useCallback(async (products: RowData[]) => { + // Ensure initialized + const ready = await initialize(); + if (!ready) return; + + // Filter to products that need fetching + const productsToFetch = products.filter(p => { + if (!p.__index) return false; + if (loadingRef.current.has(p.__index)) return false; + if (suggestionsRef.current.has(p.__index)) return false; + + const productData = buildProductRequest(p); + return productData.name || productData.description || productData.company_name; + }); + + if (productsToFetch.length === 0) return; + + // Mark all as loading + productsToFetch.forEach(p => { + loadingRef.current.add(p.__index); + notifySubscribers(p.__index); + }); + + try { + const response = await fetch(`${API_BASE}/suggestions/batch`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + products: productsToFetch.map(p => ({ + _index: p.__index, + ...buildProductRequest(p), + })), + }), + }); + + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + + const data = await response.json(); + + // Store results + for (const result of data.results || []) { + const product = productsToFetch[result.index]; + if (product?.__index) { + suggestionsRef.current.set(product.__index, { + categories: result.categories, + themes: result.themes, + colors: result.colors, + }); + + fieldValuesRef.current.set(product.__index, { + company: product.company, + line: product.line, + name: product.name, + description: product.description, + }); + } + } + } catch (error) { + console.error('[AiSuggestions] Batch fetch error:', error); + } finally { + productsToFetch.forEach(p => { + loadingRef.current.delete(p.__index); + notifySubscribers(p.__index); + }); + } + }, [initialize, buildProductRequest, notifySubscribers]); + + /** + * Handle field blur - refresh suggestions if embedding field changed + */ + const handleFieldBlur = useCallback((product: RowData, fieldKey: string) => { + if (!EMBEDDING_FIELDS.includes(fieldKey)) return; + + const productIndex = product.__index; + if (!productIndex) return; + + // Check if value actually changed + const prevValues = fieldValuesRef.current.get(productIndex); + if (prevValues) { + const prevValue = prevValues[fieldKey]; + const currentValue = product[fieldKey]; + if (prevValue === currentValue) return; + } + + // Clear existing suggestions and refetch + suggestionsRef.current.delete(productIndex); + fieldValuesRef.current.delete(productIndex); + + // Debounce the fetch (simple timeout-based debounce) + setTimeout(() => fetchSuggestions(product), 300); + }, [fetchSuggestions]); + + /** + * Get suggestions for a product + */ + const getSuggestions = useCallback((productIndex: string): ProductSuggestions | undefined => { + return suggestionsRef.current.get(productIndex); + }, []); + + /** + * Check if loading + */ + const isLoading = useCallback((productIndex: string): boolean => { + return loadingRef.current.has(productIndex); + }, []); + + /** + * Get category suggestions + */ + const getCategorySuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => { + return suggestionsRef.current.get(productIndex)?.categories || []; + }, []); + + /** + * Get theme suggestions + */ + const getThemeSuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => { + return suggestionsRef.current.get(productIndex)?.themes || []; + }, []); + + /** + * Get color suggestions + */ + const getColorSuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => { + return suggestionsRef.current.get(productIndex)?.colors || []; + }, []); + + /** + * Subscribe to suggestion changes + */ + const subscribe = useCallback((productIndex: string, callback: () => void): (() => void) => { + if (!subscribersRef.current.has(productIndex)) { + subscribersRef.current.set(productIndex, new Set()); + } + subscribersRef.current.get(productIndex)!.add(callback); + + return () => { + subscribersRef.current.get(productIndex)?.delete(callback); + }; + }, []); + + /** + * Refresh all products + */ + const refreshAll = useCallback(() => { + // Clear all cached suggestions + suggestionsRef.current.clear(); + fieldValuesRef.current.clear(); + + // If we have initial products, refetch them + if (initialProducts && initialProducts.length > 0) { + fetchBatchSuggestions(initialProducts); + } + }, [initialProducts, fetchBatchSuggestions]); + + // Auto-initialize and fetch initial products + useEffect(() => { + if (!autoInitialize) return; + + const init = async () => { + const ready = await initialize(); + if (ready && initialProducts && initialProducts.length > 0) { + // Small delay to avoid blocking initial render + setTimeout(() => fetchBatchSuggestions(initialProducts), 100); + } + }; + + init(); + }, [autoInitialize, initialize, initialProducts, fetchBatchSuggestions]); + + const contextValue: AiSuggestionsContextValue = { + isInitialized, + getSuggestions, + isLoading, + fetchSuggestions, + handleFieldBlur, + getCategorySuggestions, + getThemeSuggestions, + getColorSuggestions, + subscribe, + refreshAll, + }; + + return ( + + {children} + + ); +} + +// ============================================================================ +// Hook +// ============================================================================ + +export function useAiSuggestionsContext(): AiSuggestionsContextValue | null { + return useContext(AiSuggestionsContext); +} + +/** + * Hook for cells to get suggestions with re-render on update + * Only use this in cell components that need to display suggestions + */ +export function useCellSuggestions(productIndex: string) { + const context = useAiSuggestionsContext(); + const [, forceUpdate] = useState({}); + + useEffect(() => { + if (!context) return; + + // Subscribe to changes for this product + const unsubscribe = context.subscribe(productIndex, () => { + forceUpdate({}); + }); + + return unsubscribe; + }, [context, productIndex]); + + if (!context) { + return { + categories: [] as TaxonomySuggestion[], + themes: [] as TaxonomySuggestion[], + colors: [] as TaxonomySuggestion[], + isLoading: false, + }; + } + + return { + categories: context.getCategorySuggestions(productIndex), + themes: context.getThemeSuggestions(productIndex), + colors: context.getColorSuggestions(productIndex), + isLoading: context.isLoading(productIndex), + }; +} + +export default AiSuggestionsContext; diff --git a/inventory/src/components/product-import/steps/ValidationStep/hooks/useCopyDownValidation.ts b/inventory/src/components/product-import/steps/ValidationStep/hooks/useCopyDownValidation.ts new file mode 100644 index 0000000..b9e3332 --- /dev/null +++ b/inventory/src/components/product-import/steps/ValidationStep/hooks/useCopyDownValidation.ts @@ -0,0 +1,54 @@ +/** + * useCopyDownValidation Hook + * + * Watches for copy-down operations on UPC-related fields (supplier, upc, barcode) + * and triggers UPC validation for affected rows using the existing validateUpc function. + * + * This avoids duplicating UPC validation logic - we reuse the same code path + * that handles individual cell blur events. + */ + +import { useEffect } from 'react'; +import { useValidationStore } from '../store/validationStore'; +import { useUpcValidation } from './useUpcValidation'; + +/** + * Hook that handles UPC validation after copy-down operations. + * Should be called once in ValidationContainer to ensure validation runs. + */ +export const useCopyDownValidation = () => { + const { validateUpc } = useUpcValidation(); + + // Subscribe to pending copy-down validation + const pendingValidation = useValidationStore((state) => state.pendingCopyDownValidation); + const clearPendingCopyDownValidation = useValidationStore((state) => state.clearPendingCopyDownValidation); + + useEffect(() => { + if (!pendingValidation) return; + + const { fieldKey, affectedRows } = pendingValidation; + + // Get current rows to check supplier and UPC values + const rows = useValidationStore.getState().rows; + + // Process each affected row + const validationPromises = affectedRows.map(async (rowIndex) => { + const row = rows[rowIndex]; + if (!row) return; + + // Get supplier and UPC values + const supplierId = row.supplier ? String(row.supplier) : ''; + const upcValue = row.upc ? String(row.upc) : (row.barcode ? String(row.barcode) : ''); + + // Only validate if we have both supplier and UPC + if (supplierId && upcValue) { + await validateUpc(rowIndex, supplierId, upcValue); + } + }); + + // Run all validations and then clear the pending state + Promise.all(validationPromises).then(() => { + clearPendingCopyDownValidation(); + }); + }, [pendingValidation, validateUpc, clearPendingCopyDownValidation]); +}; diff --git a/inventory/src/components/product-import/steps/ValidationStep/store/types.ts b/inventory/src/components/product-import/steps/ValidationStep/store/types.ts index 7f41b34..6187b9f 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/store/types.ts +++ b/inventory/src/components/product-import/steps/ValidationStep/store/types.ts @@ -151,6 +151,15 @@ export interface CopyDownState { targetRowIndex: number | null; // Hover preview - which row the user is hovering on } +/** + * Tracks rows that need UPC validation after copy-down completes. + * This allows reusing the existing validateUpc logic instead of duplicating it. + */ +export interface PendingCopyDownValidation { + fieldKey: string; + affectedRows: number[]; +} + // ============================================================================= // Dialog State Types // ============================================================================= @@ -218,6 +227,37 @@ export interface AiValidationState { revertedChanges: Set; // Format: "productIndex:fieldKey" } +// ============================================================================= +// AI Suggestions Types (Embedding-based) +// ============================================================================= + +export interface TaxonomySuggestion { + id: number; + name: string; + fullPath?: string; + similarity: number; +} + +export interface ProductSuggestions { + categories: TaxonomySuggestion[]; + themes: TaxonomySuggestion[]; + colors: TaxonomySuggestion[]; + latencyMs?: number; +} + +export interface AiSuggestionsState { + initialized: boolean; + initializing: boolean; + /** Map of product __index to their embedding */ + embeddings: Map; + /** Map of product __index to their suggestions */ + suggestions: Map; + /** Products currently being processed */ + processing: Set; + /** Last error if any */ + error: string | null; +} + // ============================================================================= // Initialization Types // ============================================================================= @@ -292,6 +332,7 @@ export interface ValidationState { // === Copy-Down Mode === copyDownMode: CopyDownState; + pendingCopyDownValidation: PendingCopyDownValidation | null; // === Dialogs === dialogs: DialogState; @@ -376,6 +417,7 @@ export interface ValidationActions { cancelCopyDown: () => void; completeCopyDown: (targetRowIndex: number) => void; setTargetRowHover: (rowIndex: number | null) => void; + clearPendingCopyDownValidation: () => void; // === Dialogs === setDialogs: (updates: Partial) => void; diff --git a/inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts b/inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts index 10c67d6..a8336c3 100644 --- a/inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts +++ b/inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts @@ -29,6 +29,7 @@ import type { AiValidationResults, CopyDownState, DialogState, + PendingCopyDownValidation, } from './types'; import type { Field, SelectOption } from '../../../types'; @@ -57,6 +58,9 @@ const initialCopyDownState: CopyDownState = { targetRowIndex: null, }; +// Fields that require UPC validation when changed via copy-down +const UPC_VALIDATION_FIELDS = ['supplier', 'upc', 'barcode']; + const initialDialogState: DialogState = { templateFormOpen: false, templateFormData: null, @@ -105,6 +109,7 @@ const getInitialState = (): ValidationState => ({ // Copy-Down Mode copyDownMode: { ...initialCopyDownState }, + pendingCopyDownValidation: null, // Dialogs dialogs: { ...initialDialogState }, @@ -574,9 +579,13 @@ export const useValidationStore = create()( const hasValue = sourceValue !== null && sourceValue !== '' && !(Array.isArray(sourceValue) && sourceValue.length === 0); + // Track affected rows for UPC validation + const affectedRows: number[] = []; + for (let i = sourceRowIndex + 1; i <= targetRowIndex; i++) { if (state.rows[i]) { state.rows[i][fieldKey] = cloneValue(sourceValue); + affectedRows.push(i); // Clear validation errors for this field if value is non-empty if (hasValue) { @@ -596,6 +605,15 @@ export const useValidationStore = create()( // Reset copy-down mode state.copyDownMode = { ...initialCopyDownState }; + + // If this field affects UPC validation, store the affected rows + // so a hook can trigger validation using the existing validateUpc function + if (UPC_VALIDATION_FIELDS.includes(fieldKey) && affectedRows.length > 0) { + state.pendingCopyDownValidation = { + fieldKey, + affectedRows, + }; + } }); }, @@ -607,6 +625,12 @@ export const useValidationStore = create()( }); }, + clearPendingCopyDownValidation: () => { + set((state) => { + state.pendingCopyDownValidation = null; + }); + }, + // ========================================================================= // Dialogs // =========================================================================