Add AI embeddings and suggestions for categories, a few validation step tweaks/fixes

This commit is contained in:
2026-01-19 11:34:55 -05:00
parent 9ce84fe5b9
commit 43d76e011d
20 changed files with 5311 additions and 176 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,283 @@
#!/usr/bin/env node
/**
* Embedding Proof-of-Concept Script
*
* Demonstrates how category embeddings work for product matching.
* Uses OpenAI text-embedding-3-small model.
*
* Usage: node scripts/embedding-poc.js
*/
const path = require('path');
require('dotenv').config({ path: path.join(__dirname, '../.env') });
const { getDbConnection, closeAllConnections } = require('../src/utils/dbConnection');
// ============================================================================
// Configuration
// ============================================================================
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
const EMBEDDING_MODEL = 'text-embedding-3-small';
const EMBEDDING_DIMENSIONS = 1536;
// Sample products to test (you can modify these)
const TEST_PRODUCTS = [
{
name: "Cosmos Infinity Chipboard - Stamperia",
description: "Laser-cut chipboard shapes featuring celestial designs for mixed media projects"
},
{
name: "Distress Oxide Ink Pad - Mermaid Lagoon",
description: "Water-reactive dye ink that creates an oxidized effect"
},
{
name: "Hedwig Puffy Stickers - Paper House Productions",
description: "3D puffy stickers featuring Harry Potter's owl Hedwig"
},
{
name: "Black Velvet Watercolor Brush Size 6",
description: "Round brush for watercolor painting with synthetic bristles"
},
{
name: "Floral Washi Tape Set",
description: "Decorative paper tape with flower patterns, pack of 6 rolls"
}
];
// ============================================================================
// OpenAI Embedding Functions
// ============================================================================
async function getEmbeddings(texts) {
const response = await fetch('https://api.openai.com/v1/embeddings', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${OPENAI_API_KEY}`
},
body: JSON.stringify({
input: texts.map(t => t.substring(0, 8000)), // Max 8k chars per text
model: EMBEDDING_MODEL,
dimensions: EMBEDDING_DIMENSIONS
})
});
if (!response.ok) {
const error = await response.json();
throw new Error(`OpenAI API error: ${error.error?.message || response.status}`);
}
const data = await response.json();
// Sort by index to ensure order matches input
const sorted = data.data.sort((a, b) => a.index - b.index);
return {
embeddings: sorted.map(item => item.embedding),
usage: data.usage,
model: data.model
};
}
// ============================================================================
// Vector Math
// ============================================================================
function cosineSimilarity(a, b) {
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
function findTopMatches(queryEmbedding, categoryEmbeddings, topK = 10) {
const scored = categoryEmbeddings.map(cat => ({
...cat,
similarity: cosineSimilarity(queryEmbedding, cat.embedding)
}));
scored.sort((a, b) => b.similarity - a.similarity);
return scored.slice(0, topK);
}
// ============================================================================
// Database Functions
// ============================================================================
async function fetchCategories(connection) {
console.log('\n📂 Fetching categories from database...');
// Fetch hierarchical categories (types 10-13)
const [rows] = await connection.query(`
SELECT
cat_id,
name,
master_cat_id,
type
FROM product_categories
WHERE type IN (10, 11, 12, 13)
ORDER BY type, name
`);
console.log(` Found ${rows.length} category records`);
// Build category paths
const byId = new Map(rows.map(r => [r.cat_id, r]));
const categories = [];
for (const row of rows) {
const path = [];
let current = row;
// Walk up the tree to build full path
while (current) {
path.unshift(current.name);
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
}
categories.push({
id: row.cat_id,
name: row.name,
type: row.type,
fullPath: path.join(' > '),
embeddingText: path.join(' ') // For embedding generation
});
}
// Count by level
const levels = {
10: categories.filter(c => c.type === 10).length,
11: categories.filter(c => c.type === 11).length,
12: categories.filter(c => c.type === 12).length,
13: categories.filter(c => c.type === 13).length,
};
console.log(` Level breakdown: ${levels[10]} top-level, ${levels[11]} L2, ${levels[12]} L3, ${levels[13]} L4`);
return categories;
}
// ============================================================================
// Main Script
// ============================================================================
async function main() {
console.log('═══════════════════════════════════════════════════════════════');
console.log(' EMBEDDING PROOF-OF-CONCEPT');
console.log(' Model: ' + EMBEDDING_MODEL);
console.log('═══════════════════════════════════════════════════════════════');
if (!OPENAI_API_KEY) {
console.error('❌ OPENAI_API_KEY not found in environment');
process.exit(1);
}
let connection;
try {
// Step 1: Connect to database
console.log('\n🔌 Connecting to database via SSH tunnel...');
const { connection: conn } = await getDbConnection();
connection = conn;
console.log(' ✅ Connected');
// Step 2: Fetch categories
const categories = await fetchCategories(connection);
// Step 3: Generate embeddings for categories
console.log('\n🧮 Generating embeddings for categories...');
console.log(' This will cost approximately $' + (categories.length * 0.00002).toFixed(4));
const startTime = Date.now();
// Process in batches of 100 (OpenAI limit is 2048)
const BATCH_SIZE = 100;
let totalTokens = 0;
for (let i = 0; i < categories.length; i += BATCH_SIZE) {
const batch = categories.slice(i, i + BATCH_SIZE);
const texts = batch.map(c => c.embeddingText);
const result = await getEmbeddings(texts);
// Attach embeddings to categories
for (let j = 0; j < batch.length; j++) {
batch[j].embedding = result.embeddings[j];
}
totalTokens += result.usage.total_tokens;
console.log(` Batch ${Math.floor(i / BATCH_SIZE) + 1}/${Math.ceil(categories.length / BATCH_SIZE)}: ${batch.length} categories embedded`);
}
const embeddingTime = Date.now() - startTime;
console.log(` ✅ Generated ${categories.length} embeddings in ${embeddingTime}ms`);
console.log(` 📊 Total tokens used: ${totalTokens} (~$${(totalTokens * 0.00002).toFixed(4)})`);
// Step 4: Test with sample products
console.log('\n═══════════════════════════════════════════════════════════════');
console.log(' TESTING WITH SAMPLE PRODUCTS');
console.log('═══════════════════════════════════════════════════════════════');
for (const product of TEST_PRODUCTS) {
console.log('\n┌─────────────────────────────────────────────────────────────');
console.log(`│ Product: "${product.name}"`);
console.log(`│ Description: "${product.description.substring(0, 60)}..."`);
console.log('├─────────────────────────────────────────────────────────────');
// Generate embedding for product
const productText = `${product.name} ${product.description}`;
const { embeddings: [productEmbedding] } = await getEmbeddings([productText]);
// Find top matches
const matches = findTopMatches(productEmbedding, categories, 10);
console.log('│ Top 10 Category Matches:');
matches.forEach((match, i) => {
const similarity = (match.similarity * 100).toFixed(1);
const bar = '█'.repeat(Math.round(match.similarity * 20));
const marker = i < 3 ? ' ✅' : '';
console.log(`${(i + 1).toString().padStart(2)}. [${similarity.padStart(5)}%] ${bar.padEnd(20)} ${match.fullPath}${marker}`);
});
console.log('└─────────────────────────────────────────────────────────────');
}
// Step 5: Summary
console.log('\n═══════════════════════════════════════════════════════════════');
console.log(' SUMMARY');
console.log('═══════════════════════════════════════════════════════════════');
console.log(` Categories embedded: ${categories.length}`);
console.log(` Embedding time: ${embeddingTime}ms (one-time cost)`);
console.log(` Per-product lookup: ~${(Date.now() - startTime) / TEST_PRODUCTS.length}ms`);
console.log(` Vector dimensions: ${EMBEDDING_DIMENSIONS}`);
console.log(` Memory usage: ~${(categories.length * EMBEDDING_DIMENSIONS * 4 / 1024 / 1024).toFixed(2)} MB (in-memory vectors)`);
console.log('');
console.log(' 💡 In production:');
console.log(' - Category embeddings are computed once and cached');
console.log(' - Only product embedding is computed per-request (~$0.00002)');
console.log(' - Vector search is instant (in-memory cosine similarity)');
console.log(' - Top 10 results go to AI for final selection (~$0.0001)');
console.log('═══════════════════════════════════════════════════════════════\n');
} catch (error) {
console.error('\n❌ Error:', error.message);
if (error.stack) {
console.error(error.stack);
}
process.exit(1);
} finally {
await closeAllConnections();
console.log('🔌 Database connections closed');
}
}
// Run the script
main();

View File

@@ -0,0 +1,281 @@
/**
* AI Routes
*
* API endpoints for AI-powered product validation features.
* Provides embedding generation and similarity-based suggestions.
*/
const express = require('express');
const router = express.Router();
const aiService = require('../services/ai');
const { getDbConnection, closeAllConnections } = require('../utils/dbConnection');
// Track initialization state
let initializationPromise = null;
/**
* Ensure AI service is initialized
* Uses lazy initialization on first request
*/
async function ensureInitialized() {
if (aiService.isReady()) {
return true;
}
if (initializationPromise) {
await initializationPromise;
return aiService.isReady();
}
initializationPromise = (async () => {
try {
console.log('[AI Routes] Initializing AI service...');
// Get database connection for taxonomy
const { connection } = await getDbConnection();
const result = await aiService.initialize({
openaiApiKey: process.env.OPENAI_API_KEY,
mysqlConnection: connection,
logger: console
});
if (!result.success) {
console.error('[AI Routes] AI service initialization failed:', result.message);
return false;
}
console.log('[AI Routes] AI service initialized:', result.stats);
return true;
} catch (error) {
console.error('[AI Routes] Failed to initialize AI service:', error);
return false;
}
})();
await initializationPromise;
return aiService.isReady();
}
/**
* GET /api/ai/status
* Get AI service status
*/
router.get('/status', async (req, res) => {
try {
const status = aiService.getStatus();
res.json(status);
} catch (error) {
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/initialize
* Manually trigger initialization (also happens automatically on first use)
*/
router.post('/initialize', async (req, res) => {
try {
const ready = await ensureInitialized();
const status = aiService.getStatus();
res.json({
success: ready,
...status
});
} catch (error) {
console.error('[AI Routes] Initialize error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* GET /api/ai/taxonomy
* Get all taxonomy data (categories, themes, colors) without embeddings
*/
router.get('/taxonomy', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const taxonomy = aiService.getTaxonomyData();
res.json(taxonomy);
} catch (error) {
console.error('[AI Routes] Taxonomy error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/embedding
* Generate embedding for a single product
*
* Body: { product: { name, description, company_name, line_name } }
* Returns: { embedding: number[], latencyMs: number }
*/
router.post('/embedding', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const { product } = req.body;
if (!product) {
return res.status(400).json({ error: 'Product is required' });
}
const result = await aiService.getProductEmbedding(product);
res.json(result);
} catch (error) {
console.error('[AI Routes] Embedding error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/embeddings
* Generate embeddings for multiple products
*
* Body: { products: Array<{ name, description, company_name, line_name }> }
* Returns: { embeddings: Array<{ index, embedding }>, latencyMs }
*/
router.post('/embeddings', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const { products } = req.body;
if (!Array.isArray(products)) {
return res.status(400).json({ error: 'Products array is required' });
}
const result = await aiService.getProductEmbeddings(products);
res.json(result);
} catch (error) {
console.error('[AI Routes] Embeddings error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/suggestions
* Get category/theme/color suggestions for a single product
* Generates embedding and finds similar taxonomy items
*
* Body: { product: { name, description, company_name, line_name }, options?: { topCategories, topThemes, topColors } }
* Returns: { categories: Array, themes: Array, colors: Array, latencyMs }
*/
router.post('/suggestions', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const { product, options } = req.body;
if (!product) {
return res.status(400).json({ error: 'Product is required' });
}
const suggestions = await aiService.getSuggestionsForProduct(product, options);
res.json(suggestions);
} catch (error) {
console.error('[AI Routes] Suggestions error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/suggestions/batch
* Get suggestions for multiple products
* More efficient than calling /suggestions multiple times
*
* Body: { products: Array, options?: { topCategories, topThemes, topColors } }
* Returns: { results: Array<{ index, categories, themes, colors }>, latencyMs }
*/
router.post('/suggestions/batch', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const { products, options } = req.body;
if (!Array.isArray(products)) {
return res.status(400).json({ error: 'Products array is required' });
}
const startTime = Date.now();
// Generate all embeddings at once
const { embeddings, latencyMs: embeddingLatency } = await aiService.getProductEmbeddings(products);
// Find suggestions for each embedding
const results = embeddings.map(({ index, embedding }) => {
const suggestions = aiService.findSimilarTaxonomy(embedding, options);
return {
index,
...suggestions
};
});
const totalLatency = Date.now() - startTime;
res.json({
results,
latencyMs: totalLatency,
embeddingLatencyMs: embeddingLatency,
searchLatencyMs: totalLatency - embeddingLatency,
productCount: products.length,
embeddingCount: embeddings.length
});
} catch (error) {
console.error('[AI Routes] Batch suggestions error:', error);
res.status(500).json({ error: error.message });
}
});
/**
* POST /api/ai/similar
* Find similar taxonomy items given a pre-computed embedding
* Useful when frontend has cached the embedding
*
* Body: { embedding: number[], options?: { topCategories, topThemes, topColors } }
* Returns: { categories, themes, colors }
*/
router.post('/similar', async (req, res) => {
try {
const ready = await ensureInitialized();
if (!ready) {
return res.status(503).json({ error: 'AI service not available' });
}
const { embedding, options } = req.body;
if (!embedding || !Array.isArray(embedding)) {
return res.status(400).json({ error: 'Embedding array is required' });
}
const startTime = Date.now();
const suggestions = aiService.findSimilarTaxonomy(embedding, options);
res.json({
...suggestions,
latencyMs: Date.now() - startTime
});
} catch (error) {
console.error('[AI Routes] Similar error:', error);
res.status(500).json({ error: error.message });
}
});
module.exports = router;

View File

@@ -15,6 +15,7 @@ const configRouter = require('./routes/config');
const metricsRouter = require('./routes/metrics'); const metricsRouter = require('./routes/metrics');
const importRouter = require('./routes/import'); const importRouter = require('./routes/import');
const aiValidationRouter = require('./routes/ai-validation'); const aiValidationRouter = require('./routes/ai-validation');
const aiRouter = require('./routes/ai');
const templatesRouter = require('./routes/templates'); const templatesRouter = require('./routes/templates');
const aiPromptsRouter = require('./routes/ai-prompts'); const aiPromptsRouter = require('./routes/ai-prompts');
const reusableImagesRouter = require('./routes/reusable-images'); const reusableImagesRouter = require('./routes/reusable-images');
@@ -124,6 +125,7 @@ async function startServer() {
app.use('/api/brands-aggregate', brandsAggregateRouter); app.use('/api/brands-aggregate', brandsAggregateRouter);
app.use('/api/import', importRouter); app.use('/api/import', importRouter);
app.use('/api/ai-validation', aiValidationRouter); app.use('/api/ai-validation', aiValidationRouter);
app.use('/api/ai', aiRouter);
app.use('/api/templates', templatesRouter); app.use('/api/templates', templatesRouter);
app.use('/api/ai-prompts', aiPromptsRouter); app.use('/api/ai-prompts', aiPromptsRouter);
app.use('/api/reusable-images', reusableImagesRouter); app.use('/api/reusable-images', reusableImagesRouter);

View File

@@ -0,0 +1,82 @@
/**
* Vector similarity utilities
*/
/**
* Compute cosine similarity between two vectors
* @param {number[]} a
* @param {number[]} b
* @returns {number} Similarity score between -1 and 1
*/
function cosineSimilarity(a, b) {
if (!a || !b || a.length !== b.length) {
return 0;
}
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
if (denominator === 0) return 0;
return dotProduct / denominator;
}
/**
* Find top K most similar items from a collection
* @param {number[]} queryEmbedding - The embedding to search for
* @param {Array<{id: any, embedding: number[]}>} items - Items with embeddings
* @param {number} topK - Number of results to return
* @returns {Array<{id: any, similarity: number}>}
*/
function findTopMatches(queryEmbedding, items, topK = 10) {
if (!queryEmbedding || !items || items.length === 0) {
return [];
}
const scored = items.map(item => ({
id: item.id,
similarity: cosineSimilarity(queryEmbedding, item.embedding)
}));
scored.sort((a, b) => b.similarity - a.similarity);
return scored.slice(0, topK);
}
/**
* Find matches above a similarity threshold
* @param {number[]} queryEmbedding
* @param {Array<{id: any, embedding: number[]}>} items
* @param {number} threshold - Minimum similarity (0-1)
* @returns {Array<{id: any, similarity: number}>}
*/
function findMatchesAboveThreshold(queryEmbedding, items, threshold = 0.5) {
if (!queryEmbedding || !items || items.length === 0) {
return [];
}
const scored = items
.map(item => ({
id: item.id,
similarity: cosineSimilarity(queryEmbedding, item.embedding)
}))
.filter(item => item.similarity >= threshold);
scored.sort((a, b) => b.similarity - a.similarity);
return scored;
}
module.exports = {
cosineSimilarity,
findTopMatches,
findMatchesAboveThreshold
};

View File

@@ -0,0 +1,323 @@
/**
* Taxonomy Embedding Service
*
* Generates and caches embeddings for categories, themes, and colors.
* Excludes "Black Friday", "Gifts", "Deals" categories and their children.
*/
const { findTopMatches } = require('./similarity');
// Categories to exclude (and all their children)
const EXCLUDED_CATEGORY_NAMES = ['black friday', 'gifts', 'deals'];
class TaxonomyEmbeddings {
constructor({ provider, logger }) {
this.provider = provider;
this.logger = logger || console;
// Cached taxonomy with embeddings
this.categories = [];
this.themes = [];
this.colors = [];
// Raw data without embeddings (for lookup)
this.categoryMap = new Map();
this.themeMap = new Map();
this.colorMap = new Map();
this.initialized = false;
this.initializing = false;
}
/**
* Initialize embeddings - fetch taxonomy and generate embeddings
*/
async initialize(connection) {
if (this.initialized) {
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
}
if (this.initializing) {
// Wait for existing initialization
while (this.initializing) {
await new Promise(resolve => setTimeout(resolve, 100));
}
return { categories: this.categories.length, themes: this.themes.length, colors: this.colors.length };
}
this.initializing = true;
try {
this.logger.info('[TaxonomyEmbeddings] Starting initialization...');
// Fetch raw taxonomy data
const [categories, themes, colors] = await Promise.all([
this._fetchCategories(connection),
this._fetchThemes(connection),
this._fetchColors(connection)
]);
this.logger.info(`[TaxonomyEmbeddings] Fetched ${categories.length} categories, ${themes.length} themes, ${colors.length} colors`);
// Generate embeddings in parallel
const [catEmbeddings, themeEmbeddings, colorEmbeddings] = await Promise.all([
this._generateEmbeddings(categories, 'categories'),
this._generateEmbeddings(themes, 'themes'),
this._generateEmbeddings(colors, 'colors')
]);
// Store with embeddings
this.categories = catEmbeddings;
this.themes = themeEmbeddings;
this.colors = colorEmbeddings;
// Build lookup maps
this.categoryMap = new Map(this.categories.map(c => [c.id, c]));
this.themeMap = new Map(this.themes.map(t => [t.id, t]));
this.colorMap = new Map(this.colors.map(c => [c.id, c]));
this.initialized = true;
this.logger.info('[TaxonomyEmbeddings] Initialization complete');
return {
categories: this.categories.length,
themes: this.themes.length,
colors: this.colors.length
};
} catch (error) {
this.logger.error('[TaxonomyEmbeddings] Initialization failed:', error);
throw error;
} finally {
this.initializing = false;
}
}
/**
* Find similar categories for a product embedding
*/
findSimilarCategories(productEmbedding, topK = 10) {
if (!this.initialized || !productEmbedding) {
return [];
}
const matches = findTopMatches(productEmbedding, this.categories, topK);
return matches.map(match => {
const cat = this.categoryMap.get(match.id);
return {
id: match.id,
name: cat?.name || '',
fullPath: cat?.fullPath || '',
similarity: match.similarity
};
});
}
/**
* Find similar themes for a product embedding
*/
findSimilarThemes(productEmbedding, topK = 5) {
if (!this.initialized || !productEmbedding) {
return [];
}
const matches = findTopMatches(productEmbedding, this.themes, topK);
return matches.map(match => {
const theme = this.themeMap.get(match.id);
return {
id: match.id,
name: theme?.name || '',
fullPath: theme?.fullPath || '',
similarity: match.similarity
};
});
}
/**
* Find similar colors for a product embedding
*/
findSimilarColors(productEmbedding, topK = 5) {
if (!this.initialized || !productEmbedding) {
return [];
}
const matches = findTopMatches(productEmbedding, this.colors, topK);
return matches.map(match => {
const color = this.colorMap.get(match.id);
return {
id: match.id,
name: color?.name || '',
similarity: match.similarity
};
});
}
/**
* Get all taxonomy data (without embeddings) for frontend
*/
getTaxonomyData() {
return {
categories: this.categories.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
themes: this.themes.map(({ id, name, fullPath, parentId }) => ({ id, name, fullPath, parentId })),
colors: this.colors.map(({ id, name }) => ({ id, name }))
};
}
/**
* Check if service is ready
*/
isReady() {
return this.initialized;
}
// ============================================================================
// Private Methods
// ============================================================================
async _fetchCategories(connection) {
// Fetch hierarchical categories (types 10-13)
const [rows] = await connection.query(`
SELECT cat_id, name, master_cat_id, type
FROM product_categories
WHERE type IN (10, 11, 12, 13)
ORDER BY type, name
`);
// Build lookup for hierarchy
const byId = new Map(rows.map(r => [r.cat_id, r]));
// Find IDs of excluded top-level categories and all their descendants
const excludedIds = new Set();
// First pass: find excluded top-level categories
for (const row of rows) {
if (row.type === 10 && EXCLUDED_CATEGORY_NAMES.includes(row.name.toLowerCase())) {
excludedIds.add(row.cat_id);
}
}
// Multiple passes to find all descendants
let foundNew = true;
while (foundNew) {
foundNew = false;
for (const row of rows) {
if (!excludedIds.has(row.cat_id) && excludedIds.has(row.master_cat_id)) {
excludedIds.add(row.cat_id);
foundNew = true;
}
}
}
this.logger.info(`[TaxonomyEmbeddings] Excluding ${excludedIds.size} categories (Black Friday, Gifts, Deals and children)`);
// Build category objects with full paths, excluding filtered ones
const categories = [];
for (const row of rows) {
if (excludedIds.has(row.cat_id)) {
continue;
}
const path = [];
let current = row;
// Walk up the tree to build full path
while (current) {
path.unshift(current.name);
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
}
categories.push({
id: row.cat_id,
name: row.name,
parentId: row.master_cat_id,
type: row.type,
fullPath: path.join(' > '),
embeddingText: path.join(' ')
});
}
return categories;
}
async _fetchThemes(connection) {
// Fetch themes (types 20-21)
const [rows] = await connection.query(`
SELECT cat_id, name, master_cat_id, type
FROM product_categories
WHERE type IN (20, 21)
ORDER BY type, name
`);
const byId = new Map(rows.map(r => [r.cat_id, r]));
const themes = [];
for (const row of rows) {
const path = [];
let current = row;
while (current) {
path.unshift(current.name);
current = current.master_cat_id ? byId.get(current.master_cat_id) : null;
}
themes.push({
id: row.cat_id,
name: row.name,
parentId: row.master_cat_id,
type: row.type,
fullPath: path.join(' > '),
embeddingText: path.join(' ')
});
}
return themes;
}
async _fetchColors(connection) {
const [rows] = await connection.query(`
SELECT color, name, hex_color
FROM product_color_list
ORDER BY \`order\`
`);
return rows.map(row => ({
id: row.color,
name: row.name,
hexColor: row.hex_color,
embeddingText: row.name
}));
}
async _generateEmbeddings(items, label) {
if (items.length === 0) {
return items;
}
const startTime = Date.now();
const texts = items.map(item => item.embeddingText);
const results = [...items];
// Process in batches
let batchNum = 0;
for await (const chunk of this.provider.embedBatchChunked(texts, { batchSize: 100 })) {
batchNum++;
for (let i = 0; i < chunk.embeddings.length; i++) {
const globalIndex = chunk.startIndex + i;
results[globalIndex] = {
...results[globalIndex],
embedding: chunk.embeddings[i]
};
}
}
const elapsed = Date.now() - startTime;
this.logger.info(`[TaxonomyEmbeddings] Generated ${items.length} ${label} embeddings in ${elapsed}ms`);
return results;
}
}
module.exports = { TaxonomyEmbeddings };

View File

@@ -0,0 +1,273 @@
/**
* AI Service
*
* Main entry point for AI functionality including embeddings.
* Provides embedding generation and similarity search for product validation.
*/
const { OpenAIProvider } = require('./providers/openaiProvider');
const { TaxonomyEmbeddings } = require('./embeddings/taxonomyEmbeddings');
const { cosineSimilarity, findTopMatches } = require('./embeddings/similarity');
let initialized = false;
let initializing = false;
let openaiProvider = null;
let taxonomyEmbeddings = null;
let logger = console;
/**
* Initialize the AI service
* @param {Object} options
* @param {string} options.openaiApiKey - OpenAI API key
* @param {Object} options.mysqlConnection - MySQL connection for taxonomy data
* @param {Object} [options.logger] - Logger instance
*/
async function initialize({ openaiApiKey, mysqlConnection, logger: customLogger }) {
if (initialized) {
return { success: true, message: 'Already initialized' };
}
if (initializing) {
// Wait for existing initialization
while (initializing) {
await new Promise(resolve => setTimeout(resolve, 100));
}
return { success: initialized, message: initialized ? 'Initialized' : 'Initialization failed' };
}
initializing = true;
try {
if (customLogger) {
logger = customLogger;
}
if (!openaiApiKey) {
throw new Error('OpenAI API key is required');
}
logger.info('[AI] Initializing AI service...');
// Create OpenAI provider
openaiProvider = new OpenAIProvider({ apiKey: openaiApiKey });
// Create and initialize taxonomy embeddings
taxonomyEmbeddings = new TaxonomyEmbeddings({
provider: openaiProvider,
logger
});
const stats = await taxonomyEmbeddings.initialize(mysqlConnection);
initialized = true;
logger.info('[AI] AI service initialized', stats);
return {
success: true,
message: 'Initialized',
stats
};
} catch (error) {
logger.error('[AI] Initialization failed:', error);
return { success: false, message: error.message };
} finally {
initializing = false;
}
}
/**
* Check if service is ready
*/
function isReady() {
return initialized && taxonomyEmbeddings?.isReady();
}
/**
* Build weighted product text for embedding.
* Weights the product name heavily by repeating it, and truncates long descriptions
* to prevent verbose marketing copy from drowning out the product signal.
*
* @param {Object} product - Product with name, description, company, line
* @returns {string} - Combined text for embedding
*/
function buildProductText(product) {
const parts = [];
const name = product.name?.trim();
const description = product.description?.trim();
const company = (product.company_name || product.company)?.trim();
const line = (product.line_name || product.line)?.trim();
// Name is most important - repeat 3x to weight it heavily in the embedding
if (name) {
parts.push(name, name, name);
}
// Company and line provide context
if (company) {
parts.push(company);
}
if (line) {
parts.push(line);
}
// Truncate description to prevent it from overwhelming the signal
if (description) {
const truncated = description.length > 500
? description.substring(0, 500) + '...'
: description;
parts.push(truncated);
}
return parts.join(' ').trim();
}
/**
* Generate embedding for a product
* @param {Object} product - Product with name, description, company, line
* @returns {Promise<{embedding: number[], latencyMs: number}>}
*/
async function getProductEmbedding(product) {
if (!initialized || !openaiProvider) {
throw new Error('AI service not initialized');
}
const text = buildProductText(product);
if (!text) {
return { embedding: null, latencyMs: 0 };
}
const result = await openaiProvider.embed(text);
return {
embedding: result.embeddings[0],
latencyMs: result.latencyMs
};
}
/**
* Generate embeddings for multiple products
* @param {Object[]} products - Array of products
* @returns {Promise<{embeddings: Array<{index: number, embedding: number[]}>, latencyMs: number}>}
*/
async function getProductEmbeddings(products) {
if (!initialized || !openaiProvider) {
throw new Error('AI service not initialized');
}
const texts = products.map(buildProductText);
// Track which products have empty text
const validIndices = texts.map((t, i) => t ? i : -1).filter(i => i >= 0);
const validTexts = texts.filter(t => t);
if (validTexts.length === 0) {
return { embeddings: [], latencyMs: 0 };
}
const result = await openaiProvider.embed(validTexts);
// Map embeddings back to original indices
const embeddings = validIndices.map((originalIndex, resultIndex) => ({
index: originalIndex,
embedding: result.embeddings[resultIndex]
}));
return {
embeddings,
latencyMs: result.latencyMs
};
}
/**
* Find similar taxonomy items for a product embedding
* @param {number[]} productEmbedding
* @param {Object} options
* @returns {{categories: Array, themes: Array, colors: Array}}
*/
function findSimilarTaxonomy(productEmbedding, options = {}) {
if (!initialized || !taxonomyEmbeddings) {
throw new Error('AI service not initialized');
}
const topCategories = options.topCategories ?? 10;
const topThemes = options.topThemes ?? 5;
const topColors = options.topColors ?? 5;
return {
categories: taxonomyEmbeddings.findSimilarCategories(productEmbedding, topCategories),
themes: taxonomyEmbeddings.findSimilarThemes(productEmbedding, topThemes),
colors: taxonomyEmbeddings.findSimilarColors(productEmbedding, topColors)
};
}
/**
* Get product embedding and find similar taxonomy in one call
* @param {Object} product
* @param {Object} options
*/
async function getSuggestionsForProduct(product, options = {}) {
const { embedding, latencyMs: embeddingLatency } = await getProductEmbedding(product);
if (!embedding) {
return {
categories: [],
themes: [],
colors: [],
latencyMs: embeddingLatency
};
}
const startSearch = Date.now();
const suggestions = findSimilarTaxonomy(embedding, options);
const searchLatency = Date.now() - startSearch;
return {
...suggestions,
latencyMs: embeddingLatency + searchLatency,
embeddingLatencyMs: embeddingLatency,
searchLatencyMs: searchLatency
};
}
/**
* Get all taxonomy data (without embeddings) for frontend
*/
function getTaxonomyData() {
if (!initialized || !taxonomyEmbeddings) {
throw new Error('AI service not initialized');
}
return taxonomyEmbeddings.getTaxonomyData();
}
/**
* Get service status
*/
function getStatus() {
return {
initialized,
ready: isReady(),
hasProvider: !!openaiProvider,
hasTaxonomy: !!taxonomyEmbeddings,
taxonomyStats: taxonomyEmbeddings ? {
categories: taxonomyEmbeddings.categories?.length || 0,
themes: taxonomyEmbeddings.themes?.length || 0,
colors: taxonomyEmbeddings.colors?.length || 0
} : null
};
}
module.exports = {
initialize,
isReady,
getProductEmbedding,
getProductEmbeddings,
findSimilarTaxonomy,
getSuggestionsForProduct,
getTaxonomyData,
getStatus,
// Re-export utilities
cosineSimilarity,
findTopMatches
};

View File

@@ -0,0 +1,117 @@
/**
* OpenAI Provider - Handles embedding generation
*/
const EMBEDDING_MODEL = 'text-embedding-3-small';
const EMBEDDING_DIMENSIONS = 1536;
const MAX_BATCH_SIZE = 2048;
class OpenAIProvider {
constructor({ apiKey, baseUrl = 'https://api.openai.com/v1', timeoutMs = 60000 }) {
if (!apiKey) {
throw new Error('OpenAI API key is required');
}
this.apiKey = apiKey;
this.baseUrl = baseUrl;
this.timeoutMs = timeoutMs;
}
/**
* Generate embeddings for one or more texts
* @param {string|string[]} input - Text or array of texts
* @param {Object} options
* @returns {Promise<{embeddings: number[][], usage: Object, model: string, latencyMs: number}>}
*/
async embed(input, options = {}) {
const texts = Array.isArray(input) ? input : [input];
const model = options.model || EMBEDDING_MODEL;
const dimensions = options.dimensions || EMBEDDING_DIMENSIONS;
const timeoutMs = options.timeoutMs || this.timeoutMs;
if (texts.length > MAX_BATCH_SIZE) {
throw new Error(`Batch size ${texts.length} exceeds max of ${MAX_BATCH_SIZE}`);
}
const started = Date.now();
// Clean and truncate input texts
const cleanedTexts = texts.map(t =>
(t || '').replace(/\n+/g, ' ').trim().substring(0, 8000)
);
const body = {
input: cleanedTexts,
model,
encoding_format: 'float'
};
// Only embedding-3 models support dimensions parameter
if (model.includes('embedding-3')) {
body.dimensions = dimensions;
}
const response = await this._makeRequest('embeddings', body, timeoutMs);
// Sort by index to ensure order matches input
const sortedData = response.data.sort((a, b) => a.index - b.index);
return {
embeddings: sortedData.map(item => item.embedding),
usage: {
promptTokens: response.usage?.prompt_tokens || 0,
totalTokens: response.usage?.total_tokens || 0
},
model: response.model || model,
latencyMs: Date.now() - started
};
}
/**
* Generator for processing large batches in chunks
*/
async *embedBatchChunked(texts, options = {}) {
const batchSize = Math.min(options.batchSize || 100, MAX_BATCH_SIZE);
for (let i = 0; i < texts.length; i += batchSize) {
const chunk = texts.slice(i, i + batchSize);
const result = await this.embed(chunk, options);
yield {
embeddings: result.embeddings,
startIndex: i,
endIndex: i + chunk.length,
usage: result.usage,
model: result.model,
latencyMs: result.latencyMs
};
}
}
async _makeRequest(endpoint, body, timeoutMs) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(`${this.baseUrl}/${endpoint}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
},
body: JSON.stringify(body),
signal: controller.signal
});
if (!response.ok) {
const error = await response.json().catch(() => ({}));
throw new Error(error.error?.message || `OpenAI API error: ${response.status}`);
}
return response.json();
} finally {
clearTimeout(timeout);
}
}
}
module.exports = { OpenAIProvider, EMBEDDING_MODEL, EMBEDDING_DIMENSIONS };

View File

@@ -93,7 +93,7 @@ export const BASE_IMPORT_FIELDS = [
description: "Internal notions number", description: "Internal notions number",
alternateMatches: ["notions #","nmc"], alternateMatches: ["notions #","nmc"],
fieldType: { type: "input" }, fieldType: { type: "input" },
width: 100, width: 110,
validations: [ validations: [
{ rule: "required", errorMessage: "Required", level: "error" }, { rule: "required", errorMessage: "Required", level: "error" },
{ rule: "unique", errorMessage: "Must be unique", level: "error" }, { rule: "unique", errorMessage: "Must be unique", level: "error" },
@@ -106,7 +106,7 @@ export const BASE_IMPORT_FIELDS = [
description: "Product name/title", description: "Product name/title",
alternateMatches: ["sku description","product name"], alternateMatches: ["sku description","product name"],
fieldType: { type: "input" }, fieldType: { type: "input" },
width: 500, width: 400,
validations: [ validations: [
{ rule: "required", errorMessage: "Required", level: "error" }, { rule: "required", errorMessage: "Required", level: "error" },
{ rule: "unique", errorMessage: "Must be unique", level: "error" }, { rule: "unique", errorMessage: "Must be unique", level: "error" },
@@ -133,7 +133,7 @@ export const BASE_IMPORT_FIELDS = [
description: "Quantity of items per individual unit", description: "Quantity of items per individual unit",
alternateMatches: ["inner pack", "inner", "min qty", "unit qty", "min. order qty", "supplier qty/unit"], alternateMatches: ["inner pack", "inner", "min qty", "unit qty", "min. order qty", "supplier qty/unit"],
fieldType: { type: "input" }, fieldType: { type: "input" },
width: 80, width: 100,
validations: [ validations: [
{ rule: "required", errorMessage: "Required", level: "error" }, { rule: "required", errorMessage: "Required", level: "error" },
{ rule: "regex", value: "^[0-9]+$", errorMessage: "Must be a number", level: "error" }, { rule: "regex", value: "^[0-9]+$", errorMessage: "Must be a number", level: "error" },
@@ -148,7 +148,7 @@ export const BASE_IMPORT_FIELDS = [
type: "input", type: "input",
price: true price: true
}, },
width: 100, width: 110,
validations: [ validations: [
{ rule: "required", errorMessage: "Required", level: "error" }, { rule: "required", errorMessage: "Required", level: "error" },
{ rule: "regex", value: "^[0-9]*.?[0-9]+$", errorMessage: "Must be a number", level: "error" }, { rule: "regex", value: "^[0-9]*.?[0-9]+$", errorMessage: "Must be a number", level: "error" },
@@ -312,7 +312,7 @@ export const BASE_IMPORT_FIELDS = [
type: "multi-select", type: "multi-select",
options: [], // Will be populated from API options: [], // Will be populated from API
}, },
width: 350, width: 400,
validations: [{ rule: "required", errorMessage: "Required", level: "error" }], validations: [{ rule: "required", errorMessage: "Required", level: "error" }],
}, },
{ {

View File

@@ -1917,18 +1917,11 @@ const MatchColumnsStepComponent = <T extends string>({
)} )}
<div className="flex items-center gap-2 ml-auto"> <div className="flex items-center gap-2 ml-auto">
<Button
variant="outline"
disabled={isLoading}
onClick={() => handleOnContinue(false)}
>
{translations.matchColumnsStep.nextButtonTitle}
</Button>
<Button <Button
disabled={isLoading} disabled={isLoading}
onClick={() => handleOnContinue(true)} onClick={() => handleOnContinue(true)}
> >
{translations.matchColumnsStep.nextButtonTitle} (New Validation) {translations.matchColumnsStep.nextButtonTitle}
</Button> </Button>
</div> </div>
</div> </div>

View File

@@ -4,7 +4,6 @@ import { UploadStep } from "./UploadStep/UploadStep"
import { SelectHeaderStep } from "./SelectHeaderStep/SelectHeaderStep" import { SelectHeaderStep } from "./SelectHeaderStep/SelectHeaderStep"
import { SelectSheetStep } from "./SelectSheetStep/SelectSheetStep" import { SelectSheetStep } from "./SelectSheetStep/SelectSheetStep"
import { mapWorkbook } from "../utils/mapWorkbook" import { mapWorkbook } from "../utils/mapWorkbook"
import { ValidationStepNew } from "./ValidationStepNew"
import { ValidationStep } from "./ValidationStep" import { ValidationStep } from "./ValidationStep"
import { ImageUploadStep } from "./ImageUploadStep/ImageUploadStep" import { ImageUploadStep } from "./ImageUploadStep/ImageUploadStep"
import { MatchColumnsStep } from "./MatchColumnsStep/MatchColumnsStep" import { MatchColumnsStep } from "./MatchColumnsStep/MatchColumnsStep"
@@ -220,36 +219,8 @@ export const UploadFlow = ({ state, onNext, onBack }: Props) => {
/> />
) )
case StepType.validateData: case StepType.validateData:
// Always use the new ValidationStepNew component
return (
<ValidationStepNew
initialData={state.data}
file={uploadedFile || new File([], "empty.xlsx")}
onBack={() => {
// If we started from scratch, we need to go back to the upload step
if (state.isFromScratch) {
onNext({
type: StepType.upload
});
} else if (onBack) {
// Use the provided onBack function
onBack();
}
}}
onNext={(validatedData: any[]) => {
// Go to image upload step with the validated data
onNext({
type: StepType.imageUpload,
data: validatedData,
file: uploadedFile || new File([], "empty.xlsx"),
globalSelections: state.globalSelections
});
}}
isFromScratch={state.isFromScratch}
/>
)
case StepType.validateDataNew: case StepType.validateDataNew:
// New Zustand-based ValidationStep component // Zustand-based ValidationStep component (both cases now use this)
return ( return (
<ValidationStep <ValidationStep
initialData={state.data} initialData={state.data}
@@ -282,7 +253,15 @@ export const UploadFlow = ({ state, onNext, onBack }: Props) => {
<ImageUploadStep <ImageUploadStep
data={state.data} data={state.data}
file={state.file} file={state.file}
onBack={onBack} onBack={() => {
// Go back to the validation step with the current data
onNext({
type: StepType.validateDataNew,
data: state.data,
file: state.file,
globalSelections: state.globalSelections
});
}}
onSubmit={(data, file, options) => { onSubmit={(data, file, options) => {
// Create a Result object from the array data // Create a Result object from the array data
const result = { const result = {

View File

@@ -0,0 +1,192 @@
/**
* Suggestion Badges Component
*
* Displays AI-suggested options inline for categories, themes, and colors.
* Shows similarity scores and allows one-click selection.
*/
import { Sparkles, Loader2, Plus, Check } from 'lucide-react';
import { cn } from '@/lib/utils';
import type { TaxonomySuggestion } from '../store/types';
interface SuggestionBadgesProps {
/** Suggestions to display */
suggestions: TaxonomySuggestion[];
/** Currently selected values (IDs) */
selectedValues: (string | number)[];
/** Callback when a suggestion is clicked */
onSelect: (id: number) => void;
/** Whether suggestions are loading */
isLoading?: boolean;
/** Maximum suggestions to show */
maxSuggestions?: number;
/** Minimum similarity to show (0-1) */
minSimilarity?: number;
/** Label for the section */
label?: string;
/** Compact mode for smaller displays */
compact?: boolean;
/** Show similarity scores */
showScores?: boolean;
/** Custom class name */
className?: string;
}
export function SuggestionBadges({
suggestions,
selectedValues,
onSelect,
isLoading = false,
maxSuggestions = 5,
minSimilarity = 0,
label = 'Suggested',
compact = false,
showScores = true,
className,
}: SuggestionBadgesProps) {
// Filter and limit suggestions
const filteredSuggestions = suggestions
.filter(s => s.similarity >= minSimilarity)
.slice(0, maxSuggestions);
// Don't render if no suggestions and not loading
if (!isLoading && filteredSuggestions.length === 0) {
return null;
}
const isSelected = (id: number) => {
return selectedValues.some(v => String(v) === String(id));
};
return (
<div className={cn('flex flex-wrap items-center gap-1.5', className)}>
{/* Label */}
<div className={cn(
'flex items-center gap-1 text-purple-600 dark:text-purple-400',
compact ? 'text-[10px]' : 'text-xs'
)}>
<Sparkles className={compact ? 'h-2.5 w-2.5' : 'h-3 w-3'} />
{!compact && <span className="font-medium">{label}:</span>}
</div>
{/* Loading state */}
{isLoading && (
<div className="flex items-center gap-1 text-gray-400">
<Loader2 className={cn('animate-spin', compact ? 'h-2.5 w-2.5' : 'h-3 w-3')} />
{!compact && <span className="text-xs">Loading...</span>}
</div>
)}
{/* Suggestion badges */}
{filteredSuggestions.map((suggestion) => {
const selected = isSelected(suggestion.id);
const similarityPercent = Math.round(suggestion.similarity * 100);
return (
<button
key={suggestion.id}
type="button"
onClick={() => onSelect(suggestion.id)}
disabled={selected}
className={cn(
'inline-flex items-center gap-1 rounded-full border transition-colors',
compact ? 'px-1.5 py-0.5 text-[10px]' : 'px-2 py-0.5 text-xs',
selected
? 'border-green-300 bg-green-50 text-green-700 dark:border-green-700 dark:bg-green-950 dark:text-green-400'
: 'border-purple-200 bg-purple-50 text-purple-700 hover:bg-purple-100 dark:border-purple-800 dark:bg-purple-950/50 dark:text-purple-300 dark:hover:bg-purple-900/50'
)}
title={suggestion.fullPath || suggestion.name}
>
{selected ? (
<Check className={compact ? 'h-2 w-2' : 'h-2.5 w-2.5'} />
) : (
<Plus className={compact ? 'h-2 w-2' : 'h-2.5 w-2.5'} />
)}
<span className="truncate max-w-[120px]">{suggestion.name}</span>
{showScores && !compact && (
<span className={cn(
'opacity-60',
selected ? 'text-green-600' : 'text-purple-500'
)}>
{similarityPercent}%
</span>
)}
</button>
);
})}
</div>
);
}
/**
* Inline suggestion for a single field (used inside dropdowns)
*/
interface InlineSuggestionProps {
suggestion: TaxonomySuggestion;
isSelected: boolean;
onSelect: () => void;
showScore?: boolean;
}
export function InlineSuggestion({
suggestion,
isSelected,
onSelect,
showScore = true,
}: InlineSuggestionProps) {
const similarityPercent = Math.round(suggestion.similarity * 100);
return (
<div
className={cn(
'flex items-center justify-between px-2 py-1.5 cursor-pointer',
isSelected
? 'bg-green-50 dark:bg-green-950/30'
: 'bg-purple-50/50 hover:bg-purple-100/50 dark:bg-purple-950/20 dark:hover:bg-purple-900/30'
)}
onClick={onSelect}
>
<div className="flex items-center gap-2 min-w-0">
<Sparkles className="h-3 w-3 text-purple-500 flex-shrink-0" />
<span className="truncate text-sm">
{suggestion.fullPath || suggestion.name}
</span>
</div>
<div className="flex items-center gap-2 flex-shrink-0 ml-2">
{showScore && (
<span className="text-xs text-purple-500 dark:text-purple-400">
{similarityPercent}%
</span>
)}
{isSelected ? (
<Check className="h-3.5 w-3.5 text-green-500" />
) : (
<Plus className="h-3.5 w-3.5 text-purple-400" />
)}
</div>
</div>
);
}
/**
* Suggestion section header for dropdowns
*/
interface SuggestionSectionHeaderProps {
isLoading?: boolean;
count?: number;
}
export function SuggestionSectionHeader({ isLoading, count }: SuggestionSectionHeaderProps) {
return (
<div className="flex items-center gap-2 px-2 py-1.5 text-xs font-medium text-purple-600 dark:text-purple-400 bg-purple-50/80 dark:bg-purple-950/40 border-b border-purple-100 dark:border-purple-900">
<Sparkles className="h-3 w-3" />
<span>AI Suggested</span>
{isLoading && <Loader2 className="h-3 w-3 animate-spin" />}
{!isLoading && count !== undefined && (
<span className="text-purple-400 dark:text-purple-500">({count})</span>
)}
</div>
);
}
export default SuggestionBadges;

View File

@@ -6,7 +6,7 @@
* Note: Initialization effects are in index.tsx so they run before this mounts. * Note: Initialization effects are in index.tsx so they run before this mounts.
*/ */
import { useCallback, useMemo } from 'react'; import { useCallback, useMemo, useRef } from 'react';
import { useValidationStore } from '../store/validationStore'; import { useValidationStore } from '../store/validationStore';
import { import {
useTotalErrorCount, useTotalErrorCount,
@@ -21,11 +21,13 @@ import { FloatingSelectionBar } from './FloatingSelectionBar';
import { useAiValidationFlow } from '../hooks/useAiValidation'; import { useAiValidationFlow } from '../hooks/useAiValidation';
import { useFieldOptions } from '../hooks/useFieldOptions'; import { useFieldOptions } from '../hooks/useFieldOptions';
import { useTemplateManagement } from '../hooks/useTemplateManagement'; import { useTemplateManagement } from '../hooks/useTemplateManagement';
import { useCopyDownValidation } from '../hooks/useCopyDownValidation';
import { AiValidationProgressDialog } from '../dialogs/AiValidationProgress'; import { AiValidationProgressDialog } from '../dialogs/AiValidationProgress';
import { AiValidationResultsDialog } from '../dialogs/AiValidationResults'; import { AiValidationResultsDialog } from '../dialogs/AiValidationResults';
import { AiDebugDialog } from '../dialogs/AiDebugDialog'; import { AiDebugDialog } from '../dialogs/AiDebugDialog';
import { TemplateForm } from '@/components/templates/TemplateForm'; import { TemplateForm } from '@/components/templates/TemplateForm';
import type { CleanRowData } from '../store/types'; import { AiSuggestionsProvider } from '../contexts/AiSuggestionsContext';
import type { CleanRowData, RowData } from '../store/types';
interface ValidationContainerProps { interface ValidationContainerProps {
onBack?: () => void; onBack?: () => void;
@@ -57,6 +59,32 @@ export const ValidationContainer = ({
const { data: fieldOptionsData } = useFieldOptions(); const { data: fieldOptionsData } = useFieldOptions();
const { loadTemplates } = useTemplateManagement(); const { loadTemplates } = useTemplateManagement();
// Handle UPC validation after copy-down operations on supplier/upc fields
useCopyDownValidation();
// Get initial products for AI suggestions (read once via ref to avoid re-fetching)
const initialProductsRef = useRef<RowData[] | null>(null);
if (initialProductsRef.current === null) {
initialProductsRef.current = useValidationStore.getState().rows;
}
// Create stable lookup functions for company/line names
const getCompanyName = useCallback((id: string): string | undefined => {
const companies = fieldOptionsData?.companies || [];
const company = companies.find(c => c.value === id);
return company?.label;
}, [fieldOptionsData?.companies]);
const getLineName = useCallback((id: string): string | undefined => {
// Lines are fetched dynamically per company, check the cache
const cache = useValidationStore.getState().productLinesCache;
for (const lines of cache.values()) {
const line = lines.find(l => l.value === id);
if (line) return line.label;
}
return undefined;
}, []);
// Convert field options to TemplateForm format // Convert field options to TemplateForm format
const templateFormFieldOptions = useMemo(() => { const templateFormFieldOptions = useMemo(() => {
if (!fieldOptionsData) return null; if (!fieldOptionsData) return null;
@@ -94,6 +122,12 @@ export const ValidationContainer = ({
}, [onBack]); }, [onBack]);
return ( return (
<AiSuggestionsProvider
getCompanyName={getCompanyName}
getLineName={getLineName}
initialProducts={initialProductsRef.current || undefined}
autoInitialize={!!fieldOptionsData}
>
<div className="flex flex-col h-[calc(100vh-9.5rem)] overflow-hidden"> <div className="flex flex-col h-[calc(100vh-9.5rem)] overflow-hidden">
{/* Toolbar */} {/* Toolbar */}
<ValidationToolbar <ValidationToolbar
@@ -158,5 +192,6 @@ export const ValidationContainer = ({
fieldOptions={templateFormFieldOptions} fieldOptions={templateFormFieldOptions}
/> />
</div> </div>
</AiSuggestionsProvider>
); );
}; };

View File

@@ -47,6 +47,12 @@ import { ComboboxCell } from './cells/ComboboxCell';
import { MultiSelectCell } from './cells/MultiSelectCell'; import { MultiSelectCell } from './cells/MultiSelectCell';
import { MultilineInput } from './cells/MultilineInput'; import { MultilineInput } from './cells/MultilineInput';
// AI Suggestions context
import { useAiSuggestionsContext } from '../contexts/AiSuggestionsContext';
// Fields that trigger AI suggestion refresh when changed
const AI_EMBEDDING_FIELDS = ['company', 'line', 'name', 'description'] as const;
// Threshold for switching to ComboboxCell (with search) instead of SelectCell // Threshold for switching to ComboboxCell (with search) instead of SelectCell
const COMBOBOX_OPTION_THRESHOLD = 50; const COMBOBOX_OPTION_THRESHOLD = 50;
@@ -96,6 +102,8 @@ const EMPTY_ROW_ERRORS: Record<string, ValidationError[]> = {};
interface CellWrapperProps { interface CellWrapperProps {
field: Field<string>; field: Field<string>;
rowIndex: number; rowIndex: number;
/** Product's unique __index for AI suggestions */
productIndex: string;
value: unknown; value: unknown;
errors: ValidationError[]; errors: ValidationError[];
isValidating: boolean; isValidating: boolean;
@@ -124,6 +132,7 @@ interface CellWrapperProps {
const CellWrapper = memo(({ const CellWrapper = memo(({
field, field,
rowIndex, rowIndex,
productIndex,
value, value,
errors, errors,
isValidating, isValidating,
@@ -142,6 +151,11 @@ const CellWrapper = memo(({
const needsCompany = field.key === 'line'; const needsCompany = field.key === 'line';
const needsLine = field.key === 'subline'; const needsLine = field.key === 'subline';
// AI suggestions context - for notifying when embedding fields change
// This uses stable callbacks so it won't cause re-renders
const aiSuggestions = useAiSuggestionsContext();
const isEmbeddingField = AI_EMBEDDING_FIELDS.includes(field.key as typeof AI_EMBEDDING_FIELDS[number]);
// Check if cell has a value (for showing copy-down button) // Check if cell has a value (for showing copy-down button)
const hasValue = value !== undefined && value !== null && value !== ''; const hasValue = value !== undefined && value !== null && value !== '';
@@ -390,8 +404,17 @@ const CellWrapper = memo(({
} }
} }
} }
// Notify AI suggestions system when embedding fields change
// This triggers a refresh of category/theme/color suggestions
if (isEmbeddingField && aiSuggestions) {
const currentRow = useValidationStore.getState().rows[rowIndex];
if (currentRow) {
aiSuggestions.handleFieldBlur(currentRow, field.key);
}
}
}, 0); }, 0);
}, [rowIndex, field.key]); }, [rowIndex, field.key, isEmbeddingField, aiSuggestions]);
// Stable callback for fetching options (for line/subline dropdowns) // Stable callback for fetching options (for line/subline dropdowns)
const handleFetchOptions = useCallback(async () => { const handleFetchOptions = useCallback(async () => {
@@ -481,6 +504,7 @@ const CellWrapper = memo(({
field={field} field={field}
options={options} options={options}
rowIndex={rowIndex} rowIndex={rowIndex}
productIndex={productIndex}
isValidating={isValidating} isValidating={isValidating}
errors={errors} errors={errors}
onChange={handleChange} onChange={handleChange}
@@ -553,6 +577,7 @@ CellWrapper.displayName = 'CellWrapper';
* Template column width * Template column width
*/ */
const TEMPLATE_COLUMN_WIDTH = 200; const TEMPLATE_COLUMN_WIDTH = 200;
const NAME_COLUMN_STICKY_LEFT = 0;
/** /**
* TemplateCell Component * TemplateCell Component
@@ -777,9 +802,12 @@ const VirtualRow = memo(({
useCallback((state) => state.errors.get(rowIndex) ?? EMPTY_ROW_ERRORS, [rowIndex]) useCallback((state) => state.errors.get(rowIndex) ?? EMPTY_ROW_ERRORS, [rowIndex])
); );
// DON'T subscribe to validatingCells - check it during render instead // DON'T subscribe to validatingCells for most fields - check it during render instead
// This avoids creating new objects in selectors which causes infinite loops // This avoids creating new objects in selectors which causes infinite loops
// Validation status changes are rare, so reading via getState() is fine // EXCEPTION: Subscribe specifically for item_number so it shows loading state during UPC validation
const isItemNumberValidating = useValidationStore(
useCallback((state) => state.validatingCells.has(`${rowIndex}-item_number`), [rowIndex])
);
// Subscribe to selection status // Subscribe to selection status
const isSelected = useValidationStore( const isSelected = useValidationStore(
@@ -860,7 +888,10 @@ const VirtualRow = memo(({
const columnWidth = columns[fieldIndex + 2]?.size || field.width || 150; const columnWidth = columns[fieldIndex + 2]?.size || field.width || 150;
const fieldErrors = rowErrors[field.key] || EMPTY_ERRORS; const fieldErrors = rowErrors[field.key] || EMPTY_ERRORS;
// Check validating status via getState() - not subscribed to avoid object creation // Check validating status via getState() - not subscribed to avoid object creation
const isValidating = useValidationStore.getState().validatingCells.has(`${rowIndex}-${field.key}`); // EXCEPTION: item_number uses the subscribed isItemNumberValidating for reactive loading state
const isValidating = field.key === 'item_number'
? isItemNumberValidating
: useValidationStore.getState().validatingCells.has(`${rowIndex}-${field.key}`);
// CRITICAL: Only pass company/line to cells that need them! // CRITICAL: Only pass company/line to cells that need them!
// Passing to all cells breaks memoization - when company changes, ALL cells re-render // Passing to all cells breaks memoization - when company changes, ALL cells re-render
@@ -889,20 +920,27 @@ const VirtualRow = memo(({
copyDownMode.targetRowIndex !== null && copyDownMode.targetRowIndex !== null &&
rowIndex <= copyDownMode.targetRowIndex; rowIndex <= copyDownMode.targetRowIndex;
const isNameColumn = field.key === 'name';
return ( return (
<div <div
key={field.key} key={field.key}
data-cell-field={field.key} data-cell-field={field.key}
className="px-2 py-1 border-r last:border-r-0 flex items-center overflow-hidden" className={cn(
"px-2 py-1 border-r last:border-r-0 flex items-center overflow-hidden",
isNameColumn && "lg:sticky lg:z-10 lg:bg-background lg:shadow-md"
)}
style={{ style={{
width: columnWidth, width: columnWidth,
minWidth: columnWidth, minWidth: columnWidth,
flexShrink: 0, flexShrink: 0,
...(isNameColumn && { left: NAME_COLUMN_STICKY_LEFT }),
}} }}
> >
<CellWrapper <CellWrapper
field={field} field={field}
rowIndex={rowIndex} rowIndex={rowIndex}
productIndex={rowId}
value={rowData?.[field.key]} value={rowData?.[field.key]}
errors={fieldErrors} errors={fieldErrors}
isValidating={isValidating} isValidating={isValidating}
@@ -1088,21 +1126,28 @@ export const ValidationTable = () => {
className="flex h-full" className="flex h-full"
style={{ minWidth: totalTableWidth }} style={{ minWidth: totalTableWidth }}
> >
{columns.map((column, index) => ( {columns.map((column, index) => {
const isNameColumn = column.id === 'name';
return (
<div <div
key={column.id || index} key={column.id || index}
className="px-3 flex items-center text-left text-sm font-medium text-muted-foreground border-r last:border-r-0" className={cn(
"px-3 flex items-center text-left text-sm font-medium text-muted-foreground border-r last:border-r-0",
isNameColumn && "lg:sticky lg:z-20 lg:bg-muted lg:shadow-md"
)}
style={{ style={{
width: column.size || 150, width: column.size || 150,
minWidth: column.size || 150, minWidth: column.size || 150,
flexShrink: 0, flexShrink: 0,
...(isNameColumn && { left: NAME_COLUMN_STICKY_LEFT }),
}} }}
> >
{typeof column.header === 'function' {typeof column.header === 'function'
? column.header({} as any) ? column.header({} as any)
: column.header} : column.header}
</div> </div>
))} );
})}
</div> </div>
</div> </div>

View File

@@ -6,10 +6,14 @@
* *
* PERFORMANCE: Uses uncontrolled open state for Popover. * PERFORMANCE: Uses uncontrolled open state for Popover.
* Controlled open state can cause delays due to React state processing. * Controlled open state can cause delays due to React state processing.
*
* AI SUGGESTIONS: For categories, themes, and colors fields, this component
* displays AI-powered suggestions based on product embeddings. Suggestions
* appear at the top of the dropdown with similarity scores.
*/ */
import { useCallback, useMemo, memo, useState } from 'react'; import { useCallback, useMemo, memo, useState } from 'react';
import { Check, ChevronsUpDown, AlertCircle } from 'lucide-react'; import { Check, ChevronsUpDown, AlertCircle, Sparkles, Loader2 } from 'lucide-react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { import {
Command, Command,
@@ -34,8 +38,9 @@ import { Badge } from '@/components/ui/badge';
import { Skeleton } from '@/components/ui/skeleton'; import { Skeleton } from '@/components/ui/skeleton';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import type { Field, SelectOption } from '../../../../types'; import type { Field, SelectOption } from '../../../../types';
import type { ValidationError } from '../../store/types'; import type { ValidationError, TaxonomySuggestion } from '../../store/types';
import { ErrorType } from '../../store/types'; import { ErrorType } from '../../store/types';
import { useCellSuggestions } from '../../contexts/AiSuggestionsContext';
// Extended option type to include hex color values // Extended option type to include hex color values
interface MultiSelectOption extends SelectOption { interface MultiSelectOption extends SelectOption {
@@ -49,6 +54,8 @@ interface MultiSelectCellProps {
field: Field<string>; field: Field<string>;
options?: SelectOption[]; options?: SelectOption[];
rowIndex: number; rowIndex: number;
/** Product's unique __index for AI suggestions */
productIndex?: string;
isValidating: boolean; isValidating: boolean;
errors: ValidationError[]; errors: ValidationError[];
onChange: (value: unknown) => void; onChange: (value: unknown) => void;
@@ -56,6 +63,10 @@ interface MultiSelectCellProps {
onFetchOptions?: () => void; onFetchOptions?: () => void;
} }
// Fields that support AI suggestions
const SUGGESTION_FIELDS = ['categories', 'themes', 'colors'] as const;
type SuggestionField = typeof SUGGESTION_FIELDS[number];
/** /**
* Helper to extract hex color from option * Helper to extract hex color from option
* Supports hex, hexColor, and hex_color field names * Supports hex, hexColor, and hex_color field names
@@ -79,6 +90,7 @@ const MultiSelectCellComponent = ({
value, value,
field, field,
options = [], options = [],
productIndex,
isValidating, isValidating,
errors, errors,
onChange: _onChange, // Unused - onBlur handles both update and validation onChange: _onChange, // Unused - onBlur handles both update and validation
@@ -86,6 +98,21 @@ const MultiSelectCellComponent = ({
}: MultiSelectCellProps) => { }: MultiSelectCellProps) => {
const [open, setOpen] = useState(false); const [open, setOpen] = useState(false);
// Get AI suggestions for categories, themes, and colors
const supportsSuggestions = SUGGESTION_FIELDS.includes(field.key as SuggestionField);
const suggestions = useCellSuggestions(productIndex || '');
// Get the right suggestions based on field type
const fieldSuggestions: TaxonomySuggestion[] = useMemo(() => {
if (!supportsSuggestions || !productIndex) return [];
switch (field.key) {
case 'categories': return suggestions.categories;
case 'themes': return suggestions.themes;
case 'colors': return suggestions.colors;
default: return [];
}
}, [supportsSuggestions, productIndex, field.key, suggestions]);
// Handle wheel scroll in dropdown - stop propagation to prevent table scroll // Handle wheel scroll in dropdown - stop propagation to prevent table scroll
const handleWheel = useCallback((e: React.WheelEvent<HTMLDivElement>) => { const handleWheel = useCallback((e: React.WheelEvent<HTMLDivElement>) => {
e.stopPropagation(); e.stopPropagation();
@@ -216,17 +243,108 @@ const MultiSelectCellComponent = ({
<ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" /> <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
</Button> </Button>
</PopoverTrigger> </PopoverTrigger>
<PopoverContent className="w-[300px] p-0" align="start"> <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0" align="start">
<Command> <Command>
<CommandInput placeholder={`Search ${field.label}...`} /> <CommandInput placeholder={`Search ${field.label}...`} />
<CommandList> <CommandList>
<CommandEmpty>No options found.</CommandEmpty> <CommandEmpty>No options found.</CommandEmpty>
<div <div
className="max-h-[200px] overflow-y-auto overscroll-contain" className="max-h-[250px] overflow-y-auto overscroll-contain"
onWheel={handleWheel} onWheel={handleWheel}
> >
{/* Selected items section - floats to top of dropdown */}
{selectedValues.length > 0 && (
<CommandGroup> <CommandGroup>
{options.map((option) => { <div className="flex items-center gap-2 px-2 py-1.5 text-xs font-medium text-green-600 dark:text-green-400 bg-green-50/80 dark:bg-green-950/40 border-b border-green-100 dark:border-green-900">
<Check className="h-3 w-3" />
<span>Selected ({selectedValues.length})</span>
</div>
{selectedValues.map((selectedVal) => {
const option = options.find((opt) => opt.value === selectedVal) as MultiSelectOption | undefined;
const hexColor = field.key === 'colors' && option ? getOptionHex(option) : undefined;
const isWhite = hexColor ? isWhiteColor(hexColor) : false;
const label = option?.label || selectedVal;
return (
<CommandItem
key={`selected-${selectedVal}`}
value={`selected-${label}`}
onSelect={() => handleSelect(selectedVal)}
className="bg-green-50/50 dark:bg-green-950/30"
>
<Check className="mr-2 h-4 w-4 opacity-100 text-green-600" />
{field.key === 'colors' && hexColor && (
<span
className={cn(
'inline-block h-3.5 w-3.5 rounded-full mr-2 flex-shrink-0',
isWhite && 'border border-black'
)}
style={{ backgroundColor: hexColor }}
/>
)}
{label}
</CommandItem>
);
})}
</CommandGroup>
)}
{/* AI Suggestions section - shown below selected items */}
{supportsSuggestions && (fieldSuggestions.length > 0 || suggestions.isLoading) && (
<CommandGroup>
<div className="flex items-center gap-2 px-2 py-1.5 text-xs font-medium text-purple-600 dark:text-purple-400 bg-purple-50/80 dark:bg-purple-950/40 border-b border-purple-100 dark:border-purple-900">
<Sparkles className="h-3 w-3" />
<span>Suggested</span>
{suggestions.isLoading && <Loader2 className="h-3 w-3 animate-spin" />}
</div>
{fieldSuggestions.slice(0, 5).map((suggestion) => {
const isSelected = selectedValues.includes(String(suggestion.id));
// Skip suggestions that are already in the Selected section
if (isSelected) return null;
const similarityPercent = Math.round(suggestion.similarity * 100);
const hexColor = field.key === 'colors'
? options.find(o => o.value === String(suggestion.id)) as MultiSelectOption | undefined
: undefined;
const suggestionHex = hexColor ? getOptionHex(hexColor) : undefined;
return (
<CommandItem
key={`suggestion-${suggestion.id}`}
value={`suggestion-${suggestion.name}`}
onSelect={() => handleSelect(String(suggestion.id))}
className="bg-purple-50/30 dark:bg-purple-950/20"
>
<div className="flex items-center gap-2 min-w-0 flex-1">
<Check className="h-4 w-4 flex-shrink-0 opacity-0" />
{/* Color circle for colors */}
{field.key === 'colors' && suggestionHex && (
<span
className={cn(
'inline-block h-3.5 w-3.5 rounded-full flex-shrink-0',
isWhiteColor(suggestionHex) && 'border border-black'
)}
style={{ backgroundColor: suggestionHex }}
/>
)}
{/* Show full path for categories/themes, just name for colors */}
<span className="" title={suggestion.fullPath || suggestion.name}>
{field.key === 'colors' ? suggestion.name : (suggestion.fullPath || suggestion.name)}
</span>
</div>
<span className="text-xs text-purple-500 dark:text-purple-400 ml-2 flex-shrink-0">
{similarityPercent}%
</span>
</CommandItem>
);
})}
</CommandGroup>
)}
{/* Regular options - excludes selected items (shown in Selected section above) */}
<CommandGroup heading={selectedValues.length > 0 || (supportsSuggestions && fieldSuggestions.length > 0) ? "All Options" : undefined}>
{options
.filter((option) => !selectedValues.includes(option.value))
.map((option) => {
const hexColor = field.key === 'colors' ? getOptionHex(option as MultiSelectOption) : undefined; const hexColor = field.key === 'colors' ? getOptionHex(option as MultiSelectOption) : undefined;
const isWhite = hexColor ? isWhiteColor(hexColor) : false; const isWhite = hexColor ? isWhiteColor(hexColor) : false;
@@ -236,14 +354,7 @@ const MultiSelectCellComponent = ({
value={option.label} value={option.label}
onSelect={() => handleSelect(option.value)} onSelect={() => handleSelect(option.value)}
> >
<Check <Check className="mr-2 h-4 w-4 opacity-0" />
className={cn(
'mr-2 h-4 w-4',
selectedValues.includes(option.value)
? 'opacity-100'
: 'opacity-0'
)}
/>
{/* Color circle for colors field */} {/* Color circle for colors field */}
{field.key === 'colors' && hexColor && ( {field.key === 'colors' && hexColor && (
<span <span

View File

@@ -9,6 +9,12 @@ import { useState, useCallback, useRef, useEffect, memo } from 'react';
import { Textarea } from '@/components/ui/textarea'; import { Textarea } from '@/components/ui/textarea';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { Popover, PopoverTrigger, PopoverContent } from '@/components/ui/popover'; import { Popover, PopoverTrigger, PopoverContent } from '@/components/ui/popover';
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from '@/components/ui/tooltip';
import { X, Loader2 } from 'lucide-react'; import { X, Loader2 } from 'lucide-react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import type { Field, SelectOption } from '../../../../types'; import type { Field, SelectOption } from '../../../../types';
@@ -114,9 +120,16 @@ const MultilineInputComponent = ({
// Calculate display value // Calculate display value
const displayValue = localDisplayValue !== null ? localDisplayValue : String(value ?? ''); const displayValue = localDisplayValue !== null ? localDisplayValue : String(value ?? '');
// Tooltip content - show full description or error message
const tooltipContent = errorMessage || displayValue;
const showTooltip = tooltipContent && tooltipContent.length > 30;
return ( return (
<div className="w-full relative" ref={cellRef}> <div className="w-full relative" ref={cellRef}>
<Popover open={popoverOpen} onOpenChange={handlePopoverOpenChange}> <Popover open={popoverOpen} onOpenChange={handlePopoverOpenChange}>
<TooltipProvider>
<Tooltip delayDuration={300}>
<TooltipTrigger asChild>
<PopoverTrigger asChild> <PopoverTrigger asChild>
<div <div
onClick={handleTriggerClick} onClick={handleTriggerClick}
@@ -127,11 +140,22 @@ const MultilineInputComponent = ({
hasError ? 'border-destructive bg-destructive/5' : 'border-input', hasError ? 'border-destructive bg-destructive/5' : 'border-input',
isValidating && 'opacity-50' isValidating && 'opacity-50'
)} )}
title={errorMessage || displayValue}
> >
{displayValue} {displayValue}
</div> </div>
</PopoverTrigger> </PopoverTrigger>
</TooltipTrigger>
{showTooltip && !popoverOpen && (
<TooltipContent
side="top"
align="start"
className="max-w-[400px] whitespace-pre-wrap"
>
<p>{tooltipContent}</p>
</TooltipContent>
)}
</Tooltip>
</TooltipProvider>
<PopoverContent <PopoverContent
className="p-0 shadow-lg rounded-md" className="p-0 shadow-lg rounded-md"
style={{ width: Math.max(cellRef.current?.offsetWidth || 300, 300) }} style={{ width: Math.max(cellRef.current?.offsetWidth || 300, 300) }}

View File

@@ -0,0 +1,429 @@
/**
* AI Suggestions Context
*
* Provides embedding-based suggestions to cells without causing re-renders.
* Uses refs to store suggestion data and callbacks, so consumers can read
* values on-demand without subscribing to state changes.
*
* PERFORMANCE: This context deliberately uses refs instead of state to avoid
* cascading re-renders through the virtualized table. Cells read suggestions
* when they need them (e.g., when dropdown opens).
*/
import React, { createContext, useContext, useRef, useCallback, useEffect, useState } from 'react';
import type { RowData, ProductSuggestions, TaxonomySuggestion } from '../store/types';
// ============================================================================
// Types
// ============================================================================
interface AiSuggestionsContextValue {
/** Check if service is initialized */
isInitialized: boolean;
/** Get suggestions for a product by index */
getSuggestions: (productIndex: string) => ProductSuggestions | undefined;
/** Check if suggestions are loading for a product */
isLoading: (productIndex: string) => boolean;
/** Trigger suggestion fetch for a product */
fetchSuggestions: (product: RowData) => void;
/** Handle field blur - refreshes suggestions if relevant field changed */
handleFieldBlur: (product: RowData, fieldKey: string) => void;
/** Get category suggestions for a product */
getCategorySuggestions: (productIndex: string) => TaxonomySuggestion[];
/** Get theme suggestions for a product */
getThemeSuggestions: (productIndex: string) => TaxonomySuggestion[];
/** Get color suggestions for a product */
getColorSuggestions: (productIndex: string) => TaxonomySuggestion[];
/** Subscribe to suggestion changes for a product (returns unsubscribe fn) */
subscribe: (productIndex: string, callback: () => void) => () => void;
/** Force refresh suggestions for all products */
refreshAll: () => void;
}
interface AiSuggestionsProviderProps {
children: React.ReactNode;
/** Get company name by ID */
getCompanyName?: (id: string) => string | undefined;
/** Get line name by ID */
getLineName?: (id: string) => string | undefined;
/** Initial products to fetch suggestions for */
initialProducts?: RowData[];
/** Whether to auto-initialize (default: true) */
autoInitialize?: boolean;
}
// ============================================================================
// Context
// ============================================================================
const AiSuggestionsContext = createContext<AiSuggestionsContextValue | null>(null);
// Fields that affect embeddings
const EMBEDDING_FIELDS = ['company', 'line', 'name', 'description'];
const API_BASE = '/api/ai';
// ============================================================================
// Provider
// ============================================================================
export function AiSuggestionsProvider({
children,
getCompanyName,
getLineName,
initialProducts,
autoInitialize = true,
}: AiSuggestionsProviderProps) {
// State for initialization status (this can cause re-render, but it's rare)
const [isInitialized, setIsInitialized] = useState(false);
// Refs for data that shouldn't trigger re-renders
const suggestionsRef = useRef<Map<string, ProductSuggestions>>(new Map());
const loadingRef = useRef<Set<string>>(new Set());
const fieldValuesRef = useRef<Map<string, Record<string, unknown>>>(new Map());
const subscribersRef = useRef<Map<string, Set<() => void>>>(new Map());
// Ref for lookup functions (updated on each render)
const lookupFnsRef = useRef({ getCompanyName, getLineName });
lookupFnsRef.current = { getCompanyName, getLineName };
/**
* Notify subscribers when suggestions change
*/
const notifySubscribers = useCallback((productIndex: string) => {
const callbacks = subscribersRef.current.get(productIndex);
if (callbacks) {
callbacks.forEach(cb => cb());
}
}, []);
/**
* Initialize the AI service
*/
const initialize = useCallback(async (): Promise<boolean> => {
if (isInitialized) return true;
try {
const response = await fetch(`${API_BASE}/initialize`, { method: 'POST' });
const data = await response.json();
if (!response.ok || !data.success) {
console.error('[AiSuggestions] Initialization failed:', data.error);
return false;
}
setIsInitialized(true);
return true;
} catch (error) {
console.error('[AiSuggestions] Initialization error:', error);
return false;
}
}, [isInitialized]);
/**
* Build product data for API request
*/
const buildProductRequest = useCallback((product: RowData) => {
const { getCompanyName: getCompany, getLineName: getLine } = lookupFnsRef.current;
return {
name: product.name,
description: product.description,
company_name: product.company ? getCompany?.(String(product.company)) : undefined,
line_name: product.line ? getLine?.(String(product.line)) : undefined,
};
}, []);
/**
* Fetch suggestions for a single product
*/
const fetchSuggestions = useCallback(async (product: RowData) => {
const productIndex = product.__index;
if (!productIndex) return;
// Skip if already loading
if (loadingRef.current.has(productIndex)) return;
// Ensure initialized
const ready = await initialize();
if (!ready) return;
// Check if product has enough data for meaningful suggestions
const productData = buildProductRequest(product);
const hasText = productData.name || productData.description || productData.company_name;
if (!hasText) return;
// Mark as loading
loadingRef.current.add(productIndex);
notifySubscribers(productIndex);
try {
const response = await fetch(`${API_BASE}/suggestions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ product: productData }),
});
if (!response.ok) {
throw new Error(`API error: ${response.status}`);
}
const suggestions: ProductSuggestions = await response.json();
// Store suggestions
suggestionsRef.current.set(productIndex, suggestions);
// Store field values for change detection
fieldValuesRef.current.set(productIndex, {
company: product.company,
line: product.line,
name: product.name,
description: product.description,
});
} catch (error) {
console.error('[AiSuggestions] Fetch error:', error);
} finally {
loadingRef.current.delete(productIndex);
notifySubscribers(productIndex);
}
}, [initialize, buildProductRequest, notifySubscribers]);
/**
* Fetch suggestions for multiple products in batch
*/
const fetchBatchSuggestions = useCallback(async (products: RowData[]) => {
// Ensure initialized
const ready = await initialize();
if (!ready) return;
// Filter to products that need fetching
const productsToFetch = products.filter(p => {
if (!p.__index) return false;
if (loadingRef.current.has(p.__index)) return false;
if (suggestionsRef.current.has(p.__index)) return false;
const productData = buildProductRequest(p);
return productData.name || productData.description || productData.company_name;
});
if (productsToFetch.length === 0) return;
// Mark all as loading
productsToFetch.forEach(p => {
loadingRef.current.add(p.__index);
notifySubscribers(p.__index);
});
try {
const response = await fetch(`${API_BASE}/suggestions/batch`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
products: productsToFetch.map(p => ({
_index: p.__index,
...buildProductRequest(p),
})),
}),
});
if (!response.ok) {
throw new Error(`API error: ${response.status}`);
}
const data = await response.json();
// Store results
for (const result of data.results || []) {
const product = productsToFetch[result.index];
if (product?.__index) {
suggestionsRef.current.set(product.__index, {
categories: result.categories,
themes: result.themes,
colors: result.colors,
});
fieldValuesRef.current.set(product.__index, {
company: product.company,
line: product.line,
name: product.name,
description: product.description,
});
}
}
} catch (error) {
console.error('[AiSuggestions] Batch fetch error:', error);
} finally {
productsToFetch.forEach(p => {
loadingRef.current.delete(p.__index);
notifySubscribers(p.__index);
});
}
}, [initialize, buildProductRequest, notifySubscribers]);
/**
* Handle field blur - refresh suggestions if embedding field changed
*/
const handleFieldBlur = useCallback((product: RowData, fieldKey: string) => {
if (!EMBEDDING_FIELDS.includes(fieldKey)) return;
const productIndex = product.__index;
if (!productIndex) return;
// Check if value actually changed
const prevValues = fieldValuesRef.current.get(productIndex);
if (prevValues) {
const prevValue = prevValues[fieldKey];
const currentValue = product[fieldKey];
if (prevValue === currentValue) return;
}
// Clear existing suggestions and refetch
suggestionsRef.current.delete(productIndex);
fieldValuesRef.current.delete(productIndex);
// Debounce the fetch (simple timeout-based debounce)
setTimeout(() => fetchSuggestions(product), 300);
}, [fetchSuggestions]);
/**
* Get suggestions for a product
*/
const getSuggestions = useCallback((productIndex: string): ProductSuggestions | undefined => {
return suggestionsRef.current.get(productIndex);
}, []);
/**
* Check if loading
*/
const isLoading = useCallback((productIndex: string): boolean => {
return loadingRef.current.has(productIndex);
}, []);
/**
* Get category suggestions
*/
const getCategorySuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => {
return suggestionsRef.current.get(productIndex)?.categories || [];
}, []);
/**
* Get theme suggestions
*/
const getThemeSuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => {
return suggestionsRef.current.get(productIndex)?.themes || [];
}, []);
/**
* Get color suggestions
*/
const getColorSuggestions = useCallback((productIndex: string): TaxonomySuggestion[] => {
return suggestionsRef.current.get(productIndex)?.colors || [];
}, []);
/**
* Subscribe to suggestion changes
*/
const subscribe = useCallback((productIndex: string, callback: () => void): (() => void) => {
if (!subscribersRef.current.has(productIndex)) {
subscribersRef.current.set(productIndex, new Set());
}
subscribersRef.current.get(productIndex)!.add(callback);
return () => {
subscribersRef.current.get(productIndex)?.delete(callback);
};
}, []);
/**
* Refresh all products
*/
const refreshAll = useCallback(() => {
// Clear all cached suggestions
suggestionsRef.current.clear();
fieldValuesRef.current.clear();
// If we have initial products, refetch them
if (initialProducts && initialProducts.length > 0) {
fetchBatchSuggestions(initialProducts);
}
}, [initialProducts, fetchBatchSuggestions]);
// Auto-initialize and fetch initial products
useEffect(() => {
if (!autoInitialize) return;
const init = async () => {
const ready = await initialize();
if (ready && initialProducts && initialProducts.length > 0) {
// Small delay to avoid blocking initial render
setTimeout(() => fetchBatchSuggestions(initialProducts), 100);
}
};
init();
}, [autoInitialize, initialize, initialProducts, fetchBatchSuggestions]);
const contextValue: AiSuggestionsContextValue = {
isInitialized,
getSuggestions,
isLoading,
fetchSuggestions,
handleFieldBlur,
getCategorySuggestions,
getThemeSuggestions,
getColorSuggestions,
subscribe,
refreshAll,
};
return (
<AiSuggestionsContext.Provider value={contextValue}>
{children}
</AiSuggestionsContext.Provider>
);
}
// ============================================================================
// Hook
// ============================================================================
export function useAiSuggestionsContext(): AiSuggestionsContextValue | null {
return useContext(AiSuggestionsContext);
}
/**
* Hook for cells to get suggestions with re-render on update
* Only use this in cell components that need to display suggestions
*/
export function useCellSuggestions(productIndex: string) {
const context = useAiSuggestionsContext();
const [, forceUpdate] = useState({});
useEffect(() => {
if (!context) return;
// Subscribe to changes for this product
const unsubscribe = context.subscribe(productIndex, () => {
forceUpdate({});
});
return unsubscribe;
}, [context, productIndex]);
if (!context) {
return {
categories: [] as TaxonomySuggestion[],
themes: [] as TaxonomySuggestion[],
colors: [] as TaxonomySuggestion[],
isLoading: false,
};
}
return {
categories: context.getCategorySuggestions(productIndex),
themes: context.getThemeSuggestions(productIndex),
colors: context.getColorSuggestions(productIndex),
isLoading: context.isLoading(productIndex),
};
}
export default AiSuggestionsContext;

View File

@@ -0,0 +1,54 @@
/**
* useCopyDownValidation Hook
*
* Watches for copy-down operations on UPC-related fields (supplier, upc, barcode)
* and triggers UPC validation for affected rows using the existing validateUpc function.
*
* This avoids duplicating UPC validation logic - we reuse the same code path
* that handles individual cell blur events.
*/
import { useEffect } from 'react';
import { useValidationStore } from '../store/validationStore';
import { useUpcValidation } from './useUpcValidation';
/**
* Hook that handles UPC validation after copy-down operations.
* Should be called once in ValidationContainer to ensure validation runs.
*/
export const useCopyDownValidation = () => {
const { validateUpc } = useUpcValidation();
// Subscribe to pending copy-down validation
const pendingValidation = useValidationStore((state) => state.pendingCopyDownValidation);
const clearPendingCopyDownValidation = useValidationStore((state) => state.clearPendingCopyDownValidation);
useEffect(() => {
if (!pendingValidation) return;
const { fieldKey, affectedRows } = pendingValidation;
// Get current rows to check supplier and UPC values
const rows = useValidationStore.getState().rows;
// Process each affected row
const validationPromises = affectedRows.map(async (rowIndex) => {
const row = rows[rowIndex];
if (!row) return;
// Get supplier and UPC values
const supplierId = row.supplier ? String(row.supplier) : '';
const upcValue = row.upc ? String(row.upc) : (row.barcode ? String(row.barcode) : '');
// Only validate if we have both supplier and UPC
if (supplierId && upcValue) {
await validateUpc(rowIndex, supplierId, upcValue);
}
});
// Run all validations and then clear the pending state
Promise.all(validationPromises).then(() => {
clearPendingCopyDownValidation();
});
}, [pendingValidation, validateUpc, clearPendingCopyDownValidation]);
};

View File

@@ -151,6 +151,15 @@ export interface CopyDownState {
targetRowIndex: number | null; // Hover preview - which row the user is hovering on targetRowIndex: number | null; // Hover preview - which row the user is hovering on
} }
/**
* Tracks rows that need UPC validation after copy-down completes.
* This allows reusing the existing validateUpc logic instead of duplicating it.
*/
export interface PendingCopyDownValidation {
fieldKey: string;
affectedRows: number[];
}
// ============================================================================= // =============================================================================
// Dialog State Types // Dialog State Types
// ============================================================================= // =============================================================================
@@ -218,6 +227,37 @@ export interface AiValidationState {
revertedChanges: Set<string>; // Format: "productIndex:fieldKey" revertedChanges: Set<string>; // Format: "productIndex:fieldKey"
} }
// =============================================================================
// AI Suggestions Types (Embedding-based)
// =============================================================================
export interface TaxonomySuggestion {
id: number;
name: string;
fullPath?: string;
similarity: number;
}
export interface ProductSuggestions {
categories: TaxonomySuggestion[];
themes: TaxonomySuggestion[];
colors: TaxonomySuggestion[];
latencyMs?: number;
}
export interface AiSuggestionsState {
initialized: boolean;
initializing: boolean;
/** Map of product __index to their embedding */
embeddings: Map<string, number[]>;
/** Map of product __index to their suggestions */
suggestions: Map<string, ProductSuggestions>;
/** Products currently being processed */
processing: Set<string>;
/** Last error if any */
error: string | null;
}
// ============================================================================= // =============================================================================
// Initialization Types // Initialization Types
// ============================================================================= // =============================================================================
@@ -292,6 +332,7 @@ export interface ValidationState {
// === Copy-Down Mode === // === Copy-Down Mode ===
copyDownMode: CopyDownState; copyDownMode: CopyDownState;
pendingCopyDownValidation: PendingCopyDownValidation | null;
// === Dialogs === // === Dialogs ===
dialogs: DialogState; dialogs: DialogState;
@@ -376,6 +417,7 @@ export interface ValidationActions {
cancelCopyDown: () => void; cancelCopyDown: () => void;
completeCopyDown: (targetRowIndex: number) => void; completeCopyDown: (targetRowIndex: number) => void;
setTargetRowHover: (rowIndex: number | null) => void; setTargetRowHover: (rowIndex: number | null) => void;
clearPendingCopyDownValidation: () => void;
// === Dialogs === // === Dialogs ===
setDialogs: (updates: Partial<DialogState>) => void; setDialogs: (updates: Partial<DialogState>) => void;

View File

@@ -29,6 +29,7 @@ import type {
AiValidationResults, AiValidationResults,
CopyDownState, CopyDownState,
DialogState, DialogState,
PendingCopyDownValidation,
} from './types'; } from './types';
import type { Field, SelectOption } from '../../../types'; import type { Field, SelectOption } from '../../../types';
@@ -57,6 +58,9 @@ const initialCopyDownState: CopyDownState = {
targetRowIndex: null, targetRowIndex: null,
}; };
// Fields that require UPC validation when changed via copy-down
const UPC_VALIDATION_FIELDS = ['supplier', 'upc', 'barcode'];
const initialDialogState: DialogState = { const initialDialogState: DialogState = {
templateFormOpen: false, templateFormOpen: false,
templateFormData: null, templateFormData: null,
@@ -105,6 +109,7 @@ const getInitialState = (): ValidationState => ({
// Copy-Down Mode // Copy-Down Mode
copyDownMode: { ...initialCopyDownState }, copyDownMode: { ...initialCopyDownState },
pendingCopyDownValidation: null,
// Dialogs // Dialogs
dialogs: { ...initialDialogState }, dialogs: { ...initialDialogState },
@@ -574,9 +579,13 @@ export const useValidationStore = create<ValidationStore>()(
const hasValue = sourceValue !== null && sourceValue !== '' && const hasValue = sourceValue !== null && sourceValue !== '' &&
!(Array.isArray(sourceValue) && sourceValue.length === 0); !(Array.isArray(sourceValue) && sourceValue.length === 0);
// Track affected rows for UPC validation
const affectedRows: number[] = [];
for (let i = sourceRowIndex + 1; i <= targetRowIndex; i++) { for (let i = sourceRowIndex + 1; i <= targetRowIndex; i++) {
if (state.rows[i]) { if (state.rows[i]) {
state.rows[i][fieldKey] = cloneValue(sourceValue); state.rows[i][fieldKey] = cloneValue(sourceValue);
affectedRows.push(i);
// Clear validation errors for this field if value is non-empty // Clear validation errors for this field if value is non-empty
if (hasValue) { if (hasValue) {
@@ -596,6 +605,15 @@ export const useValidationStore = create<ValidationStore>()(
// Reset copy-down mode // Reset copy-down mode
state.copyDownMode = { ...initialCopyDownState }; state.copyDownMode = { ...initialCopyDownState };
// If this field affects UPC validation, store the affected rows
// so a hook can trigger validation using the existing validateUpc function
if (UPC_VALIDATION_FIELDS.includes(fieldKey) && affectedRows.length > 0) {
state.pendingCopyDownValidation = {
fieldKey,
affectedRows,
};
}
}); });
}, },
@@ -607,6 +625,12 @@ export const useValidationStore = create<ValidationStore>()(
}); });
}, },
clearPendingCopyDownValidation: () => {
set((state) => {
state.pendingCopyDownValidation = null;
});
},
// ========================================================================= // =========================================================================
// Dialogs // Dialogs
// ========================================================================= // =========================================================================