diff --git a/inventory-server/src/routes/ai-validation.js b/inventory-server/src/routes/ai-validation.js index d128e97..3920f33 100644 --- a/inventory-server/src/routes/ai-validation.js +++ b/inventory-server/src/routes/ai-validation.js @@ -1,387 +1,550 @@ -const express = require('express'); +const express = require("express"); const router = express.Router(); -const OpenAI = require('openai'); -const fs = require('fs').promises; -const path = require('path'); -const dotenv = require('dotenv'); +const OpenAI = require("openai"); +const fs = require("fs").promises; +const path = require("path"); +const dotenv = require("dotenv"); +const mysql = require('mysql2/promise'); +const { Client } = require('ssh2'); // Ensure environment variables are loaded -dotenv.config({ path: path.join(__dirname, '../../.env') }); +dotenv.config({ path: path.join(__dirname, "../../.env") }); const openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY + apiKey: process.env.OPENAI_API_KEY, }); if (!process.env.OPENAI_API_KEY) { - console.error('Warning: OPENAI_API_KEY is not set in environment variables'); + console.error("Warning: OPENAI_API_KEY is not set in environment variables"); } -// Cache configuration -const CACHE_TTL = 60 * 60 * 1000; // 1 hour in milliseconds - -// Cache structure with TTL -let cache = { - taxonomyData: null, - validationPrompt: null, - lastUpdated: null -}; - -// Function to check if cache is valid -function isCacheValid() { - return cache.lastUpdated && (Date.now() - cache.lastUpdated) < CACHE_TTL; -} - -// Function to clear cache -function clearCache() { - cache = { - taxonomyData: null, - validationPrompt: null, - lastUpdated: null +// Helper function to setup SSH tunnel to production database +async function setupSshTunnel() { + const sshConfig = { + host: process.env.PROD_SSH_HOST, + port: process.env.PROD_SSH_PORT || 22, + username: process.env.PROD_SSH_USER, + privateKey: process.env.PROD_SSH_KEY_PATH + ? require('fs').readFileSync(process.env.PROD_SSH_KEY_PATH) + : undefined, + compress: true }; + + const dbConfig = { + host: process.env.PROD_DB_HOST || 'localhost', + user: process.env.PROD_DB_USER, + password: process.env.PROD_DB_PASSWORD, + database: process.env.PROD_DB_NAME, + port: process.env.PROD_DB_PORT || 3306, + timezone: 'Z' + }; + + return new Promise((resolve, reject) => { + const ssh = new Client(); + + ssh.on('error', (err) => { + console.error('SSH connection error:', err); + reject(err); + }); + + ssh.on('ready', () => { + ssh.forwardOut( + '127.0.0.1', + 0, + dbConfig.host, + dbConfig.port, + (err, stream) => { + if (err) reject(err); + resolve({ ssh, stream, dbConfig }); + } + ); + }).connect(sshConfig); + }); } -// Debug endpoint for viewing prompt and cache status -router.post('/debug', async (req, res) => { +// Debug endpoint for viewing prompt +router.post("/debug", async (req, res) => { try { - console.log('Debug POST endpoint called'); - const pool = req.app.locals.pool; - const { products } = req.body; + console.log("Debug POST endpoint called"); - console.log('Received products:', { + const { products } = req.body; + + console.log("Received products for debug:", { isArray: Array.isArray(products), length: products?.length, firstProduct: products?.[0], - lastProduct: products?.[products?.length - 1] + lastProduct: products?.[products?.length - 1], }); - + if (!Array.isArray(products)) { - console.error('Invalid input: products is not an array'); - return res.status(400).json({ error: 'Products must be an array' }); + console.error("Invalid input: products is not an array"); + return res.status(400).json({ error: "Products must be an array" }); } if (products.length === 0) { - console.error('Invalid input: products array is empty'); - return res.status(400).json({ error: 'Products array cannot be empty' }); + console.error("Invalid input: products array is empty"); + return res.status(400).json({ error: "Products array cannot be empty" }); } - + // Clean the products array to remove any internal fields - const cleanedProducts = products.map(product => { + const cleanedProducts = products.map((product) => { const { __errors, __index, ...cleanProduct } = product; return cleanProduct; }); - - return await generateDebugResponse(pool, cleanedProducts, res); + + console.log("Processing debug request with cleaned products:", { + length: cleanedProducts.length, + sample: cleanedProducts[0], + }); + + try { + return await generateDebugResponse(cleanedProducts, res); + } catch (generateError) { + console.error("Error generating debug response:", generateError); + return res.status(500).json({ + error: "Error generating debug response: " + generateError.message, + stack: generateError.stack, + name: generateError.name, + code: generateError.code, + sqlMessage: generateError.sqlMessage, + }); + } } catch (error) { - console.error('Debug POST endpoint error:', error); - res.status(500).json({ error: error.message }); + console.error("Debug POST endpoint error:", error); + res.status(500).json({ + error: error.message, + stack: error.stack, + code: error.code || null, + name: error.name || null + }); } }); // Helper function to generate debug response -async function generateDebugResponse(pool, productsToUse, res) { - // Load taxonomy data first - console.log('Loading taxonomy data...'); - const taxonomy = await getTaxonomyData(pool); - console.log('Taxonomy data loaded:', { - categoriesCount: taxonomy.categories.length, - themesCount: taxonomy.themes.length, - colorsCount: taxonomy.colors.length, - taxCodesCount: taxonomy.taxCodes.length, - sizeCategoriesCount: taxonomy.sizeCategories.length, - suppliersCount: taxonomy.suppliers.length, - companiesCount: taxonomy.companies.length, - artistsCount: taxonomy.artists.length - }); +async function generateDebugResponse(productsToUse, res) { + let taxonomy = null; + let mysqlConnection = null; + let ssh = null; + + try { + // Load taxonomy data first + console.log("Loading taxonomy data..."); + try { + // Setup MySQL connection via SSH tunnel + const tunnel = await setupSshTunnel(); + ssh = tunnel.ssh; + + mysqlConnection = await mysql.createConnection({ + ...tunnel.dbConfig, + stream: tunnel.stream + }); + + console.log("MySQL connection established successfully"); + + taxonomy = await getTaxonomyData(mysqlConnection); + console.log("Successfully loaded taxonomy data"); + } catch (taxonomyError) { + console.error("Failed to load taxonomy data:", taxonomyError); + return res.status(500).json({ + error: "Error fetching taxonomy data: " + taxonomyError.message, + sqlMessage: taxonomyError.sqlMessage || null, + sqlState: taxonomyError.sqlState || null, + code: taxonomyError.code || null, + errno: taxonomyError.errno || null, + sql: taxonomyError.sql || null, + }); + } finally { + // Make sure we close the connection + if (mysqlConnection) await mysqlConnection.end(); + if (ssh) ssh.end(); + } + + // Verify the taxonomy data structure + console.log("Verifying taxonomy structure..."); + if (!taxonomy) { + console.error("Taxonomy data is null"); + return res.status(500).json({ error: "Taxonomy data is null" }); + } + + // Check if each taxonomy component exists + const taxonomyComponents = [ + "categories", "themes", "colors", "taxCodes", "sizeCategories", + "suppliers", "companies", "artists", "lines", "subLines" + ]; + + const missingComponents = taxonomyComponents.filter(comp => !taxonomy[comp]); + if (missingComponents.length > 0) { + console.error("Missing taxonomy components:", missingComponents); + } + + // Log detailed taxonomy stats for debugging + console.log("Taxonomy data loaded with details:", { + categories: { + length: taxonomy.categories?.length || 0, + sample: taxonomy.categories?.length > 0 ? JSON.stringify(taxonomy.categories[0]).substring(0, 100) + "..." : null + }, + themes: { + length: taxonomy.themes?.length || 0, + sample: taxonomy.themes?.length > 0 ? JSON.stringify(taxonomy.themes[0]).substring(0, 100) + "..." : null + }, + colors: { + length: taxonomy.colors?.length || 0, + sample: taxonomy.colors?.length > 0 ? JSON.stringify(taxonomy.colors[0]) : null + }, + taxCodes: { + length: taxonomy.taxCodes?.length || 0, + sample: taxonomy.taxCodes?.length > 0 ? JSON.stringify(taxonomy.taxCodes[0]) : null + }, + sizeCategories: { + length: taxonomy.sizeCategories?.length || 0, + sample: taxonomy.sizeCategories?.length > 0 ? JSON.stringify(taxonomy.sizeCategories[0]) : null + }, + suppliers: { + length: taxonomy.suppliers?.length || 0, + sample: taxonomy.suppliers?.length > 0 ? JSON.stringify(taxonomy.suppliers[0]) : null + }, + companies: { + length: taxonomy.companies?.length || 0, + sample: taxonomy.companies?.length > 0 ? JSON.stringify(taxonomy.companies[0]) : null + }, + artists: { + length: taxonomy.artists?.length || 0, + sample: taxonomy.artists?.length > 0 ? JSON.stringify(taxonomy.artists[0]) : null + } + }); - // Load the prompt using the same function used by validation - console.log('Loading prompt...'); - const prompt = await loadPrompt(pool, productsToUse); - const fullPrompt = prompt + '\n' + JSON.stringify(productsToUse); + // Load the prompt using the same function used by validation + console.log("Loading prompt..."); + + // Setup a new connection for loading the prompt + const promptTunnel = await setupSshTunnel(); + const promptConnection = await mysql.createConnection({ + ...promptTunnel.dbConfig, + stream: promptTunnel.stream + }); + + try { + const prompt = await loadPrompt(promptConnection, productsToUse); + const fullPrompt = prompt + "\n" + JSON.stringify(productsToUse); - const response = { - cacheStatus: { - isCacheValid: isCacheValid(), - lastUpdated: cache.lastUpdated ? new Date(cache.lastUpdated).toISOString() : null, - timeUntilExpiry: cache.lastUpdated ? - Math.max(0, CACHE_TTL - (Date.now() - cache.lastUpdated)) / 1000 + ' seconds' : - 'expired', - }, - taxonomyStats: taxonomy ? { - categories: countItems(taxonomy.categories), - themes: taxonomy.themes.length, - colors: taxonomy.colors.length, - taxCodes: taxonomy.taxCodes.length, - sizeCategories: taxonomy.sizeCategories.length, - suppliers: taxonomy.suppliers.length, - companies: taxonomy.companies.length, - artists: taxonomy.artists.length, - // Add filtered counts when products are provided - filtered: productsToUse ? { - suppliers: taxonomy.suppliers.filter(([id]) => - productsToUse.some(p => Number(p.supplierid) === Number(id))).length, - companies: taxonomy.companies.filter(([id]) => - productsToUse.some(p => Number(p.company) === Number(id))).length, - artists: taxonomy.artists.filter(([id]) => - productsToUse.some(p => Number(p.artist) === Number(id))).length - } : null - } : null, - basePrompt: prompt, - sampleFullPrompt: fullPrompt, - promptLength: fullPrompt.length, - }; + // Create the response with taxonomy stats + let categoriesCount = 0; + try { + categoriesCount = taxonomy?.categories?.length ? countItems(taxonomy.categories) : 0; + } catch (countError) { + console.error("Error counting categories:", countError); + categoriesCount = taxonomy?.categories?.length || 0; // Fallback to simple length + } + + const response = { + taxonomyStats: taxonomy + ? { + categories: categoriesCount, + themes: taxonomy.themes?.length || 0, + colors: taxonomy.colors?.length || 0, + taxCodes: taxonomy.taxCodes?.length || 0, + sizeCategories: taxonomy.sizeCategories?.length || 0, + suppliers: taxonomy.suppliers?.length || 0, + companies: taxonomy.companies?.length || 0, + artists: taxonomy.artists?.length || 0, + // Add filtered counts when products are provided + filtered: productsToUse + ? { + suppliers: taxonomy.suppliers?.filter(([id]) => + productsToUse.some( + (p) => Number(p.supplierid) === Number(id) + ) + )?.length || 0, + companies: taxonomy.companies?.filter(([id]) => + productsToUse.some((p) => Number(p.company) === Number(id)) + )?.length || 0, + artists: taxonomy.artists?.filter(([id]) => + productsToUse.some((p) => Number(p.artist) === Number(id)) + )?.length || 0, + } + : null, + } + : null, + basePrompt: prompt, + sampleFullPrompt: fullPrompt, + promptLength: fullPrompt.length, + }; - console.log('Sending response with stats:', response.taxonomyStats); - return res.json(response); + console.log("Sending response with taxonomy stats:", response.taxonomyStats); + return res.json(response); + } finally { + if (promptConnection) await promptConnection.end(); + if (promptTunnel.ssh) promptTunnel.ssh.end(); + } + } catch (error) { + console.error("Error generating debug response:", error); + return res.status(500).json({ + error: error.message, + stack: error.stack, + sqlMessage: error.sqlMessage || null, + sqlState: error.sqlState || null, + code: error.code || null, + errno: error.errno || null, + taxonomyState: taxonomy ? "loaded" : "failed", + }); + } } // Helper function to count total items in hierarchical structure function countItems(items) { return items.reduce((count, item) => { - return count + 1 + (item.subcategories ? countItems(item.subcategories) : 0); + return ( + count + 1 + (item.subcategories ? countItems(item.subcategories) : 0) + ); }, 0); } -// Force cache refresh endpoint -router.post('/refresh-cache', async (req, res) => { - try { - clearCache(); - const pool = req.app.locals.pool; - await loadPrompt(pool); // This will rebuild the cache - res.json({ - success: true, - message: 'Cache refreshed successfully', - newCacheTime: new Date(cache.lastUpdated).toISOString() - }); - } catch (error) { - console.error('Cache refresh error:', error); - res.status(500).json({ error: error.message }); - } -}); - // Function to fetch and format taxonomy data -async function getTaxonomyData(pool) { - if (cache.taxonomyData && isCacheValid()) { - return cache.taxonomyData; +async function getTaxonomyData(connection) { + try { + console.log("Starting taxonomy data fetch..."); + // Fetch categories with hierarchy + const [categories] = await connection.query(` +SELECT cat_id,name,NULL AS master_cat_id,1 AS level_order FROM product_categories s WHERE type=10 UNION ALL SELECT c.cat_id,c.name,c.master_cat_id,2 AS level_order FROM product_categories c JOIN product_categories s ON c.master_cat_id=s.cat_id WHERE c.type=11 AND s.type=10 UNION ALL SELECT sc.cat_id,sc.name,sc.master_cat_id,3 AS level_order FROM product_categories sc JOIN product_categories c ON sc.master_cat_id=c.cat_id JOIN product_categories s ON c.master_cat_id=s.cat_id WHERE sc.type=12 AND c.type=11 AND s.type=10 UNION ALL SELECT ssc.cat_id,ssc.name,ssc.master_cat_id,4 AS level_order FROM product_categories ssc JOIN product_categories sc ON ssc.master_cat_id=sc.cat_id JOIN product_categories c ON sc.master_cat_id=c.cat_id JOIN product_categories s ON c.master_cat_id=s.cat_id WHERE ssc.type=13 AND sc.type=12 AND c.type=11 AND s.type=10 ORDER BY level_order,cat_id; + `); + console.log("Categories fetched:", categories.length); + + // Fetch themes with hierarchy + const [themes] = await connection.query(` +SELECT t.cat_id,t.name,null as master_cat_id,1 AS level_order FROM product_categories t WHERE t.type=20 UNION ALL SELECT ts.cat_id,ts.name,ts.master_cat_id,2 AS level_order FROM product_categories ts JOIN product_categories t ON ts.master_cat_id=t.cat_id WHERE ts.type=21 AND t.type=20 ORDER BY level_order,name + `); + console.log("Themes fetched:", themes.length); + + // Fetch colors + const [colors] = await connection.query( + `SELECT color, name, hex_color FROM product_color_list ORDER BY \`order\`` + ); + console.log("Colors fetched:", colors.length); + + // Fetch tax codes + const [taxCodes] = await connection.query( + `SELECT tax_code_id, name FROM product_tax_codes ORDER BY name` + ); + console.log("Tax codes fetched:", taxCodes.length); + + // Fetch size categories + const [sizeCategories] = await connection.query( + `SELECT cat_id, name FROM product_categories WHERE type=50 ORDER BY name` + ); + console.log("Size categories fetched:", sizeCategories.length); + + // Fetch suppliers + const [suppliers] = await connection.query(` + SELECT supplierid, companyname as name + FROM suppliers + WHERE companyname <> '' + ORDER BY companyname + `); + console.log("Suppliers fetched:", suppliers.length); + + // Fetch companies (type 1) + const [companies] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 1 + ORDER BY name + `); + console.log("Companies fetched:", companies.length); + + // Fetch artists (type 40) + const [artists] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 40 + ORDER BY name + `); + console.log("Artists fetched:", artists.length); + + // Fetch lines (type 2) + const [lines] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 2 + ORDER BY name + `); + console.log("Lines fetched:", lines.length); + + // Fetch sub-lines (type 3) + const [subLines] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 3 + ORDER BY name + `); + console.log("Sub-lines fetched:", subLines.length); + + // Format categories into a hierarchical structure + const formatHierarchy = (items, level = 1, parentId = null) => { + return items + .filter( + (item) => + item.level_order === level && item.master_cat_id === parentId + ) + .map((item) => { + const children = formatHierarchy(items, level + 1, item.cat_id); + return children.length > 0 + ? [item.cat_id, item.name, children] + : [item.cat_id, item.name]; + }); + }; + + // Format themes similarly but with only two levels + const formatThemes = (items) => { + return items + .filter((item) => item.level_order === 1) + .map((item) => { + const subthemes = items + .filter((subitem) => subitem.master_cat_id === item.cat_id) + .map((subitem) => [subitem.cat_id, subitem.name]); + return subthemes.length > 0 + ? [item.cat_id, item.name, subthemes] + : [item.cat_id, item.name]; + }); + }; + + // Log first item of each taxonomy category to check structure + console.log("Sample category:", categories.length > 0 ? categories[0] : "No categories"); + console.log("Sample theme:", themes.length > 0 ? themes[0] : "No themes"); + console.log("Sample color:", colors.length > 0 ? colors[0] : "No colors"); + + const formattedData = { + categories: formatHierarchy(categories), + themes: formatThemes(themes), + colors: colors.map((c) => [c.color, c.name, c.hex_color]), + taxCodes: (taxCodes || []).map((tc) => [tc.tax_code_id, tc.name]), + sizeCategories: (sizeCategories || []).map((sc) => [sc.cat_id, sc.name]), + suppliers: suppliers.map((s) => [s.supplierid, s.name]), + companies: companies.map((c) => [c.cat_id, c.name]), + artists: artists.map((a) => [a.cat_id, a.name]), + lines: lines.map((l) => [l.cat_id, l.name]), + subLines: subLines.map((sl) => [sl.cat_id, sl.name]), + }; + + // Check the formatted structure + console.log("Formatted categories count:", formattedData.categories.length); + console.log("Formatted themes count:", formattedData.themes.length); + console.log("Formatted colors count:", formattedData.colors.length); + + return formattedData; + } catch (error) { + console.error("Error fetching taxonomy data:", error); + console.error("Full error details:", { + message: error.message, + stack: error.stack, + code: error.code, + errno: error.errno, + sqlMessage: error.sqlMessage, + sqlState: error.sqlState, + sql: error.sql + }); + + // Instead of silently returning empty arrays, throw the error to be handled by the caller + throw error; } - - // Fetch categories with hierarchy - const [categories] = await pool.query(` - SELECT cat_id, name, master_cat_id, level_order - FROM ( - SELECT cat_id,name,NULL AS master_cat_id,1 AS level_order - FROM product_categories s - WHERE type=10 - UNION ALL - SELECT c.cat_id,c.name,c.master_cat_id,2 AS level_order - FROM product_categories c - JOIN product_categories s ON c.master_cat_id=s.cat_id - WHERE c.type=11 AND s.type=10 - UNION ALL - SELECT sc.cat_id,sc.name,sc.master_cat_id,3 AS level_order - FROM product_categories sc - JOIN product_categories c ON sc.master_cat_id=c.cat_id - JOIN product_categories s ON c.master_cat_id=s.cat_id - WHERE sc.type=12 AND c.type=11 AND s.type=10 - UNION ALL - SELECT ssc.cat_id,ssc.name,ssc.master_cat_id,4 AS level_order - FROM product_categories ssc - JOIN product_categories sc ON ssc.master_cat_id=sc.cat_id - JOIN product_categories c ON sc.master_cat_id=c.cat_id - JOIN product_categories s ON c.master_cat_id=s.cat_id - WHERE ssc.type=13 AND sc.type=12 AND c.type=11 AND s.type=10 - ) AS hierarchy - ORDER BY level_order,cat_id - `); - - // Fetch themes with hierarchy - const [themes] = await pool.query(` - SELECT cat_id, name, master_cat_id, level_order - FROM ( - SELECT t.cat_id,t.name,null as master_cat_id,1 AS level_order - FROM product_categories t - WHERE t.type=20 - UNION ALL - SELECT ts.cat_id,ts.name,ts.master_cat_id,2 AS level_order - FROM product_categories ts - JOIN product_categories t ON ts.master_cat_id=t.cat_id - WHERE ts.type=21 AND t.type=20 - ) AS hierarchy - ORDER BY level_order,name - `); - - // Fetch colors - const [colors] = await pool.query('SELECT color, name FROM product_color_list ORDER BY name'); - - // Fetch tax codes - const [taxCodes] = await pool.query('SELECT tax_code_id, name FROM product_tax_codes ORDER BY name'); - - // Fetch size categories - const [sizeCategories] = await pool.query('SELECT cat_id, name FROM product_categories WHERE type=50 ORDER BY name'); - - // Fetch suppliers - const [suppliers] = await pool.query(` - SELECT supplierid, companyname as name - FROM suppliers - WHERE companyname <> '' - ORDER BY companyname - `); - - // Fetch companies (type 1) - const [companies] = await pool.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 1 - ORDER BY name - `); - - // Fetch artists (type 40) - const [artists] = await pool.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 40 - ORDER BY name - `); - - // Fetch lines (type 2) - const [lines] = await pool.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 2 - ORDER BY name - `); - - // Fetch sub-lines (type 3) - const [subLines] = await pool.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 3 - ORDER BY name - `); - - // Format categories into a hierarchical structure - const formatHierarchy = (items, level = 1, parentId = null) => { - return items - .filter(item => item.level_order === level && item.master_cat_id === parentId) - .map(item => { - const children = formatHierarchy(items, level + 1, item.cat_id); - return children.length > 0 ? - [item.cat_id, item.name, children] : - [item.cat_id, item.name]; - }); - }; - - // Format themes similarly but with only two levels - const formatThemes = (items) => { - return items - .filter(item => item.level_order === 1) - .map(item => { - const subthemes = items - .filter(subitem => subitem.master_cat_id === item.cat_id) - .map(subitem => [subitem.cat_id, subitem.name]); - return subthemes.length > 0 ? - [item.cat_id, item.name, subthemes] : - [item.cat_id, item.name]; - }); - }; - - cache.taxonomyData = { - categories: formatHierarchy(categories), - themes: formatThemes(themes), - colors: colors.map(c => [c.color, c.name]), - taxCodes: (taxCodes || []).map(tc => [tc.tax_code_id, tc.name]), - sizeCategories: (sizeCategories || []).map(sc => [sc.cat_id, sc.name]), - suppliers: suppliers.map(s => [s.supplierid, s.name]), - companies: companies.map(c => [c.cat_id, c.name]), - artists: artists.map(a => [a.cat_id, a.name]), - lines: lines.map(l => [l.cat_id, l.name]), - subLines: subLines.map(sl => [sl.cat_id, sl.name]) - }; - cache.lastUpdated = Date.now(); - - return cache.taxonomyData; } // Load the prompt from file and inject taxonomy data -async function loadPrompt(pool, productsToValidate = null) { - const promptPath = path.join(__dirname, '..', 'prompts', 'product-validation.txt'); - const basePrompt = await fs.readFile(promptPath, 'utf8'); - - // Get taxonomy data - const taxonomy = await getTaxonomyData(pool); +async function loadPrompt(connection, productsToValidate = null) { + try { + const promptPath = path.join( + __dirname, + "..", + "prompts", + "product-validation.txt" + ); + const basePrompt = await fs.readFile(promptPath, "utf8"); - // Add system instructions to the prompt - const systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. + // Get taxonomy data using the provided MySQL connection + const taxonomy = await getTaxonomyData(connection); + + // Add system instructions to the prompt + const systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. `; - // If we have products to validate, create a filtered prompt - if (productsToValidate) { - console.log('Creating filtered prompt for products:', productsToValidate); - - // Extract unique values from products for non-core attributes - const uniqueValues = { - supplierIds: new Set(), - companyIds: new Set(), - artistIds: new Set(), - lineIds: new Set(), - subLineIds: new Set() - }; + // If we have products to validate, create a filtered prompt + if (productsToValidate) { + console.log("Creating filtered prompt for products:", productsToValidate); - // Collect any values that exist in the products - productsToValidate.forEach(product => { - Object.entries(product).forEach(([key, value]) => { - if (value === undefined || value === null) return; - - // Map field names to their respective sets - const fieldMap = { - supplierid: 'supplierIds', - supplier: 'supplierIds', - company: 'companyIds', - artist: 'artistIds', - line: 'lineIds', - subline: 'subLineIds' - }; - - if (fieldMap[key]) { - uniqueValues[fieldMap[key]].add(Number(value)); - } + // Extract unique values from products for non-core attributes + const uniqueValues = { + supplierIds: new Set(), + companyIds: new Set(), + artistIds: new Set(), + lineIds: new Set(), + subLineIds: new Set(), + }; + + // Collect any values that exist in the products + productsToValidate.forEach((product) => { + Object.entries(product).forEach(([key, value]) => { + if (value === undefined || value === null) return; + + // Map field names to their respective sets + const fieldMap = { + supplierid: "supplierIds", + supplier: "supplierIds", + company: "companyIds", + artist: "artistIds", + line: "lineIds", + subline: "subLineIds", + }; + + if (fieldMap[key]) { + uniqueValues[fieldMap[key]].add(Number(value)); + } + }); }); - }); - console.log('Unique values collected:', { - suppliers: Array.from(uniqueValues.supplierIds), - companies: Array.from(uniqueValues.companyIds), - artists: Array.from(uniqueValues.artistIds), - lines: Array.from(uniqueValues.lineIds), - subLines: Array.from(uniqueValues.subLineIds) - }); + console.log("Unique values collected:", { + suppliers: Array.from(uniqueValues.supplierIds), + companies: Array.from(uniqueValues.companyIds), + artists: Array.from(uniqueValues.artistIds), + lines: Array.from(uniqueValues.lineIds), + subLines: Array.from(uniqueValues.subLineIds), + }); - // Create mixed taxonomy with filtered non-core data and full core data - const mixedTaxonomy = { - // Keep full data for core attributes - categories: taxonomy.categories, - themes: taxonomy.themes, - colors: taxonomy.colors, - taxCodes: taxonomy.taxCodes, - sizeCategories: taxonomy.sizeCategories, - // For non-core data, only include items that are actually used - suppliers: taxonomy.suppliers.filter(([id]) => uniqueValues.supplierIds.has(Number(id))), - companies: taxonomy.companies.filter(([id]) => uniqueValues.companyIds.has(Number(id))), - artists: taxonomy.artists.filter(([id]) => uniqueValues.artistIds.has(Number(id))), - lines: taxonomy.lines.filter(([id]) => uniqueValues.lineIds.has(Number(id))), - subLines: taxonomy.subLines.filter(([id]) => uniqueValues.subLineIds.has(Number(id))) - }; + // Create mixed taxonomy with filtered non-core data and full core data + const mixedTaxonomy = { + // Keep full data for core attributes + categories: taxonomy.categories, + themes: taxonomy.themes, + colors: taxonomy.colors, + taxCodes: taxonomy.taxCodes, + sizeCategories: taxonomy.sizeCategories, + // For non-core data, only include items that are actually used + suppliers: taxonomy.suppliers.filter(([id]) => + uniqueValues.supplierIds.has(Number(id)) + ), + companies: taxonomy.companies.filter(([id]) => + uniqueValues.companyIds.has(Number(id)) + ), + artists: taxonomy.artists.filter(([id]) => + uniqueValues.artistIds.has(Number(id)) + ), + lines: taxonomy.lines.filter(([id]) => + uniqueValues.lineIds.has(Number(id)) + ), + subLines: taxonomy.subLines.filter(([id]) => + uniqueValues.subLineIds.has(Number(id)) + ), + }; - console.log('Filtered taxonomy counts:', { - suppliers: mixedTaxonomy.suppliers.length, - companies: mixedTaxonomy.companies.length, - artists: mixedTaxonomy.artists.length, - lines: mixedTaxonomy.lines.length, - subLines: mixedTaxonomy.subLines.length - }); + console.log("Filtered taxonomy counts:", { + suppliers: mixedTaxonomy.suppliers.length, + companies: mixedTaxonomy.companies.length, + artists: mixedTaxonomy.artists.length, + lines: mixedTaxonomy.lines.length, + subLines: mixedTaxonomy.subLines.length, + }); - // Format taxonomy data for the prompt, only including sections with values - const taxonomySection = ` + // Format taxonomy data for the prompt, only including sections with values + const taxonomySection = ` All Available Categories: ${JSON.stringify(mixedTaxonomy.categories)} @@ -395,21 +558,46 @@ All Available Tax Codes: ${JSON.stringify(mixedTaxonomy.taxCodes)} All Available Size Categories: -${JSON.stringify(mixedTaxonomy.sizeCategories)}${mixedTaxonomy.suppliers.length ? `\n\nSuppliers Used In This Data:\n${JSON.stringify(mixedTaxonomy.suppliers)}` : ''}${mixedTaxonomy.companies.length ? `\n\nCompanies Used In This Data:\n${JSON.stringify(mixedTaxonomy.companies)}` : ''}${mixedTaxonomy.artists.length ? `\n\nArtists Used In This Data:\n${JSON.stringify(mixedTaxonomy.artists)}` : ''}${mixedTaxonomy.lines.length ? `\n\nLines Used In This Data:\n${JSON.stringify(mixedTaxonomy.lines)}` : ''}${mixedTaxonomy.subLines.length ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify(mixedTaxonomy.subLines)}` : ''} +${JSON.stringify(mixedTaxonomy.sizeCategories)}${ + mixedTaxonomy.suppliers.length + ? `\n\nSuppliers Used In This Data:\n${JSON.stringify( + mixedTaxonomy.suppliers + )}` + : "" + }${ + mixedTaxonomy.companies.length + ? `\n\nCompanies Used In This Data:\n${JSON.stringify( + mixedTaxonomy.companies + )}` + : "" + }${ + mixedTaxonomy.artists.length + ? `\n\nArtists Used In This Data:\n${JSON.stringify( + mixedTaxonomy.artists + )}` + : "" + }${ + mixedTaxonomy.lines.length + ? `\n\nLines Used In This Data:\n${JSON.stringify( + mixedTaxonomy.lines + )}` + : "" + }${ + mixedTaxonomy.subLines.length + ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify( + mixedTaxonomy.subLines + )}` + : "" + } ----------Here is the product data to validate----------`; - // Return the filtered prompt without caching - return systemInstructions + basePrompt + '\n' + taxonomySection; - } + // Return the filtered prompt + return systemInstructions + basePrompt + "\n" + taxonomySection; + } - // For debug/display purposes, if no products provided and cache is valid, return cached prompt - if (!productsToValidate && cache.validationPrompt && isCacheValid()) { - return cache.validationPrompt; - } - - // Generate and cache the full unfiltered prompt - const taxonomySection = ` + // Generate the full unfiltered prompt + const taxonomySection = ` Available Categories: ${JSON.stringify(taxonomy.categories)} @@ -434,93 +622,238 @@ ${JSON.stringify(taxonomy.companies)} Available Artists: ${JSON.stringify(taxonomy.artists)} -Available Shipping Restrictions: -${JSON.stringify(taxonomy.shippingRestrictions)} - Here is the product data to validate:`; - // Cache the full prompt only when no specific products are provided - cache.validationPrompt = systemInstructions + basePrompt + '\n' + taxonomySection; - cache.lastUpdated = Date.now(); - - return cache.validationPrompt; + return systemInstructions + basePrompt + "\n" + taxonomySection; + } catch (error) { + console.error("Error loading prompt:", error); + throw error; // Re-throw to be handled by the calling function + } } -// Set up cache clearing interval -setInterval(clearCache, CACHE_TTL); - -router.post('/validate', async (req, res) => { +router.post("/validate", async (req, res) => { try { const { products } = req.body; - console.log('🔍 Received products for validation:', JSON.stringify(products, null, 2)); + + console.log("🔍 Received products for validation:", { + isArray: Array.isArray(products), + length: products?.length, + firstProduct: products?.[0], + lastProduct: products?.[products?.length - 1], + }); if (!Array.isArray(products)) { - console.error('❌ Invalid input: products is not an array'); - return res.status(400).json({ error: 'Products must be an array' }); + console.error("❌ Invalid input: products is not an array"); + return res.status(400).json({ error: "Products must be an array" }); } - // Load the prompt with the products data to filter taxonomy - const prompt = await loadPrompt(req.app.locals.pool, products); - const fullPrompt = prompt + '\n' + JSON.stringify(products); - console.log('📝 Generated prompt:', fullPrompt); + if (products.length === 0) { + console.error("❌ Invalid input: products array is empty"); + return res.status(400).json({ error: "Products array cannot be empty" }); + } - console.log('🤖 Sending request to OpenAI...'); - const completion = await openai.chat.completions.create({ - model: "gpt-4o", - messages: [ - { - role: "user", - content: fullPrompt - } - ], - temperature: 0.2, - response_format: { type: "json_object" } - }); - - console.log('✅ Received response from OpenAI'); - const rawResponse = completion.choices[0].message.content; - console.log('📄 Raw AI response:', rawResponse); - - const aiResponse = JSON.parse(rawResponse); - console.log('🔄 Parsed AI response:', JSON.stringify(aiResponse, null, 2)); - - // Compare original and corrected data - if (aiResponse.correctedData) { - console.log('📊 Changes summary:'); - products.forEach((original, index) => { - const corrected = aiResponse.correctedData[index]; - if (corrected) { - const changes = Object.keys(corrected).filter(key => - JSON.stringify(original[key]) !== JSON.stringify(corrected[key]) - ); - if (changes.length > 0) { - console.log(`\nProduct ${index + 1} changes:`); - changes.forEach(key => { - console.log(` ${key}:`); - console.log(` - Original: ${JSON.stringify(original[key])}`); - console.log(` - Corrected: ${JSON.stringify(corrected[key])}`); - }); - } - } + let ssh = null; + let connection = null; + + try { + // Setup MySQL connection via SSH tunnel + console.log("🔄 Setting up connection to production database..."); + const tunnel = await setupSshTunnel(); + ssh = tunnel.ssh; + + connection = await mysql.createConnection({ + ...tunnel.dbConfig, + stream: tunnel.stream }); - } + + console.log("🔄 MySQL connection established successfully"); - res.json({ - success: true, - ...aiResponse - }); + // Load the prompt with the products data to filter taxonomy + console.log("🔄 Loading prompt with filtered taxonomy..."); + const prompt = await loadPrompt(connection, products); + const fullPrompt = prompt + "\n" + JSON.stringify(products); + console.log("📝 Generated prompt length:", fullPrompt.length); + + console.log("🤖 Sending request to OpenAI..."); + const completion = await openai.chat.completions.create({ + model: "gpt-4o", + messages: [ + { + role: "user", + content: fullPrompt, + }, + ], + temperature: 0.2, + response_format: { type: "json_object" }, + }); + + console.log("✅ Received response from OpenAI"); + const rawResponse = completion.choices[0].message.content; + console.log("📄 Raw AI response length:", rawResponse.length); + + try { + const aiResponse = JSON.parse(rawResponse); + console.log( + "🔄 Parsed AI response with keys:", + Object.keys(aiResponse) + ); + + // Compare original and corrected data + if (aiResponse.correctedData) { + console.log("📊 Changes summary:"); + products.forEach((original, index) => { + const corrected = aiResponse.correctedData[index]; + if (corrected) { + const changes = Object.keys(corrected).filter( + (key) => + JSON.stringify(original[key]) !== + JSON.stringify(corrected[key]) + ); + if (changes.length > 0) { + console.log(`\nProduct ${index + 1} changes:`); + changes.forEach((key) => { + console.log(` ${key}:`); + console.log( + ` - Original: ${JSON.stringify(original[key])}` + ); + console.log( + ` - Corrected: ${JSON.stringify(corrected[key])}` + ); + }); + } + } + }); + } + + res.json({ + success: true, + ...aiResponse, + }); + } catch (parseError) { + console.error("❌ Error parsing AI response:", parseError); + console.error("Raw response that failed to parse:", rawResponse); + res.status(500).json({ + success: false, + error: "Error parsing AI response: " + parseError.message, + }); + } + } catch (openaiError) { + console.error("❌ OpenAI API Error:", openaiError); + res.status(500).json({ + success: false, + error: "OpenAI API Error: " + openaiError.message, + }); + } finally { + // Clean up database connection and SSH tunnel + if (connection) await connection.end(); + if (ssh) ssh.end(); + } } catch (error) { - console.error('❌ AI Validation Error:', error); - console.error('Error details:', { + console.error("❌ AI Validation Error:", error); + console.error("Error details:", { name: error.name, message: error.message, - stack: error.stack + stack: error.stack, }); res.status(500).json({ success: false, - error: error.message || 'Error during AI validation' + error: error.message || "Error during AI validation", }); } }); -module.exports = router; \ No newline at end of file +// Test endpoint for direct database query of taxonomy data +router.get("/test-taxonomy", async (req, res) => { + try { + console.log("Test taxonomy endpoint called"); + + let ssh = null; + let connection = null; + + try { + // Setup MySQL connection via SSH tunnel + const tunnel = await setupSshTunnel(); + ssh = tunnel.ssh; + + connection = await mysql.createConnection({ + ...tunnel.dbConfig, + stream: tunnel.stream + }); + + console.log("MySQL connection established successfully for test"); + + const results = {}; + + // Test categories query + try { + const [categories] = await connection.query(` + SELECT cat_id, name FROM product_categories WHERE type=10 LIMIT 5 + `); + results.categories = { + success: true, + count: categories.length, + sample: categories.length > 0 ? categories[0] : null + }; + } catch (error) { + results.categories = { + success: false, + error: error.message, + sqlMessage: error.sqlMessage + }; + } + + // Test themes query + try { + const [themes] = await connection.query(` + SELECT cat_id, name FROM product_categories WHERE type=20 LIMIT 5 + `); + results.themes = { + success: true, + count: themes.length, + sample: themes.length > 0 ? themes[0] : null + }; + } catch (error) { + results.themes = { + success: false, + error: error.message, + sqlMessage: error.sqlMessage + }; + } + + // Test colors query + try { + const [colors] = await connection.query(` + SELECT color, name, hex_color FROM product_color_list ORDER BY \`order\` LIMIT 5 + `); + results.colors = { + success: true, + count: colors.length, + sample: colors.length > 0 ? colors[0] : null + }; + } catch (error) { + results.colors = { + success: false, + error: error.message, + sqlMessage: error.sqlMessage + }; + } + + return res.json({ + message: "Test taxonomy queries executed", + results: results, + timestamp: new Date().toISOString() + }); + } finally { + if (connection) await connection.end(); + if (ssh) ssh.end(); + } + } catch (error) { + console.error("Test taxonomy endpoint error:", error); + return res.status(500).json({ + error: error.message, + stack: error.stack + }); + } +}); + +module.exports = router; diff --git a/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx b/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx index 508fa0f..8949082 100644 --- a/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx +++ b/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx @@ -1555,6 +1555,14 @@ export const ValidationStep = ({ }); console.log('Sending data for validation:', data); + // Clean the data to ensure we only send what's needed + const cleanedData = data.map(item => { + const { __errors, __index, ...cleanProduct } = item; + return cleanProduct; + }); + + console.log('Cleaned data for validation:', cleanedData); + setAiValidationProgress(prev => ({ ...prev, status: "Sending data to AI service and awaiting response...", @@ -1566,11 +1574,17 @@ export const ValidationStep = ({ headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ products: data }), + body: JSON.stringify({ products: cleanedData }), }); if (!response.ok) { - throw new Error('AI validation failed'); + const errorText = await response.text(); + console.error('AI validation error response:', { + status: response.status, + statusText: response.statusText, + body: errorText + }); + throw new Error(`AI validation failed: ${response.status} ${response.statusText}`); } setAiValidationProgress(prev => ({ diff --git a/inventory/src/pages/AiValidationDebug.tsx b/inventory/src/pages/AiValidationDebug.tsx index 664b390..da2b1fa 100644 --- a/inventory/src/pages/AiValidationDebug.tsx +++ b/inventory/src/pages/AiValidationDebug.tsx @@ -7,22 +7,18 @@ import { useToast } from "@/hooks/use-toast" import { Loader2 } from "lucide-react" import config from "@/config" -interface CacheStatus { - isCacheValid: boolean - lastUpdated: string | null - timeUntilExpiry: string -} - interface TaxonomyStats { categories: number themes: number colors: number taxCodes: number sizeCategories: number + suppliers: number + companies: number + artists: number } interface DebugData { - cacheStatus: CacheStatus taxonomyStats: TaxonomyStats | null basePrompt: string sampleFullPrompt: string @@ -72,39 +68,6 @@ export function AiValidationDebug() { } } - const refreshCache = async () => { - if (!confirm('Are you sure you want to refresh the cache?')) return - - setIsLoading(true) - try { - const response = await fetch(`${config.apiUrl}/ai-validation/refresh-cache`, { - method: 'POST' - }) - if (!response.ok) { - throw new Error('Failed to refresh cache') - } - const data = await response.json() - if (data.success) { - toast({ - title: "Success", - description: "Cache refreshed successfully" - }) - fetchDebugData() - } else { - throw new Error(data.error || 'Failed to refresh cache') - } - } catch (error) { - console.error('Error refreshing cache:', error) - toast({ - variant: "destructive", - title: "Error", - description: error instanceof Error ? error.message : "Failed to refresh cache" - }) - } finally { - setIsLoading(false) - } - } - useEffect(() => { fetchDebugData() }, []) @@ -122,32 +85,11 @@ export function AiValidationDebug() { {isLoading && } Refresh Data - {debugData && ( -
- - - Cache Status - - -
-
Valid: {debugData.cacheStatus.isCacheValid ? "Yes" : "No"}
-
Last Updated: {debugData.cacheStatus.lastUpdated || "never"}
-
Expires in: {debugData.cacheStatus.timeUntilExpiry}
-
- - - +
Taxonomy Stats @@ -160,6 +102,9 @@ export function AiValidationDebug() {
Colors: {debugData.taxonomyStats.colors}
Tax Codes: {debugData.taxonomyStats.taxCodes}
Size Categories: {debugData.taxonomyStats.sizeCategories}
+
Suppliers: {debugData.taxonomyStats.suppliers}
+
Companies: {debugData.taxonomyStats.companies}
+
Artists: {debugData.taxonomyStats.artists}
) : (
No taxonomy data available