From f0e2023803ab2faaa74af88327651277a034b4f6 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 4 Oct 2025 21:03:14 -0400 Subject: [PATCH] Remove fallback and hardcoded prompts to rely on database prompts only for AI --- inventory-server/src/routes/ai-validation.js | 310 ++++++++----------- 1 file changed, 128 insertions(+), 182 deletions(-) diff --git a/inventory-server/src/routes/ai-validation.js b/inventory-server/src/routes/ai-validation.js index 6134f2d..8795940 100644 --- a/inventory-server/src/routes/ai-validation.js +++ b/inventory-server/src/routes/ai-validation.js @@ -353,15 +353,14 @@ async function generateDebugResponse(productsToUse, res) { WHERE prompt_type = 'system' `); - // Get system prompt or use default - let systemPrompt = null; - if (systemPromptResult.rows.length > 0) { - systemPrompt = systemPromptResult.rows[0]; - console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id); - } else { - console.warn("⚠️ No system prompt found in database, will use default"); + if (systemPromptResult.rows.length === 0) { + console.error("❌ No system prompt found in database"); + throw new Error("No system prompt found in database"); } + const systemPrompt = systemPromptResult.rows[0]; + console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id); + // Then, fetch the general prompt using the consolidated endpoint approach const generalPromptResult = await pool.query(` SELECT * FROM ai_prompts @@ -369,7 +368,7 @@ async function generateDebugResponse(productsToUse, res) { `); if (generalPromptResult.rows.length === 0) { - console.warn("⚠️ No general prompt found in database"); + console.error("❌ No general prompt found in database"); throw new Error("No general prompt found in database"); } @@ -481,22 +480,15 @@ async function generateDebugResponse(productsToUse, res) { : null, } : null, - basePrompt: systemPrompt ? systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text : generalPrompt.prompt_text, + basePrompt: systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text, sampleFullPrompt: fullUserPrompt, promptLength: promptLength, apiFormat: apiMessages, promptSources: { - ...(systemPrompt ? { - systemPrompt: { - id: systemPrompt.id, - prompt_text: systemPrompt.prompt_text - } - } : { - systemPrompt: { - id: 0, - prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.` - } - }), + systemPrompt: { + id: systemPrompt.id, + prompt_text: systemPrompt.prompt_text + }, generalPrompt: { id: generalPrompt.id, prompt_text: generalPrompt.prompt_text @@ -702,17 +694,14 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null) WHERE prompt_type = 'system' `); - // Default system instructions in case the system prompt is not found - let systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`; - - // If system prompt exists in the database, use it - if (systemPromptResult.rows.length > 0) { - systemInstructions = systemPromptResult.rows[0].prompt_text; - console.log("📝 Loaded system prompt from database"); - } else { - console.warn("⚠️ No system prompt found in database, using default"); + if (systemPromptResult.rows.length === 0) { + console.error("❌ No system prompt found in database"); + throw new Error("No system prompt found in database"); } + const systemInstructions = systemPromptResult.rows[0].prompt_text; + console.log("📝 Loaded system prompt from database"); + // Fetch the general prompt using the consolidated endpoint approach const generalPromptResult = await pool.query(` SELECT * FROM ai_prompts @@ -720,7 +709,7 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null) `); if (generalPromptResult.rows.length === 0) { - console.warn("⚠️ No general prompt found in database"); + console.error("❌ No general prompt found in database"); throw new Error("No general prompt found in database"); } @@ -779,84 +768,87 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null) combinedPrompt += "\n--- END COMPANY-SPECIFIC INSTRUCTIONS ---\n"; } - // If we have products to validate, create a filtered prompt - if (productsToValidate) { - console.log("Creating filtered prompt for products:", productsToValidate); + // Products are required for validation + if (!productsToValidate || !Array.isArray(productsToValidate) || productsToValidate.length === 0) { + throw new Error("Products are required for prompt generation"); + } - // Extract unique values from products for non-core attributes - const uniqueValues = { - supplierIds: new Set(), - companyIds: new Set(), - artistIds: new Set(), - lineIds: new Set(), - subLineIds: new Set(), - }; + console.log("Creating filtered prompt for products:", productsToValidate); - // Collect any values that exist in the products - productsToValidate.forEach((product) => { - Object.entries(product).forEach(([key, value]) => { - if (value === undefined || value === null) return; + // Extract unique values from products for non-core attributes + const uniqueValues = { + supplierIds: new Set(), + companyIds: new Set(), + artistIds: new Set(), + lineIds: new Set(), + subLineIds: new Set(), + }; - // Map field names to their respective sets - const fieldMap = { - supplierid: "supplierIds", - supplier: "supplierIds", - company: "companyIds", - artist: "artistIds", - line: "lineIds", - subline: "subLineIds", - }; + // Collect any values that exist in the products + productsToValidate.forEach((product) => { + Object.entries(product).forEach(([key, value]) => { + if (value === undefined || value === null) return; - if (fieldMap[key]) { - uniqueValues[fieldMap[key]].add(Number(value)); - } - }); + // Map field names to their respective sets + const fieldMap = { + supplierid: "supplierIds", + supplier: "supplierIds", + company: "companyIds", + artist: "artistIds", + line: "lineIds", + subline: "subLineIds", + }; + + if (fieldMap[key]) { + uniqueValues[fieldMap[key]].add(Number(value)); + } }); + }); - console.log("Unique values collected:", { - suppliers: Array.from(uniqueValues.supplierIds), - companies: Array.from(uniqueValues.companyIds), - artists: Array.from(uniqueValues.artistIds), - lines: Array.from(uniqueValues.lineIds), - subLines: Array.from(uniqueValues.subLineIds), - }); + console.log("Unique values collected:", { + suppliers: Array.from(uniqueValues.supplierIds), + companies: Array.from(uniqueValues.companyIds), + artists: Array.from(uniqueValues.artistIds), + lines: Array.from(uniqueValues.lineIds), + subLines: Array.from(uniqueValues.subLineIds), + }); - // Create mixed taxonomy with filtered non-core data and full core data - const mixedTaxonomy = { - // Keep full data for core attributes - categories: taxonomy.categories, - themes: taxonomy.themes, - colors: taxonomy.colors, - taxCodes: taxonomy.taxCodes, - sizeCategories: taxonomy.sizeCategories, - // For non-core data, only include items that are actually used - suppliers: taxonomy.suppliers.filter(([id]) => - uniqueValues.supplierIds.has(Number(id)) - ), - companies: taxonomy.companies.filter(([id]) => - uniqueValues.companyIds.has(Number(id)) - ), - artists: taxonomy.artists.filter(([id]) => - uniqueValues.artistIds.has(Number(id)) - ), - lines: taxonomy.lines.filter(([id]) => - uniqueValues.lineIds.has(Number(id)) - ), - subLines: taxonomy.subLines.filter(([id]) => - uniqueValues.subLineIds.has(Number(id)) - ), - }; + // Create mixed taxonomy with filtered non-core data and full core data + const mixedTaxonomy = { + // Keep full data for core attributes + categories: taxonomy.categories, + themes: taxonomy.themes, + colors: taxonomy.colors, + taxCodes: taxonomy.taxCodes, + sizeCategories: taxonomy.sizeCategories, + // For non-core data, only include items that are actually used + suppliers: taxonomy.suppliers.filter(([id]) => + uniqueValues.supplierIds.has(Number(id)) + ), + companies: taxonomy.companies.filter(([id]) => + uniqueValues.companyIds.has(Number(id)) + ), + artists: taxonomy.artists.filter(([id]) => + uniqueValues.artistIds.has(Number(id)) + ), + lines: taxonomy.lines.filter(([id]) => + uniqueValues.lineIds.has(Number(id)) + ), + subLines: taxonomy.subLines.filter(([id]) => + uniqueValues.subLineIds.has(Number(id)) + ), + }; - console.log("Filtered taxonomy counts:", { - suppliers: mixedTaxonomy.suppliers.length, - companies: mixedTaxonomy.companies.length, - artists: mixedTaxonomy.artists.length, - lines: mixedTaxonomy.lines.length, - subLines: mixedTaxonomy.subLines.length, - }); + console.log("Filtered taxonomy counts:", { + suppliers: mixedTaxonomy.suppliers.length, + companies: mixedTaxonomy.companies.length, + artists: mixedTaxonomy.artists.length, + lines: mixedTaxonomy.lines.length, + subLines: mixedTaxonomy.subLines.length, + }); - // Format taxonomy data for the prompt, only including sections with values - const taxonomySection = ` + // Format taxonomy data for the prompt, only including sections with values + const taxonomySection = ` All Available Categories: ${JSON.stringify(mixedTaxonomy.categories)} @@ -871,73 +863,38 @@ ${JSON.stringify(mixedTaxonomy.taxCodes)} All Available Size Categories: ${JSON.stringify(mixedTaxonomy.sizeCategories)}${ - mixedTaxonomy.suppliers.length - ? `\n\nSuppliers Used In This Data:\n${JSON.stringify( - mixedTaxonomy.suppliers - )}` - : "" - }${ - mixedTaxonomy.companies.length - ? `\n\nCompanies Used In This Data:\n${JSON.stringify( - mixedTaxonomy.companies - )}` - : "" - }${ - mixedTaxonomy.artists.length - ? `\n\nArtists Used In This Data:\n${JSON.stringify( - mixedTaxonomy.artists - )}` - : "" - }${ - mixedTaxonomy.lines.length - ? `\n\nLines Used In This Data:\n${JSON.stringify( - mixedTaxonomy.lines - )}` - : "" - }${ - mixedTaxonomy.subLines.length - ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify( - mixedTaxonomy.subLines - )}` - : "" - } - -----------Here is the product data to validate----------`; - - // Return both system instructions and user content separately - return { - systemInstructions, - userContent: combinedPrompt + "\n" + taxonomySection - }; + mixedTaxonomy.suppliers.length + ? `\n\nSuppliers Used In This Data:\n${JSON.stringify( + mixedTaxonomy.suppliers + )}` + : "" + }${ + mixedTaxonomy.companies.length + ? `\n\nCompanies Used In This Data:\n${JSON.stringify( + mixedTaxonomy.companies + )}` + : "" + }${ + mixedTaxonomy.artists.length + ? `\n\nArtists Used In This Data:\n${JSON.stringify( + mixedTaxonomy.artists + )}` + : "" + }${ + mixedTaxonomy.lines.length + ? `\n\nLines Used In This Data:\n${JSON.stringify( + mixedTaxonomy.lines + )}` + : "" + }${ + mixedTaxonomy.subLines.length + ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify( + mixedTaxonomy.subLines + )}` + : "" } - // Generate the full unfiltered prompt for taxonomy section - const taxonomySection = ` -Available Categories: -${JSON.stringify(taxonomy.categories)} - -Available Themes: -${JSON.stringify(taxonomy.themes)} - -Available Colors: -${JSON.stringify(taxonomy.colors)} - -Available Tax Codes: -${JSON.stringify(taxonomy.taxCodes)} - -Available Size Categories: -${JSON.stringify(taxonomy.sizeCategories)} - -Available Suppliers: -${JSON.stringify(taxonomy.suppliers)} - -Available Companies: -${JSON.stringify(taxonomy.companies)} - -Available Artists: -${JSON.stringify(taxonomy.artists)} - -Here is the product data to validate:`; +----------Here is the product data to validate----------`; // Return both system instructions and user content separately return { @@ -1007,7 +964,7 @@ router.post("/validate", async (req, res) => { input: [ { role: "developer", - content: `${promptData.systemInstructions}\n\nYou MUST respond with a single valid JSON object containing the following top-level keys: correctedData, changes, warnings, summary, metadata.\n- correctedData: array of product objects reflecting the updated data.\n- changes: array of human-readable bullet points summarizing the nature of updates.\n- warnings: array of caveats or risks that still require review.\n- summary: a concise paragraph (<=75 words) describing overall data quality and improvements.\n- metadata: object containing any supplemental machine-readable information (optional fields allowed).\nDo NOT include Markdown code fences or any text outside the JSON object.`, + content: promptData.systemInstructions, }, { role: "user", @@ -1242,26 +1199,15 @@ router.post("/validate", async (req, res) => { })); // Set prompt sources - if (generalPromptResult.rows.length > 0) { + if (generalPromptResult.rows.length > 0 && systemPromptResult.rows.length > 0) { const generalPrompt = generalPromptResult.rows[0]; - let systemPrompt = null; - - if (systemPromptResult.rows.length > 0) { - systemPrompt = systemPromptResult.rows[0]; - } + const systemPrompt = systemPromptResult.rows[0]; promptSources = { - ...(systemPrompt ? { - systemPrompt: { - id: systemPrompt.id, - prompt_text: systemPrompt.prompt_text - } - } : { - systemPrompt: { - id: 0, - prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.` - } - }), + systemPrompt: { + id: systemPrompt.id, + prompt_text: systemPrompt.prompt_text + }, generalPrompt: { id: generalPrompt.id, prompt_text: generalPrompt.prompt_text