Fix AI prompts not refreshing in the frontend after save, some AI tweaks

inventory-server/src/routes/ai-validation.js

@@ -353,15 +353,14 @@ async function generateDebugResponse(productsToUse, res) {
         WHERE prompt_type = 'system'
       `);
       
-      // Get system prompt or use default
-      let systemPrompt = null;
-      if (systemPromptResult.rows.length > 0) {
-        systemPrompt = systemPromptResult.rows[0];
-        console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
-      } else {
-        console.warn("⚠️ No system prompt found in database, will use default");
+      if (systemPromptResult.rows.length === 0) {
+        console.error("❌ No system prompt found in database");
+        throw new Error("No system prompt found in database");
       }
 
+      const systemPrompt = systemPromptResult.rows[0];
+      console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
+
       // Then, fetch the general prompt using the consolidated endpoint approach
       const generalPromptResult = await pool.query(`
         SELECT * FROM ai_prompts 
@@ -369,7 +368,7 @@ async function generateDebugResponse(productsToUse, res) {
       `);
       
       if (generalPromptResult.rows.length === 0) {
-        console.warn("⚠️ No general prompt found in database");
+        console.error("❌ No general prompt found in database");
         throw new Error("No general prompt found in database");
       }
 
@@ -481,22 +480,15 @@ async function generateDebugResponse(productsToUse, res) {
                 : null,
             }
           : null,
-        basePrompt: systemPrompt ? systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text : generalPrompt.prompt_text,
+        basePrompt: systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text,
         sampleFullPrompt: fullUserPrompt,
         promptLength: promptLength,
         apiFormat: apiMessages,
         promptSources: {
-          ...(systemPrompt ? {
-            systemPrompt: {
-              id: systemPrompt.id,
-              prompt_text: systemPrompt.prompt_text
-            }
-          } : {
-            systemPrompt: {
-              id: 0,
-              prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
-            }
-          }),
+          systemPrompt: {
+            id: systemPrompt.id,
+            prompt_text: systemPrompt.prompt_text
+          },
           generalPrompt: {
             id: generalPrompt.id,
             prompt_text: generalPrompt.prompt_text
@@ -702,17 +694,14 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
       WHERE prompt_type = 'system'
     `);
     
-    // Default system instructions in case the system prompt is not found
-    let systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`;
-    
-    // If system prompt exists in the database, use it
-    if (systemPromptResult.rows.length > 0) {
-      systemInstructions = systemPromptResult.rows[0].prompt_text;
-      console.log("📝 Loaded system prompt from database");
-    } else {
-      console.warn("⚠️ No system prompt found in database, using default");
+    if (systemPromptResult.rows.length === 0) {
+      console.error("❌ No system prompt found in database");
+      throw new Error("No system prompt found in database");
     }
 
+    const systemInstructions = systemPromptResult.rows[0].prompt_text;
+    console.log("📝 Loaded system prompt from database");
+
     // Fetch the general prompt using the consolidated endpoint approach
     const generalPromptResult = await pool.query(`
       SELECT * FROM ai_prompts 
@@ -720,7 +709,7 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
     `);
     
     if (generalPromptResult.rows.length === 0) {
-      console.warn("⚠️ No general prompt found in database");
+      console.error("❌ No general prompt found in database");
       throw new Error("No general prompt found in database");
     }
 
@@ -779,84 +768,87 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
       combinedPrompt += "\n--- END COMPANY-SPECIFIC INSTRUCTIONS ---\n";
     }
 
-    // If we have products to validate, create a filtered prompt
-    if (productsToValidate) {
-      console.log("Creating filtered prompt for products:", productsToValidate);
+    // Products are required for validation
+    if (!productsToValidate || !Array.isArray(productsToValidate) || productsToValidate.length === 0) {
+      throw new Error("Products are required for prompt generation");
+    }
 
-      // Extract unique values from products for non-core attributes
-      const uniqueValues = {
-        supplierIds: new Set(),
-        companyIds: new Set(),
-        artistIds: new Set(),
-        lineIds: new Set(),
-        subLineIds: new Set(),
-      };
+    console.log("Creating filtered prompt for products:", productsToValidate);
 
-      // Collect any values that exist in the products
-      productsToValidate.forEach((product) => {
-        Object.entries(product).forEach(([key, value]) => {
-          if (value === undefined || value === null) return;
+    // Extract unique values from products for non-core attributes
+    const uniqueValues = {
+      supplierIds: new Set(),
+      companyIds: new Set(),
+      artistIds: new Set(),
+      lineIds: new Set(),
+      subLineIds: new Set(),
+    };
 
-          // Map field names to their respective sets
-          const fieldMap = {
-            supplierid: "supplierIds",
-            supplier: "supplierIds",
-            company: "companyIds",
-            artist: "artistIds",
-            line: "lineIds",
-            subline: "subLineIds",
-          };
+    // Collect any values that exist in the products
+    productsToValidate.forEach((product) => {
+      Object.entries(product).forEach(([key, value]) => {
+        if (value === undefined || value === null) return;
 
-          if (fieldMap[key]) {
-            uniqueValues[fieldMap[key]].add(Number(value));
-          }
-        });
+        // Map field names to their respective sets
+        const fieldMap = {
+          supplierid: "supplierIds",
+          supplier: "supplierIds",
+          company: "companyIds",
+          artist: "artistIds",
+          line: "lineIds",
+          subline: "subLineIds",
+        };
+
+        if (fieldMap[key]) {
+          uniqueValues[fieldMap[key]].add(Number(value));
+        }
       });
+    });
 
-      console.log("Unique values collected:", {
-        suppliers: Array.from(uniqueValues.supplierIds),
-        companies: Array.from(uniqueValues.companyIds),
-        artists: Array.from(uniqueValues.artistIds),
-        lines: Array.from(uniqueValues.lineIds),
-        subLines: Array.from(uniqueValues.subLineIds),
-      });
+    console.log("Unique values collected:", {
+      suppliers: Array.from(uniqueValues.supplierIds),
+      companies: Array.from(uniqueValues.companyIds),
+      artists: Array.from(uniqueValues.artistIds),
+      lines: Array.from(uniqueValues.lineIds),
+      subLines: Array.from(uniqueValues.subLineIds),
+    });
 
-      // Create mixed taxonomy with filtered non-core data and full core data
-      const mixedTaxonomy = {
-        // Keep full data for core attributes
-        categories: taxonomy.categories,
-        themes: taxonomy.themes,
-        colors: taxonomy.colors,
-        taxCodes: taxonomy.taxCodes,
-        sizeCategories: taxonomy.sizeCategories,
-        // For non-core data, only include items that are actually used
-        suppliers: taxonomy.suppliers.filter(([id]) =>
-          uniqueValues.supplierIds.has(Number(id))
-        ),
-        companies: taxonomy.companies.filter(([id]) =>
-          uniqueValues.companyIds.has(Number(id))
-        ),
-        artists: taxonomy.artists.filter(([id]) =>
-          uniqueValues.artistIds.has(Number(id))
-        ),
-        lines: taxonomy.lines.filter(([id]) =>
-          uniqueValues.lineIds.has(Number(id))
-        ),
-        subLines: taxonomy.subLines.filter(([id]) =>
-          uniqueValues.subLineIds.has(Number(id))
-        ),
-      };
+    // Create mixed taxonomy with filtered non-core data and full core data
+    const mixedTaxonomy = {
+      // Keep full data for core attributes
+      categories: taxonomy.categories,
+      themes: taxonomy.themes,
+      colors: taxonomy.colors,
+      taxCodes: taxonomy.taxCodes,
+      sizeCategories: taxonomy.sizeCategories,
+      // For non-core data, only include items that are actually used
+      suppliers: taxonomy.suppliers.filter(([id]) =>
+        uniqueValues.supplierIds.has(Number(id))
+      ),
+      companies: taxonomy.companies.filter(([id]) =>
+        uniqueValues.companyIds.has(Number(id))
+      ),
+      artists: taxonomy.artists.filter(([id]) =>
+        uniqueValues.artistIds.has(Number(id))
+      ),
+      lines: taxonomy.lines.filter(([id]) =>
+        uniqueValues.lineIds.has(Number(id))
+      ),
+      subLines: taxonomy.subLines.filter(([id]) =>
+        uniqueValues.subLineIds.has(Number(id))
+      ),
+    };
 
-      console.log("Filtered taxonomy counts:", {
-        suppliers: mixedTaxonomy.suppliers.length,
-        companies: mixedTaxonomy.companies.length,
-        artists: mixedTaxonomy.artists.length,
-        lines: mixedTaxonomy.lines.length,
-        subLines: mixedTaxonomy.subLines.length,
-      });
+    console.log("Filtered taxonomy counts:", {
+      suppliers: mixedTaxonomy.suppliers.length,
+      companies: mixedTaxonomy.companies.length,
+      artists: mixedTaxonomy.artists.length,
+      lines: mixedTaxonomy.lines.length,
+      subLines: mixedTaxonomy.subLines.length,
+    });
 
-      // Format taxonomy data for the prompt, only including sections with values
-      const taxonomySection = `
+    // Format taxonomy data for the prompt, only including sections with values
+    const taxonomySection = `
 All Available Categories:
 ${JSON.stringify(mixedTaxonomy.categories)}
 
@@ -871,73 +863,38 @@ ${JSON.stringify(mixedTaxonomy.taxCodes)}
 
 All Available Size Categories:
 ${JSON.stringify(mixedTaxonomy.sizeCategories)}${
-        mixedTaxonomy.suppliers.length
-          ? `\n\nSuppliers Used In This Data:\n${JSON.stringify(
-              mixedTaxonomy.suppliers
-            )}`
-          : ""
-      }${
-        mixedTaxonomy.companies.length
-          ? `\n\nCompanies Used In This Data:\n${JSON.stringify(
-              mixedTaxonomy.companies
-            )}`
-          : ""
-      }${
-        mixedTaxonomy.artists.length
-          ? `\n\nArtists Used In This Data:\n${JSON.stringify(
-              mixedTaxonomy.artists
-            )}`
-          : ""
-      }${
-        mixedTaxonomy.lines.length
-          ? `\n\nLines Used In This Data:\n${JSON.stringify(
-              mixedTaxonomy.lines
-            )}`
-          : ""
-      }${
-        mixedTaxonomy.subLines.length
-          ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify(
-              mixedTaxonomy.subLines
-            )}`
-          : ""
-      }
-
-----------Here is the product data to validate----------`;
-
-      // Return both system instructions and user content separately
-      return {
-        systemInstructions,
-        userContent: combinedPrompt + "\n" + taxonomySection
-      };
+      mixedTaxonomy.suppliers.length
+        ? `\n\nSuppliers Used In This Data:\n${JSON.stringify(
+            mixedTaxonomy.suppliers
+          )}`
+        : ""
+    }${
+      mixedTaxonomy.companies.length
+        ? `\n\nCompanies Used In This Data:\n${JSON.stringify(
+            mixedTaxonomy.companies
+          )}`
+        : ""
+    }${
+      mixedTaxonomy.artists.length
+        ? `\n\nArtists Used In This Data:\n${JSON.stringify(
+            mixedTaxonomy.artists
+          )}`
+        : ""
+    }${
+      mixedTaxonomy.lines.length
+        ? `\n\nLines Used In This Data:\n${JSON.stringify(
+            mixedTaxonomy.lines
+          )}`
+        : ""
+    }${
+      mixedTaxonomy.subLines.length
+        ? `\n\nSub-Lines Used In This Data:\n${JSON.stringify(
+            mixedTaxonomy.subLines
+          )}`
+        : ""
     }
 
-    // Generate the full unfiltered prompt for taxonomy section
-    const taxonomySection = `
-Available Categories:
-${JSON.stringify(taxonomy.categories)}
-
-Available Themes:
-${JSON.stringify(taxonomy.themes)}
-
-Available Colors:
-${JSON.stringify(taxonomy.colors)}
-
-Available Tax Codes:
-${JSON.stringify(taxonomy.taxCodes)}
-
-Available Size Categories:
-${JSON.stringify(taxonomy.sizeCategories)}
-
-Available Suppliers:
-${JSON.stringify(taxonomy.suppliers)}
-
-Available Companies:
-${JSON.stringify(taxonomy.companies)}
-
-Available Artists:
-${JSON.stringify(taxonomy.artists)}
-
-Here is the product data to validate:`;
+----------Here is the product data to validate----------`;
 
     // Return both system instructions and user content separately
     return {
@@ -1007,7 +964,7 @@ router.post("/validate", async (req, res) => {
         input: [
           {
             role: "developer",
-            content: `${promptData.systemInstructions}\n\nYou MUST respond with a single valid JSON object containing the following top-level keys: correctedData, changes, warnings, summary, metadata.\n- correctedData: array of product objects reflecting the updated data.\n- changes: array of human-readable bullet points summarizing the nature of updates.\n- warnings: array of caveats or risks that still require review.\n- summary: a concise paragraph (<=75 words) describing overall data quality and improvements.\n- metadata: object containing any supplemental machine-readable information (optional fields allowed).\nDo NOT include Markdown code fences or any text outside the JSON object.`,
+            content: promptData.systemInstructions,
           },
           {
             role: "user",
@@ -1068,61 +1025,88 @@ router.post("/validate", async (req, res) => {
           Object.keys(aiResponse)
         );
 
-        // Create a detailed comparison between original and corrected data
+        // Merge AI changes back into original products
+        // AI now only returns changed products and changed fields
+        const mergedProducts = products.map((original, index) => ({ ...original }));
         const changeDetails = [];
         
-        // Compare original and corrected data
-        if (aiResponse.correctedData) {
-          console.log("📊 Changes summary:");
+        if (aiResponse.correctedData && Array.isArray(aiResponse.correctedData)) {
+          console.log("📊 Processing AI changes - received", aiResponse.correctedData.length, "products with changes");
           
-          // Debug: Log the first product's fields
-          if (products.length > 0) {
-            console.log("🔍 First product fields:", Object.keys(products[0]));
-          }
+          // Process each changed product from AI
+          aiResponse.correctedData.forEach((changedProduct) => {
+            // Find the matching original product using stable identifiers in priority order
+            // Priority: upc > supplier_no > notions_no
+            // These fields should not change during validation
+            const identifiers = ['upc', 'supplier_no', 'notions_no'];
+            let matchedIndex = -1;
+            let matchedBy = null;
             
-          products.forEach((original, index) => {
-            const corrected = aiResponse.correctedData[index];
-            if (corrected) {
-              const productChanges = {
-                productIndex: index,
-                title: original.name || original.title || `Product ${index + 1}`,
-                changes: []
-              };
-              
-              const changes = Object.keys(corrected).filter(
-                (key) =>
-                  JSON.stringify(original[key]) !==
-                  JSON.stringify(corrected[key])
-              );
-              
-              if (changes.length > 0) {
-                console.log(`\nProduct ${index + 1} changes:`);
-                changes.forEach((key) => {
-                  console.log(`  ${key}:`);
-                  console.log(
-                    `    - Original: ${JSON.stringify(original[key])}`
-                  );
-                  console.log(
-                    `    - Corrected: ${JSON.stringify(corrected[key])}`
-                  );
-                  
-                  // Add to our detailed changes array
-                  productChanges.changes.push({
-                    field: key,
-                    original: original[key],
-                    corrected: corrected[key]
-                  });
-                });
-                
-                // Only add products that have changes
-                if (productChanges.changes.length > 0) {
-                  changeDetails.push(productChanges);
+            for (const identifier of identifiers) {
+              if (changedProduct[identifier] !== undefined && changedProduct[identifier] !== null && changedProduct[identifier] !== '') {
+                matchedIndex = products.findIndex(
+                  (p) => p[identifier] !== undefined && 
+                         p[identifier] !== null &&
+                         p[identifier] !== '' &&
+                         String(p[identifier]).trim() === String(changedProduct[identifier]).trim()
+                );
+                if (matchedIndex !== -1) {
+                  matchedBy = identifier;
+                  console.log(`✓ Matched product by ${identifier}:`, changedProduct[identifier]);
+                  break;
                 }
               }
             }
+            
+            // If no identifier match found, log an error with details
+            if (matchedIndex === -1) {
+              console.error("❌ Could not match changed product to original. Product identifiers:", {
+                upc: changedProduct.upc,
+                supplier_no: changedProduct.supplier_no,
+                notions_no: changedProduct.notions_no
+              });
+              return;
+            }
+            
+            const original = products[matchedIndex];
+            const productChanges = {
+              productIndex: matchedIndex,
+              title: original.name || original.title || `Product ${matchedIndex + 1}`,
+              changes: []
+            };
+            
+            // Apply each changed field to the merged product
+            Object.keys(changedProduct).forEach((key) => {
+              // Check if the value actually changed
+              if (JSON.stringify(original[key]) !== JSON.stringify(changedProduct[key])) {
+                console.log(`\nProduct ${matchedIndex + 1} - Field ${key}:`);
+                console.log(`  - Original: ${JSON.stringify(original[key])}`);
+                console.log(`  - Corrected: ${JSON.stringify(changedProduct[key])}`);
+                
+                // Apply the change to merged product
+                mergedProducts[matchedIndex][key] = changedProduct[key];
+                
+                // Track the change
+                productChanges.changes.push({
+                  field: key,
+                  original: original[key],
+                  corrected: changedProduct[key]
+                });
+              }
+            });
+            
+            // Only add to changeDetails if there were actual changes
+            if (productChanges.changes.length > 0) {
+              changeDetails.push(productChanges);
+            }
           });
+          
+          console.log(`📊 Applied changes to ${changeDetails.length} products`);
         }
         
+        // Replace aiResponse.correctedData with the fully merged product array
+        aiResponse.correctedData = mergedProducts;
+
         // Record performance metrics after successful validation
         const endTime = new Date();
         let performanceMetrics = {
@@ -1242,26 +1226,15 @@ router.post("/validate", async (req, res) => {
             }));
 
             // Set prompt sources
-            if (generalPromptResult.rows.length > 0) {
+            if (generalPromptResult.rows.length > 0 && systemPromptResult.rows.length > 0) {
               const generalPrompt = generalPromptResult.rows[0];
-              let systemPrompt = null;
-
-              if (systemPromptResult.rows.length > 0) {
-                systemPrompt = systemPromptResult.rows[0];
-              }
+              const systemPrompt = systemPromptResult.rows[0];
 
               promptSources = {
-                ...(systemPrompt ? {
-                  systemPrompt: {
-                    id: systemPrompt.id,
-                    prompt_text: systemPrompt.prompt_text
-                  }
-                } : {
-                  systemPrompt: {
-                    id: 0,
-                    prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
-                  }
-                }),
+                systemPrompt: {
+                  id: systemPrompt.id,
+                  prompt_text: systemPrompt.prompt_text
+                },
                 generalPrompt: {
                   id: generalPrompt.id,
                   prompt_text: generalPrompt.prompt_text

inventory/src/components/product-import/steps/ImageUploadStep/ImageUploadStep.tsx

@@ -186,7 +186,7 @@ export const ImageUploadStep = ({
     } finally {
       setIsSubmitting(false);
     }
-  }, [data, file, onSubmit, productImages]);
+  }, [data, file, onSubmit, productImages, targetEnvironment, useTestDataSource]);
 
   return (
     <div className="flex flex-col h-[calc(100vh-9.5rem)] overflow-hidden">

inventory/src/components/settings/PromptManagement.tsx

@@ -134,8 +134,12 @@ export function PromptManagement() {
       
       return response.json();
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (newPrompt) => {
+      // Optimistically update the cache with the new prompt
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [newPrompt];
+        return [...old, newPrompt];
+      });
       toast.success("Prompt created successfully");
       resetForm();
     },
@@ -163,8 +167,14 @@ export function PromptManagement() {
       
       return response.json();
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (updatedPrompt) => {
+      // Optimistically update the cache with the returned data
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [updatedPrompt];
+        return old.map((prompt) => 
+          prompt.id === updatedPrompt.id ? updatedPrompt : prompt
+        );
+      });
       toast.success("Prompt updated successfully");
       resetForm();
     },
@@ -181,9 +191,14 @@ export function PromptManagement() {
       if (!response.ok) {
         throw new Error("Failed to delete prompt");
       }
+      return id;
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (deletedId) => {
+      // Optimistically update the cache by removing the deleted prompt
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [];
+        return old.filter((prompt) => prompt.id !== deletedId);
+      });
       toast.success("Prompt deleted successfully");
     },
     onError: (error) => {