Fix AI prompts not refreshing in the frontend after save, some AI tweaks

inventory-server/src/routes/ai-validation.js

@@ -353,15 +353,14 @@ async function generateDebugResponse(productsToUse, res) {
         WHERE prompt_type = 'system'
       `);
       
-      // Get system prompt or use default
-      let systemPrompt = null;
-      if (systemPromptResult.rows.length > 0) {
-        systemPrompt = systemPromptResult.rows[0];
-        console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
-      } else {
-        console.warn("⚠️ No system prompt found in database, will use default");
+      if (systemPromptResult.rows.length === 0) {
+        console.error("❌ No system prompt found in database");
+        throw new Error("No system prompt found in database");
       }
 
+      const systemPrompt = systemPromptResult.rows[0];
+      console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
+
       // Then, fetch the general prompt using the consolidated endpoint approach
       const generalPromptResult = await pool.query(`
         SELECT * FROM ai_prompts 
@@ -369,7 +368,7 @@ async function generateDebugResponse(productsToUse, res) {
       `);
       
       if (generalPromptResult.rows.length === 0) {
-        console.warn("⚠️ No general prompt found in database");
+        console.error("❌ No general prompt found in database");
         throw new Error("No general prompt found in database");
       }
 
@@ -481,22 +480,15 @@ async function generateDebugResponse(productsToUse, res) {
                 : null,
             }
           : null,
-        basePrompt: systemPrompt ? systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text : generalPrompt.prompt_text,
+        basePrompt: systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text,
         sampleFullPrompt: fullUserPrompt,
         promptLength: promptLength,
         apiFormat: apiMessages,
         promptSources: {
-          ...(systemPrompt ? {
           systemPrompt: {
             id: systemPrompt.id,
             prompt_text: systemPrompt.prompt_text
-            }
-          } : {
-            systemPrompt: {
-              id: 0,
-              prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
-            }
-          }),
+          },
           generalPrompt: {
             id: generalPrompt.id,
             prompt_text: generalPrompt.prompt_text
@@ -702,17 +694,14 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
       WHERE prompt_type = 'system'
     `);
     
-    // Default system instructions in case the system prompt is not found
-    let systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`;
-    
-    // If system prompt exists in the database, use it
-    if (systemPromptResult.rows.length > 0) {
-      systemInstructions = systemPromptResult.rows[0].prompt_text;
-      console.log("📝 Loaded system prompt from database");
-    } else {
-      console.warn("⚠️ No system prompt found in database, using default");
+    if (systemPromptResult.rows.length === 0) {
+      console.error("❌ No system prompt found in database");
+      throw new Error("No system prompt found in database");
     }
 
+    const systemInstructions = systemPromptResult.rows[0].prompt_text;
+    console.log("📝 Loaded system prompt from database");
+
     // Fetch the general prompt using the consolidated endpoint approach
     const generalPromptResult = await pool.query(`
       SELECT * FROM ai_prompts 
@@ -720,7 +709,7 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
     `);
     
     if (generalPromptResult.rows.length === 0) {
-      console.warn("⚠️ No general prompt found in database");
+      console.error("❌ No general prompt found in database");
       throw new Error("No general prompt found in database");
     }
 
@@ -779,8 +768,11 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
       combinedPrompt += "\n--- END COMPANY-SPECIFIC INSTRUCTIONS ---\n";
     }
 
-    // If we have products to validate, create a filtered prompt
-    if (productsToValidate) {
+    // Products are required for validation
+    if (!productsToValidate || !Array.isArray(productsToValidate) || productsToValidate.length === 0) {
+      throw new Error("Products are required for prompt generation");
+    }
+
     console.log("Creating filtered prompt for products:", productsToValidate);
 
     // Extract unique values from products for non-core attributes
@@ -904,41 +896,6 @@ ${JSON.stringify(mixedTaxonomy.sizeCategories)}${
 
 ----------Here is the product data to validate----------`;
 
-      // Return both system instructions and user content separately
-      return {
-        systemInstructions,
-        userContent: combinedPrompt + "\n" + taxonomySection
-      };
-    }
-
-    // Generate the full unfiltered prompt for taxonomy section
-    const taxonomySection = `
-Available Categories:
-${JSON.stringify(taxonomy.categories)}
-
-Available Themes:
-${JSON.stringify(taxonomy.themes)}
-
-Available Colors:
-${JSON.stringify(taxonomy.colors)}
-
-Available Tax Codes:
-${JSON.stringify(taxonomy.taxCodes)}
-
-Available Size Categories:
-${JSON.stringify(taxonomy.sizeCategories)}
-
-Available Suppliers:
-${JSON.stringify(taxonomy.suppliers)}
-
-Available Companies:
-${JSON.stringify(taxonomy.companies)}
-
-Available Artists:
-${JSON.stringify(taxonomy.artists)}
-
-Here is the product data to validate:`;
-
     // Return both system instructions and user content separately
     return {
       systemInstructions,
@@ -1007,7 +964,7 @@ router.post("/validate", async (req, res) => {
         input: [
           {
             role: "developer",
-            content: `${promptData.systemInstructions}\n\nYou MUST respond with a single valid JSON object containing the following top-level keys: correctedData, changes, warnings, summary, metadata.\n- correctedData: array of product objects reflecting the updated data.\n- changes: array of human-readable bullet points summarizing the nature of updates.\n- warnings: array of caveats or risks that still require review.\n- summary: a concise paragraph (<=75 words) describing overall data quality and improvements.\n- metadata: object containing any supplemental machine-readable information (optional fields allowed).\nDo NOT include Markdown code fences or any text outside the JSON object.`,
+            content: promptData.systemInstructions,
           },
           {
             role: "user",
@@ -1068,61 +1025,88 @@ router.post("/validate", async (req, res) => {
           Object.keys(aiResponse)
         );
 
-        // Create a detailed comparison between original and corrected data
+        // Merge AI changes back into original products
+        // AI now only returns changed products and changed fields
+        const mergedProducts = products.map((original, index) => ({ ...original }));
         const changeDetails = [];
         
-        // Compare original and corrected data
-        if (aiResponse.correctedData) {
-          console.log("📊 Changes summary:");
+        if (aiResponse.correctedData && Array.isArray(aiResponse.correctedData)) {
+          console.log("📊 Processing AI changes - received", aiResponse.correctedData.length, "products with changes");
           
-          // Debug: Log the first product's fields
-          if (products.length > 0) {
-            console.log("🔍 First product fields:", Object.keys(products[0]));
+          // Process each changed product from AI
+          aiResponse.correctedData.forEach((changedProduct) => {
+            // Find the matching original product using stable identifiers in priority order
+            // Priority: upc > supplier_no > notions_no
+            // These fields should not change during validation
+            const identifiers = ['upc', 'supplier_no', 'notions_no'];
+            let matchedIndex = -1;
+            let matchedBy = null;
+            
+            for (const identifier of identifiers) {
+              if (changedProduct[identifier] !== undefined && changedProduct[identifier] !== null && changedProduct[identifier] !== '') {
+                matchedIndex = products.findIndex(
+                  (p) => p[identifier] !== undefined && 
+                         p[identifier] !== null &&
+                         p[identifier] !== '' &&
+                         String(p[identifier]).trim() === String(changedProduct[identifier]).trim()
+                );
+                if (matchedIndex !== -1) {
+                  matchedBy = identifier;
+                  console.log(`✓ Matched product by ${identifier}:`, changedProduct[identifier]);
+                  break;
+                }
+              }
             }
             
-          products.forEach((original, index) => {
-            const corrected = aiResponse.correctedData[index];
-            if (corrected) {
+            // If no identifier match found, log an error with details
+            if (matchedIndex === -1) {
+              console.error("❌ Could not match changed product to original. Product identifiers:", {
+                upc: changedProduct.upc,
+                supplier_no: changedProduct.supplier_no,
+                notions_no: changedProduct.notions_no
+              });
+              return;
+            }
+            
+            const original = products[matchedIndex];
             const productChanges = {
-                productIndex: index,
-                title: original.name || original.title || `Product ${index + 1}`,
+              productIndex: matchedIndex,
+              title: original.name || original.title || `Product ${matchedIndex + 1}`,
               changes: []
             };
             
-              const changes = Object.keys(corrected).filter(
-                (key) =>
-                  JSON.stringify(original[key]) !==
-                  JSON.stringify(corrected[key])
-              );
+            // Apply each changed field to the merged product
+            Object.keys(changedProduct).forEach((key) => {
+              // Check if the value actually changed
+              if (JSON.stringify(original[key]) !== JSON.stringify(changedProduct[key])) {
+                console.log(`\nProduct ${matchedIndex + 1} - Field ${key}:`);
+                console.log(`  - Original: ${JSON.stringify(original[key])}`);
+                console.log(`  - Corrected: ${JSON.stringify(changedProduct[key])}`);
                 
-              if (changes.length > 0) {
-                console.log(`\nProduct ${index + 1} changes:`);
-                changes.forEach((key) => {
-                  console.log(`  ${key}:`);
-                  console.log(
-                    `    - Original: ${JSON.stringify(original[key])}`
-                  );
-                  console.log(
-                    `    - Corrected: ${JSON.stringify(corrected[key])}`
-                  );
+                // Apply the change to merged product
+                mergedProducts[matchedIndex][key] = changedProduct[key];
                 
-                  // Add to our detailed changes array
+                // Track the change
                 productChanges.changes.push({
                   field: key,
                   original: original[key],
-                    corrected: corrected[key]
+                  corrected: changedProduct[key]
                 });
+              }
             });
             
-                // Only add products that have changes
+            // Only add to changeDetails if there were actual changes
             if (productChanges.changes.length > 0) {
               changeDetails.push(productChanges);
             }
-              }
-            }
           });
+          
+          console.log(`📊 Applied changes to ${changeDetails.length} products`);
         }
         
+        // Replace aiResponse.correctedData with the fully merged product array
+        aiResponse.correctedData = mergedProducts;
+
         // Record performance metrics after successful validation
         const endTime = new Date();
         let performanceMetrics = {
@@ -1242,26 +1226,15 @@ router.post("/validate", async (req, res) => {
             }));
 
             // Set prompt sources
-            if (generalPromptResult.rows.length > 0) {
+            if (generalPromptResult.rows.length > 0 && systemPromptResult.rows.length > 0) {
               const generalPrompt = generalPromptResult.rows[0];
-              let systemPrompt = null;
-
-              if (systemPromptResult.rows.length > 0) {
-                systemPrompt = systemPromptResult.rows[0];
-              }
+              const systemPrompt = systemPromptResult.rows[0];
 
               promptSources = {
-                ...(systemPrompt ? {
                 systemPrompt: {
                   id: systemPrompt.id,
                   prompt_text: systemPrompt.prompt_text
-                  }
-                } : {
-                  systemPrompt: {
-                    id: 0,
-                    prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
-                  }
-                }),
+                },
                 generalPrompt: {
                   id: generalPrompt.id,
                   prompt_text: generalPrompt.prompt_text

inventory/src/components/product-import/steps/ImageUploadStep/ImageUploadStep.tsx

@@ -186,7 +186,7 @@ export const ImageUploadStep = ({
     } finally {
       setIsSubmitting(false);
     }
-  }, [data, file, onSubmit, productImages]);
+  }, [data, file, onSubmit, productImages, targetEnvironment, useTestDataSource]);
 
   return (
     <div className="flex flex-col h-[calc(100vh-9.5rem)] overflow-hidden">

inventory/src/components/settings/PromptManagement.tsx

@@ -134,8 +134,12 @@ export function PromptManagement() {
       
       return response.json();
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (newPrompt) => {
+      // Optimistically update the cache with the new prompt
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [newPrompt];
+        return [...old, newPrompt];
+      });
       toast.success("Prompt created successfully");
       resetForm();
     },
@@ -163,8 +167,14 @@ export function PromptManagement() {
       
       return response.json();
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (updatedPrompt) => {
+      // Optimistically update the cache with the returned data
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [updatedPrompt];
+        return old.map((prompt) => 
+          prompt.id === updatedPrompt.id ? updatedPrompt : prompt
+        );
+      });
       toast.success("Prompt updated successfully");
       resetForm();
     },
@@ -181,9 +191,14 @@ export function PromptManagement() {
       if (!response.ok) {
         throw new Error("Failed to delete prompt");
       }
+      return id;
     },
-    onSuccess: () => {
-      queryClient.invalidateQueries({ queryKey: ["ai-prompts"] });
+    onSuccess: (deletedId) => {
+      // Optimistically update the cache by removing the deleted prompt
+      queryClient.setQueryData<AiPrompt[]>(["ai-prompts"], (old) => {
+        if (!old) return [];
+        return old.filter((prompt) => prompt.id !== deletedId);
+      });
       toast.success("Prompt deleted successfully");
     },
     onError: (error) => {