Enhance ai debug dialog

inventory-server/src/routes/ai-validation.js

@@ -373,8 +373,24 @@ async function generateDebugResponse(productsToUse, res) {
       });
 
       // Now use loadPrompt to get the actual combined prompt
-      const prompt = await loadPrompt(promptConnection, productsToUse, res.app.locals.pool);
+      const promptData = await loadPrompt(promptConnection, productsToUse, res.app.locals.pool);
-      const fullPrompt = prompt + "\n" + JSON.stringify(productsToUse);
+      const fullUserPrompt = promptData.userContent + "\n" + JSON.stringify(productsToUse);
+      const promptLength = promptData.systemInstructions.length + fullUserPrompt.length; // Store prompt length for performance metrics
+      console.log("📝 Generated prompt length:", promptLength);
+      console.log("📝 System instructions length:", promptData.systemInstructions.length);
+      console.log("📝 User content length:", fullUserPrompt.length);
+
+      // Format the messages as they would be sent to the API
+      const apiMessages = [
+        {
+          role: "system",
+          content: promptData.systemInstructions
+        },
+        {
+          role: "user",
+          content: fullUserPrompt
+        }
+      ];
 
       // Create the response with taxonomy stats
       let categoriesCount = 0;
@@ -415,8 +431,9 @@ async function generateDebugResponse(productsToUse, res) {
             }
           : null,
         basePrompt: systemPrompt ? systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text : generalPrompt.prompt_text,
-        sampleFullPrompt: fullPrompt,
+        sampleFullPrompt: fullUserPrompt,
-        promptLength: fullPrompt.length,
+        promptLength: promptLength,
+        apiFormat: apiMessages,
         promptSources: {
           ...(systemPrompt ? {
             systemPrompt: {
@@ -836,11 +853,14 @@ ${JSON.stringify(mixedTaxonomy.sizeCategories)}${
 
 ----------Here is the product data to validate----------`;
 
-      // Return the filtered prompt with system instructions first
+      // Return both system instructions and user content separately
-      return systemInstructions + "\n\n" + combinedPrompt + "\n" + taxonomySection;
+      return {
+        systemInstructions,
+        userContent: combinedPrompt + "\n" + taxonomySection
+      };
     }
 
-    // Generate the full unfiltered prompt
+    // Generate the full unfiltered prompt for taxonomy section
     const taxonomySection = `
 Available Categories:
 ${JSON.stringify(taxonomy.categories)}
@@ -868,8 +888,11 @@ ${JSON.stringify(taxonomy.artists)}
 
 Here is the product data to validate:`;
 
-    // Return the full prompt with system instructions first
+    // Return both system instructions and user content separately
-    return systemInstructions + "\n\n" + combinedPrompt + "\n" + taxonomySection;
+    return {
+      systemInstructions,
+      userContent: combinedPrompt + "\n" + taxonomySection
+    };
   } catch (error) {
     console.error("Error loading prompt:", error);
     throw error; // Re-throw to be handled by the calling function
@@ -917,18 +940,24 @@ router.post("/validate", async (req, res) => {
 
       // Load the prompt with the products data to filter taxonomy
       console.log("🔄 Loading prompt with filtered taxonomy...");
-      const prompt = await loadPrompt(connection, products, req.app.locals.pool);
+      const promptData = await loadPrompt(connection, products, req.app.locals.pool);
-      const fullPrompt = prompt + "\n" + JSON.stringify(products);
+      const fullUserPrompt = promptData.userContent + "\n" + JSON.stringify(products);
-      promptLength = fullPrompt.length; // Store prompt length for performance metrics
+      const promptLength = promptData.systemInstructions.length + fullUserPrompt.length; // Store prompt length for performance metrics
       console.log("📝 Generated prompt length:", promptLength);
+      console.log("📝 System instructions length:", promptData.systemInstructions.length);
+      console.log("📝 User content length:", fullUserPrompt.length);
 
       console.log("🤖 Sending request to OpenAI...");
       const completion = await openai.chat.completions.create({
-        model: "o3-mini",
+        model: "gpt-4o",
         messages: [
+          {
+            role: "system",
+            content: promptData.systemInstructions,
+          },
           {
             role: "user",
-            content: fullPrompt,
+            content: fullUserPrompt,
           },
         ],
         temperature: 0.2,

inventory/src/components/product-import/steps/ValidationStepNew/components/AiValidationDialogs.tsx

@@ -25,7 +25,6 @@ import {
 } from "../hooks/useAiValidation";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Badge } from "@/components/ui/badge";
-import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 
 interface TaxonomyStats {
   categories: number;
@@ -43,9 +42,14 @@ interface DebugData {
   basePrompt: string;
   sampleFullPrompt: string;
   promptLength: number;
+  apiFormat?: Array<{
+    role: string;
+    content: string;
+  }>;
   promptSources?: {
+    systemPrompt?: { id: number; prompt_text: string };
     generalPrompt?: { id: number; prompt_text: string };
-    companyPrompts?: Array<{ id: number; company: string; prompt_text: string }>;
+    companyPrompts?: Array<{ id: number; company: string; companyName?: string; prompt_text: string }>;
   };
   estimatedProcessingTime?: {
     seconds: number | null;
@@ -89,8 +93,20 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
   debugData,
 }) => {
   const [costPerMillionTokens, setCostPerMillionTokens] = useState(2.5); // Default cost
-  const [activeTab, setActiveTab] = useState("full");
+  const hasCompanyPrompts = currentPrompt.debugData?.promptSources?.companyPrompts && 
+                           currentPrompt.debugData.promptSources.companyPrompts.length > 0;
+
+  // Use "full" as the default tab
+  const defaultTab = "full";
+  const [activeTab, setActiveTab] = useState(defaultTab);
   
+  // Update activeTab when the dialog is opened with new data
+  React.useEffect(() => {
+    if (currentPrompt.isOpen) {
+      setActiveTab("full");
+    }
+  }, [currentPrompt.isOpen]);
+
   // Format time helper
   const formatTime = (seconds: number): string => {
     if (seconds < 60) {
@@ -110,10 +126,6 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
 
   // Use the prompt length from the current prompt
   const promptLength = currentPrompt.prompt ? currentPrompt.prompt.length : 0;
-  
-  // Check if we have company-specific prompts
-  const hasCompanyPrompts = currentPrompt.debugData?.promptSources?.companyPrompts && 
-                           currentPrompt.debugData.promptSources.companyPrompts.length > 0;
 
   return (
     <>
@@ -134,8 +146,8 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
           </DialogHeader>
 
           <div className="flex flex-col h-[calc(90vh-120px)] overflow-hidden">
-            {/* Debug Information Section */}
+            {/* Debug Information Section - Fixed at the top */}
-            <div className="mb-4 flex-shrink-0">
+            <div className="flex-shrink-0">
               {currentPrompt.isLoading ? (
                 <div className="flex justify-center items-center h-[100px]"></div>
               ) : (
@@ -249,38 +261,11 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
                       </CardContent>
                     </Card>
                   </div>
-                  
-                  {/* Prompt Sources Section */}
-                  {currentPrompt.debugData?.promptSources && (
-                    <Card className="py-2">
-                      <CardHeader className="py-2">
-                        <CardTitle className="text-base">
-                          Prompt Sources
-                        </CardTitle>
-                      </CardHeader>
-                      <CardContent className="py-2">
-                        <div className="flex flex-wrap gap-2">
-                          <Badge variant="outline" className="bg-purple-50">
-                            System
-                          </Badge>
-                          <Badge variant="outline" className="bg-green-50">
-                            General
-                          </Badge>
-                          
-                          {currentPrompt.debugData.promptSources.companyPrompts?.map((prompt, idx) => (
-                            <Badge key={idx} variant="outline" className="bg-blue-50">
-                              {prompt.companyName}
-                            </Badge>
-                          ))}
-                        </div>
-                      </CardContent>
-                    </Card>
-                  )}
                 </>
               )}
             </div>
 
-            {/* Prompt Section */}
+            {/* Prompt Section - Scrollable content */}
             <div className="flex-1 min-h-0">
               {currentPrompt.isLoading ? (
                 <div className="flex items-center justify-center h-full">
@@ -288,61 +273,186 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
                 </div>
               ) : (
                 <>
-                  {currentPrompt.debugData?.promptSources ? (
+                  {currentPrompt.debugData?.apiFormat ? (
-                    <Tabs 
+                    <div className="flex flex-col h-full">
-                      defaultValue="full" 
+                      {/* Prompt Sources Card - Fixed at the top of the content area */}
-                      value={activeTab}
+                      <Card className="py-2 mb-4 flex-shrink-0">
-                      onValueChange={setActiveTab}
+                        <CardHeader className="py-2">
-                      className="h-full flex flex-col"
+                          <CardTitle className="text-base">Prompt Sources</CardTitle>
-                    >
+                        </CardHeader>
-                      <TabsList className="mb-2 w-fit">
+                        <CardContent className="py-2">
-                        <TabsTrigger value="full">Full Prompt</TabsTrigger>
+                          <div className="flex flex-wrap gap-2">
-                        <TabsTrigger value="system">System</TabsTrigger>
+                            <Badge 
-                        <TabsTrigger value="general">General Prompt</TabsTrigger>
+                              variant="outline" 
-                        {hasCompanyPrompts && (
+                              className="bg-purple-100 hover:bg-purple-200 cursor-pointer"
-                          <TabsTrigger value="company">Company Prompts</TabsTrigger>
+                              onClick={() => document.getElementById('system-message')?.scrollIntoView({ behavior: 'smooth' })}
-                        )}
+                            >
-                      </TabsList>
+                              System
+                            </Badge>
+                            <Badge 
+                              variant="outline" 
+                              className="bg-green-100 hover:bg-green-200 cursor-pointer"
+                              onClick={() => document.getElementById('general-section')?.scrollIntoView({ behavior: 'smooth' })}
+                            >
+                              General
+                            </Badge>
+                            
+                            {currentPrompt.debugData.promptSources?.companyPrompts?.map((company, idx) => (
+                              <Badge 
+                                key={idx}
+                                variant="outline" 
+                                className="bg-blue-100 hover:bg-blue-200 cursor-pointer"
+                                onClick={() => document.getElementById('company-section')?.scrollIntoView({ behavior: 'smooth' })}
+                              >
+                                {company.companyName || `Company ${company.company}`}
+                              </Badge>
+                            ))}
+                            
+                            <Badge 
+                              variant="outline" 
+                              className="bg-amber-100 hover:bg-amber-200 cursor-pointer"
+                              onClick={() => document.getElementById('taxonomy-section')?.scrollIntoView({ behavior: 'smooth' })}
+                            >
+                              Taxonomy
+                            </Badge>
+                            <Badge 
+                              variant="outline" 
+                              className="bg-pink-100 hover:bg-pink-200 cursor-pointer"
+                              onClick={() => document.getElementById('product-section')?.scrollIntoView({ behavior: 'smooth' })}
+                            >
+                              Products
+                            </Badge>
+                          </div>
+                        </CardContent>
+                      </Card>
                       
-                      <div className="flex-1 min-h-0">
+                      <ScrollArea className="flex-1 w-full overflow-y-auto">
-                        <ScrollArea className="h-full w-full">
+                        {currentPrompt.debugData.apiFormat.map((message, idx: number) => (
-                          <TabsContent value="full" className="m-0 p-0 h-full">
+                          <div key={idx} className="border rounded-md p-2 mb-4">
-                            <Code className="whitespace-pre-wrap p-4 break-normal max-w-full">
+                            <div 
-                              {currentPrompt.prompt}
+                              id={message.role === 'system' ? 'system-message' : ''}
+                              className={`p-2 mb-2 rounded-sm font-medium ${
+                                message.role === 'system' 
+                                  ? 'bg-purple-50 text-purple-800' 
+                                  : 'bg-green-50 text-green-800'
+                              }`}
+                            >
+                              Role: {message.role}
+                            </div>
+                            
+                            <Code className={`whitespace-pre-wrap p-4 break-normal max-w-full ${
+                              message.role === 'system'
+                                ? 'bg-purple-50/30'
+                                : 'bg-green-50/30'
+                            }`}>
+                              {message.role === 'user' ? (
+                                <div className="text-wrapper">
+                                  {(() => {
+                                    const content = message.content;
+                                    
+                                    // Find section boundaries by looking for specific markers
+                                    const companySpecificStartIndex = content.indexOf('--- COMPANY-SPECIFIC INSTRUCTIONS ---');
+                                    const companySpecificEndIndex = content.indexOf('--- END COMPANY-SPECIFIC INSTRUCTIONS ---');
+                                    
+                                    const taxonomyStartIndex = content.indexOf('All Available Categories:');
+                                    const taxonomyFallbackStartIndex = content.indexOf('Available Categories:');
+                                    const actualTaxonomyStartIndex = taxonomyStartIndex >= 0 ? taxonomyStartIndex : taxonomyFallbackStartIndex;
+                                    
+                                    const productDataStartIndex = content.indexOf('----------Here is the product data to validate----------');
+                                    
+                                    // If we can't find any markers, just return the content as-is
+                                    if (actualTaxonomyStartIndex < 0 && productDataStartIndex < 0 && companySpecificStartIndex < 0) {
+                                      return content;
+                                    }
+                                    
+                                    // Determine section indices
+                                    let generalEndIndex = content.length;
+                                    
+                                    if (companySpecificStartIndex >= 0) {
+                                      generalEndIndex = companySpecificStartIndex;
+                                    } else if (actualTaxonomyStartIndex >= 0) {
+                                      generalEndIndex = actualTaxonomyStartIndex;
+                                    } else if (productDataStartIndex >= 0) {
+                                      generalEndIndex = productDataStartIndex;
+                                    }
+                                    
+                                    // Determine where taxonomy starts
+                                    let taxonomyEndIndex = content.length;
+                                    if (productDataStartIndex >= 0) {
+                                      taxonomyEndIndex = productDataStartIndex;
+                                    }
+                                    
+                                    // Segments to render with appropriate styling
+                                    const segments = [];
+                                    
+                                    // General section (beginning to company/taxonomy/product)
+                                    if (generalEndIndex > 0) {
+                                      segments.push(
+                                        <div id="general-section" key="general" className="border-l-4 border-green-500 pl-4 py-0 my-1">
+                                          <div className="text-xs font-semibold text-green-700 mb-2">
+                                            General Prompt
+                                          </div>
+                                          <pre className="whitespace-pre-wrap">
+                                            {content.substring(0, generalEndIndex)}
+                                          </pre>
+                                        </div>
+                                      );
+                                    }
+                                    
+                                    // Company-specific section if present
+                                    if (companySpecificStartIndex >= 0 && companySpecificEndIndex >= 0) {
+                                      segments.push(
+                                        <div id="company-section" key="company" className="border-l-4 border-blue-500 pl-4 py-0 my-1">
+                                          <div className="text-xs font-semibold text-blue-700 mb-2">
+                                            Company-Specific Instructions
+                                          </div>
+                                          <pre className="whitespace-pre-wrap">
+                                            {content.substring(companySpecificStartIndex, companySpecificEndIndex + '--- END COMPANY-SPECIFIC INSTRUCTIONS ---'.length)}
+                                          </pre>
+                                        </div>
+                                      );
+                                    }
+                                    
+                                    // Taxonomy section
+                                    if (actualTaxonomyStartIndex >= 0) {
+                                      const taxEnd = taxonomyEndIndex;
+                                      segments.push(
+                                        <div id="taxonomy-section" key="taxonomy" className="border-l-4 border-amber-500 pl-4 py-0 my-1">
+                                          <div className="text-xs font-semibold text-amber-700 mb-2">
+                                            Taxonomy Data
+                                          </div>
+                                          <pre className="whitespace-pre-wrap">
+                                            {content.substring(actualTaxonomyStartIndex, taxEnd)}
+                                          </pre>
+                                        </div>
+                                      );
+                                    }
+                                    
+                                    // Product data section
+                                    if (productDataStartIndex >= 0) {
+                                      segments.push(
+                                        <div id="product-section" key="product" className="border-l-4 border-pink-500 pl-4 py-0 my-1">
+                                          <div className="text-xs font-semibold text-pink-700 mb-2">
+                                            Product Data
+                                          </div>
+                                          <pre className="whitespace-pre-wrap">
+                                            {content.substring(productDataStartIndex)}
+                                          </pre>
+                                        </div>
+                                      );
+                                    }
+                                    
+                                    return <>{segments}</>;
+                                  })()}
+                                </div>
+                              ) : (
+                                <pre className="whitespace-pre-wrap">{message.content}</pre>
+                              )}
                             </Code>
-                          </TabsContent>
+                          </div>
-                          
+                        ))}
-                          <TabsContent value="system" className="m-0 p-0 h-full">
+                      </ScrollArea>
-                            <Code className="whitespace-pre-wrap p-4 break-normal max-w-full">
+                    </div>
-                              {currentPrompt.debugData.promptSources.systemPrompt?.prompt_text || "No system prompt available"}
-                            </Code>
-                          </TabsContent>
-                          
-                          <TabsContent value="general" className="m-0 p-0 h-full">
-                            <Code className="whitespace-pre-wrap p-4 break-normal max-w-full">
-                              {currentPrompt.debugData.promptSources.generalPrompt?.prompt_text || "No general prompt available"}
-                            </Code>
-                          </TabsContent>
-                          
-                          {hasCompanyPrompts && (
-                            <TabsContent value="company" className="m-0 p-0 h-full">
-                              <div className="space-y-4">
-                                {currentPrompt.debugData.promptSources.companyPrompts?.map((prompt, idx) => (
-                                  <div key={idx} className="border rounded-md p-2">
-                                    <div className="bg-muted p-2 mb-2 rounded-sm">
-                                      <strong>{prompt.companyName}</strong> <span className="text-sm text-muted-foreground">(ID: {prompt.company})</span>
-                                    </div>
-                                    <Code className="whitespace-pre-wrap p-4 break-normal max-w-full">
-                                      {prompt.prompt_text}
-                                    </Code>
-                                  </div>
-                                ))}
-                              </div>
-                            </TabsContent>
-                          )}
-                        </ScrollArea>
-                      </div>
-                    </Tabs>
                   ) : (
                     <ScrollArea className="h-full w-full">
                       <Code className="whitespace-pre-wrap p-4 break-normal max-w-full">
@@ -379,8 +489,9 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
                     className="h-full bg-primary transition-all duration-500"
                     style={{
                       width: `${
-                        aiValidationProgress.progressPercent ??
+                        aiValidationProgress.progressPercent !== undefined
-                        Math.round((aiValidationProgress.step / 5) * 100)
+                          ? Math.round(aiValidationProgress.progressPercent)
+                          : Math.round((aiValidationProgress.step / 5) * 100)
                       }%`,
                       backgroundColor:
                         aiValidationProgress.step === -1
@@ -394,8 +505,9 @@ export const AiValidationDialogs: React.FC<AiValidationDialogsProps> = ({
                 {aiValidationProgress.step === -1
                   ? "❌"
                   : `${
-                      aiValidationProgress.progressPercent ??
+                      aiValidationProgress.progressPercent !== undefined
-                      Math.round((aiValidationProgress.step / 5) * 100)
+                        ? Math.round(aiValidationProgress.progressPercent)
+                        : Math.round((aiValidationProgress.step / 5) * 100)
                     }%`}
               </div>
             </div>

inventory/src/components/product-import/steps/ValidationStepNew/hooks/useAiValidation.tsx

@@ -56,6 +56,10 @@ export interface CurrentPrompt {
     basePrompt: string;
     sampleFullPrompt: string;
     promptLength: number;
+    apiFormat?: Array<{
+      role: string;
+      content: string;
+    }>;
     promptSources?: {
       systemPrompt?: { id: number; prompt_text: string };
       generalPrompt?: { id: number; prompt_text: string };
@@ -334,7 +338,8 @@ export const useAiValidation = <T extends string>(
             sampleFullPrompt: result.sampleFullPrompt || '',
             promptLength: result.promptLength || (promptContent ? promptContent.length : 0),
             promptSources: result.promptSources,
-            estimatedProcessingTime: result.estimatedProcessingTime
+            estimatedProcessingTime: result.estimatedProcessingTime,
+            apiFormat: result.apiFormat
           }
         }));
       } else {