Fix line/subline regressions, add in AI validation tracking and improve AI results dialog

inventory-server/db/setup-schema.sql

@@ -23,6 +23,20 @@ CREATE TABLE IF NOT EXISTS templates (
     UNIQUE(company, product_type)
 );
 
+-- AI Validation Performance Tracking
+CREATE TABLE IF NOT EXISTS ai_validation_performance (
+    id SERIAL PRIMARY KEY,
+    prompt_length INTEGER NOT NULL,
+    product_count INTEGER NOT NULL,
+    start_time TIMESTAMP WITH TIME ZONE NOT NULL,
+    end_time TIMESTAMP WITH TIME ZONE NOT NULL,
+    duration_seconds DECIMAL(10,2) GENERATED ALWAYS AS (EXTRACT(EPOCH FROM (end_time - start_time))) STORED,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Create index on prompt_length for efficient querying
+CREATE INDEX IF NOT EXISTS idx_ai_validation_prompt_length ON ai_validation_performance(prompt_length);
+
 -- Function to update the updated_at timestamp
 CREATE OR REPLACE FUNCTION update_updated_at_column()
 RETURNS TRIGGER AS $$

inventory-server/src/prompts/product-validation.txt

@@ -7,6 +7,8 @@ Your response should be a JSON object with the following structure:
   "warnings": [] // Array of strings with warnings or suggestions for manual review (see below for details)
 }
 
+IMPORTANT: For all fields that use IDs (categories, supplier, company, line, subline, ship_restrictions, tax_cat, artist, themes, etc.), you MUST return the ID values, not the display names. The system will handle converting IDs to display names.
+
 Using the provided guidelines, focus on:
 1. Correcting typos and any incorrect spelling or grammar
 2. Standardizing product names
@@ -93,7 +95,7 @@ Instructions: Always return a valid numerical tax code ID from the Available Tax
 Fields: size_cat
 Changes: Allowed to correct obvious errors or inconsistencies or to add missing values
 Required: Return if present in the original data or if not present and applicable. Do not return if not applicable (e.g. if no size categories apply based on what you know about the product).
-Instructions: If present or if applicable, return one valid numerical size category ID from the Available Size Categories array below. Give preference to the value provided, but correct it if another value is more accurate. A value is not required if none of the size categories apply, but it's important to include if one clearly applies, such as if the name contains 12x12, 6x8, 2oz, etc.
+Instructions: If present or if applicable, return one valid numerical size category ID from the Available Size Categories array below. Give preference to the value provided, but correct it if another value is more accurate. If the product name contains a match for one of the size categories (such as 12x12, 6x6, 2oz, etc) you MUST return that size category with the results. A value is not required if none of the size categories apply.
 
 Fields: themes
 Changes: Allowed to correct obvious errors or inconsistencies or to add missing values

inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx

@@ -171,16 +171,34 @@ const EditableCell = memo(({ value, onChange, error, field, productLines, sublin
 
   // Determine if the field should be disabled based on its key and context
   const isFieldDisabled = useMemo(() => {
+    // If the field is already disabled by the parent component, respect that
+    if (field.disabled) return true;
+    
+    // Special handling for line and subline fields
     if (field.key === 'line') {
-      // Enable the line field if we have product lines available
-      return !productLines || productLines.length === 0;
+      // Never disable line field if it already has a value
+      if (value) return false;
+      
+      // The line field should be enabled if:
+      // 1. We have a company selected (even if product lines are still loading)
+      // 2. We have product lines available
+      return !((field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') && 
+        field.fieldType.options?.length) && !productLines?.length;
     }
     if (field.key === 'subline') {
-      // Enable subline field if we have sublines available
-      return !sublines || sublines.length === 0;
+      // Never disable subline field if it already has a value
+      if (value) return false;
+      
+      // The subline field should be enabled if:
+      // 1. We have a line selected (even if sublines are still loading)
+      // 2. We have sublines available
+      return !((field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') && 
+        field.fieldType.options?.length) && !sublines?.length;
     }
+    
+    // For other fields, use the disabled property
     return field.disabled;
-  }, [field.key, field.disabled, productLines, sublines]);
+  }, [field.key, field.disabled, field.fieldType, productLines, sublines, value]);
 
   // For debugging
   useEffect(() => {
@@ -269,21 +287,71 @@ const EditableCell = memo(({ value, onChange, error, field, productLines, sublin
     if (fieldType.type === "select" || fieldType.type === "multi-select") {
       if (fieldType.type === "select") {
         // For line and subline fields, ensure we're using the latest options
-        if (field.key === 'line' && productLines?.length) {
-          const option = productLines.find((opt: SelectOption) => opt.value === value);
-          return option?.label || value;
+        if (field.key === 'line') {
+          // Log current state for debugging
+          console.log('Getting display value for line:', {
+            value,
+            productLines: productLines?.length ?? 0,
+            options: fieldType.options?.length ?? 0
+          });
+          
+          // First try to find in productLines if available
+          if (productLines?.length) {
+            const matchingOption = productLines.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (matchingOption) {
+              console.log('Found line in productLines:', value, '->', matchingOption.label);
+              return matchingOption.label;
+            }
+          }
+          // Fall back to fieldType options if productLines not available yet
+          if (fieldType.options?.length) {
+            const fallbackOptionLine = fieldType.options.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (fallbackOptionLine) {
+              console.log('Found line in fallback options:', value, '->', fallbackOptionLine.label);
+              return fallbackOptionLine.label;
+            }
+          }
+          console.log('Unable to find display value for line:', value);
+          return value;
         }
-        if (field.key === 'subline' && sublines?.length) {
-          const option = sublines.find((opt: SelectOption) => opt.value === value);
-          return option?.label || value;
+        if (field.key === 'subline') {
+          // Log current state for debugging
+          console.log('Getting display value for subline:', {
+            value,
+            sublines: sublines?.length ?? 0,
+            options: fieldType.options?.length ?? 0
+          });
+          
+          // First try to find in sublines if available
+          if (sublines?.length) {
+            const matchingOption = sublines.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (matchingOption) {
+              console.log('Found subline in sublines:', value, '->', matchingOption.label);
+              return matchingOption.label;
+            }
+          }
+          // Fall back to fieldType options if sublines not available yet
+          if (fieldType.options?.length) {
+            const fallbackOptionSubline = fieldType.options.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (fallbackOptionSubline) {
+              console.log('Found subline in fallback options:', value, '->', fallbackOptionSubline.label);
+              return fallbackOptionSubline.label;
+            }
+          }
+          console.log('Unable to find display value for subline:', value);
+          return value;
         }
-        return fieldType.options.find((opt: SelectOption) => opt.value === value)?.label || value;
+        return fieldType.options?.find((opt: SelectOption) => String(opt.value) === String(value))?.label || value;
       }
       if (Array.isArray(value)) {
         const options = field.key === 'line' && productLines?.length ? productLines :
                        field.key === 'subline' && sublines?.length ? sublines :
                        fieldType.options;
-        return value.map(v => options.find((opt: SelectOption) => opt.value === v)?.label || v).join(", ");
+        return value.map(v => options.find((opt: SelectOption) => String(opt.value) === String(v))?.label || v).join(", ");
       }
       return value;
     }
@@ -881,21 +949,23 @@ function useTemplates<T extends string>(
       const { __index, __errors, __template, ...templateData } = selectedRow;
 
       // Clean numeric values and prepare template data
-      const cleanedData = Object.entries(templateData).reduce((acc, [key, value]) => {
+      const cleanedData: Record<string, any> = {};
+      
+      // Process each key-value pair
+      Object.entries(templateData).forEach(([key, value]) => {
         // Handle numeric values with dollar signs
         if (typeof value === 'string' && value.includes('$')) {
-          acc[key] = value.replace(/[$,\s]/g, '').trim();
+          cleanedData[key] = value.replace(/[$,\s]/g, '').trim();
         } 
         // Handle array values (like categories or ship_restrictions)
         else if (Array.isArray(value)) {
-          acc[key] = value;
+          cleanedData[key] = value;
         }
         // Handle other values
         else {
-          acc[key] = value;
+          cleanedData[key] = value;
         }
-        return acc;
-      }, {} as Record<string, any>);
+      });
 
       // Log the cleaned data before sending
       console.log('Saving template with cleaned data:', {
@@ -1049,6 +1119,19 @@ const SaveTemplateDialog = memo(({
   );
 });
 
+// Add a new interface to handle the AI validation details response
+interface ChangeDetail {
+  field: string;
+  original: any;
+  corrected: any;
+}
+
+interface ProductChangeDetail {
+  productIndex: number;
+  title: string;
+  changes: ChangeDetail[];
+}
+
 export const ValidationStep = <T extends string>({ 
   initialData,
   file,
@@ -1083,13 +1166,18 @@ export const ValidationStep = <T extends string>({
     queryKey: ["sublines", globalSelections?.line],
     queryFn: async () => {
       if (!globalSelections?.line) return [];
+      console.log('Fetching sublines for line:', globalSelections.line);
       const response = await fetch(`${config.apiUrl}/import/sublines/${globalSelections.line}`);
       if (!response.ok) {
+        console.error('Failed to fetch sublines:', response.status, response.statusText);
         throw new Error("Failed to fetch sublines");
       }
-      return response.json();
+      const data = await response.json();
+      console.log('Received sublines:', data);
+      return data;
     },
     enabled: !!globalSelections?.line,
+    staleTime: 30000, // Cache for 30 seconds
   });
 
   // Apply global selections to initial data and validate
@@ -1163,7 +1251,9 @@ export const ValidationStep = <T extends string>({
             ...field.fieldType,
             options: productLines || (field.fieldType.type === 'select' ? field.fieldType.options : []),
           },
-          disabled: (!productLines || productLines.length === 0) && !globalSelections?.line
+          // Only disable if no company is selected or if product lines failed to load
+          // when a company is selected
+          disabled: !globalSelections?.company || (globalSelections?.company && productLines !== undefined && productLines.length === 0)
         } as Field<T>;
       }
       if (field.key === 'subline') {
@@ -1179,7 +1269,7 @@ export const ValidationStep = <T extends string>({
       }
       return field;
     });
-  }, [fields, productLines, sublines, globalSelections?.line]);
+  }, [fields, productLines, sublines, globalSelections?.company, globalSelections?.line]);
 
   const [data, setData] = useState<RowData<T>[]>(initialDataWithGlobals);
 
@@ -1208,10 +1298,13 @@ export const ValidationStep = <T extends string>({
   const [aiValidationDetails, setAiValidationDetails] = useState<{
     changes: string[];
     warnings: string[];
+    changeDetails: ProductChangeDetail[];
     isOpen: boolean;
+    originalData?: (Data<T> & ExtendedMeta)[];  // Store original data for reverting changes
   }>({
     changes: [],
     warnings: [],
+    changeDetails: [],
     isOpen: false,
   });
 
@@ -1219,6 +1312,11 @@ export const ValidationStep = <T extends string>({
     isOpen: boolean;
     status: string;
     step: number;
+    estimatedSeconds?: number;
+    startTime?: Date;
+    promptLength?: number;
+    elapsedSeconds?: number;
+    progressPercent?: number;
   }>({
     isOpen: false,
     status: "",
@@ -1259,15 +1357,18 @@ export const ValidationStep = <T extends string>({
     async (rows: (Data<T> & ExtendedMeta)[], indexes?: number[]) => {
       setData(rows);
       
+      // Use fieldsWithUpdatedOptions to ensure we have the latest field definitions with proper options
+      const currentFields = fieldsWithUpdatedOptions as unknown as Fields<T>;
+      
       if (rowHook?.constructor.name === "AsyncFunction" || tableHook?.constructor.name === "AsyncFunction") {
-        const updatedData = await addErrorsAndRunHooks<T>(rows, fields, rowHook, tableHook, indexes);
+        const updatedData = await addErrorsAndRunHooks<T>(rows, currentFields, rowHook, tableHook, indexes);
         setData(updatedData as (Data<T> & ExtendedMeta)[]);
       } else {
-        const result = await addErrorsAndRunHooks<T>(rows, fields, rowHook, tableHook, indexes);
+        const result = await addErrorsAndRunHooks<T>(rows, currentFields, rowHook, tableHook, indexes);
         setData(result as (Data<T> & ExtendedMeta)[]);
       }
     },
-    [rowHook, tableHook, fields],
+    [rowHook, tableHook, fieldsWithUpdatedOptions],
   );
 
   const updateRows = useCallback(
@@ -1544,20 +1645,97 @@ export const ValidationStep = <T extends string>({
     }
   }, [data, submitData]);
 
-  // Add AI validation function
+  // Update the AI validation function
   const handleAiValidation = async () => {
     try {
+      if (isAiValidating) return;
+      
+      // Store the original data before any changes
+      const originalDataCopy = [...data];
+      
       setIsAiValidating(true);
+      const startTime = new Date();
       setAiValidationProgress({
         isOpen: true,
         status: "Preparing data for validation...",
-        step: 1
+        step: 1,
+        startTime,
+        elapsedSeconds: 0,
+        progressPercent: 0,
+        ...(aiValidationProgress.estimatedSeconds ? {
+          estimatedSeconds: aiValidationProgress.estimatedSeconds,
+          promptLength: aiValidationProgress.promptLength
+        } : {})
       });
       console.log('Sending data for validation:', data);
 
+      // Set up an interval to update progress
+      window.aiValidationTimer = setInterval(() => {
+        const now = new Date();
+        const elapsedSeconds = Math.floor((now.getTime() - startTime.getTime()) / 1000);
+        
+        setAiValidationProgress(prev => {
+          // Calculate progress percentage
+          let progressPercent = 0;
+          if (prev.estimatedSeconds && prev.estimatedSeconds > 0) {
+            // Cap at 99% if we exceed estimated time but aren't done yet
+            progressPercent = Math.min(99, Math.floor((elapsedSeconds / prev.estimatedSeconds) * 100));
+          } else {
+            // If no estimate, use step-based progress (25% per step), also capped at 99%
+            progressPercent = Math.min(99, (prev.step * 25) + Math.min(24, Math.floor((elapsedSeconds % 10) / 10 * 25)));
+          }
+          
+          // Extract the base status message without any time information
+          const baseStatus = prev.status.replace(/\s\(\d+[ms].+\)$/, '').replace(/\s\(\d+m \d+s.+\)$/, '');
+          
+          return {
+            ...prev,
+            elapsedSeconds,
+            progressPercent,
+            // Just use the base status message without time information
+            status: baseStatus
+          };
+        });
+      }, 1000);
+
+      // Clean the data to ensure we only send what's needed
+      const cleanedData = data.map(item => {
+        const { __errors, __index, ...cleanProduct } = item;
+        return cleanProduct;
+      });
+
+      console.log('Cleaned data for validation:', cleanedData);
+
+      // If we don't have an estimated time yet, try to get one
+      if (!aiValidationProgress.estimatedSeconds) {
+        try {
+          const debugResponse = await fetch(`${config.apiUrl}/ai-validation/debug`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ products: cleanedData })
+          });
+          
+          if (debugResponse.ok) {
+            const debugData = await debugResponse.json();
+            if (debugData.estimatedProcessingTime?.seconds) {
+              setAiValidationProgress(prev => ({
+                ...prev,
+                estimatedSeconds: debugData.estimatedProcessingTime.seconds,
+                promptLength: debugData.promptLength
+              }));
+            }
+          }
+        } catch (estimateError) {
+          console.error('Error getting time estimate:', estimateError);
+          // Continue without an estimate
+        }
+      }
+
       setAiValidationProgress(prev => ({
         ...prev,
-        status: "Sending data to AI service and awaiting response...",
+        status: "Sending data to AI service...",
         step: 2
       }));
 
@@ -1566,19 +1744,19 @@ export const ValidationStep = <T extends string>({
         headers: {
           'Content-Type': 'application/json',
         },
-        body: JSON.stringify({ products: data }),
+        body: JSON.stringify({ products: cleanedData }),
       });
 
       if (!response.ok) {
-        throw new Error('AI validation failed');
+        const errorText = await response.text();
+        console.error('AI validation error response:', {
+          status: response.status,
+          statusText: response.statusText,
+          body: errorText
+        });
+        throw new Error(`AI validation failed: ${response.status} ${response.statusText}`);
       }
 
-      setAiValidationProgress(prev => ({
-        ...prev,
-        status: "Processing AI response...",
-        step: 3
-      }));
-
       const result = await response.json();
       console.log('AI validation response:', result);
       
@@ -1586,51 +1764,172 @@ export const ValidationStep = <T extends string>({
         throw new Error(result.error || 'AI validation failed');
       }
 
+      // Update progress with actual processing time if available
+      if (result.performanceMetrics) {
+        console.log('Performance metrics:', result.performanceMetrics);
+        setAiValidationProgress(prev => ({
+          ...prev,
+          status: "Processing AI response...",
+          step: 3,
+          // Update with actual metrics from the server
+          estimatedSeconds: result.performanceMetrics.processingTimeSeconds || prev.estimatedSeconds,
+          promptLength: result.performanceMetrics.promptLength || prev.promptLength,
+          progressPercent: 75 // 75% complete when we're processing the AI response
+        }));
+      } else {
+        setAiValidationProgress(prev => ({
+          ...prev,
+          status: "Processing AI response...",
+          step: 3,
+          progressPercent: 75
+        }));
+      }
+
       setAiValidationProgress(prev => ({
         ...prev,
         status: "Applying corrections...",
-        step: 4
+        step: 4,
+        progressPercent: 90 // 90% complete when applying corrections
       }));
 
       // Update the data with AI suggestions
       if (result.correctedData && Array.isArray(result.correctedData)) {
-        // Log the differences
-        data.forEach((original, index) => {
-          const corrected = result.correctedData[index];
-          if (corrected) {
-            const changes = Object.keys(corrected).filter(key => {
-              const originalValue = original[key as keyof typeof original];
-              const correctedValue = corrected[key as keyof typeof corrected];
-              return JSON.stringify(originalValue) !== JSON.stringify(correctedValue);
-            });
-            if (changes.length > 0) {
-              console.log(`Changes for row ${index + 1}:`, changes.map(key => ({
-                field: key,
-                original: original[key as keyof typeof original],
-                corrected: corrected[key as keyof typeof corrected]
-              })));
+        // Process data to properly handle comma-separated values for multi-select fields
+        const processedData = result.correctedData.map((corrected: any) => {
+          const processed = { ...corrected };
+          
+          // Process each field
+          Object.keys(processed).forEach(key => {
+            const fieldConfig = fields.find(f => f.key === key);
+            
+            // Handle multi-select fields (comma-separated values)
+            if (fieldConfig?.fieldType.type === 'multi-select' && typeof processed[key] === 'string') {
+              // Split comma-separated values and trim each value
+              processed[key] = processed[key].split(',').map((v: string) => v.trim()).filter(Boolean);
             }
-          }
+            
+            // For select and multi-select fields, ensure we're working with IDs
+            // We don't convert IDs to display names here because we want to preserve IDs in the data
+            if (fieldConfig?.fieldType.type === 'select' || fieldConfig?.fieldType.type === 'multi-select') {
+              const options = fieldConfig.fieldType.options || [];
+              
+              // If the value is a string that matches a label but not a value, convert it to the corresponding ID
+              if (!Array.isArray(processed[key]) && typeof processed[key] === 'string') {
+                // Check if the value is already an ID
+                const isAlreadyId = options.some(opt => String(opt.value) === String(processed[key]));
+                
+                if (!isAlreadyId) {
+                  // Try to find the option by label
+                  const matchingOption = options.find(opt => opt.label === processed[key]);
+                  if (matchingOption) {
+                    // Convert label to ID
+                    const originalValue = processed[key];
+                    processed[key] = matchingOption.value;
+                    console.log(`Converted label "${originalValue}" to ID "${matchingOption.value}" for field "${key}"`);
+                  }
+                }
+              }
+              
+              // Handle array of values (multi-select)
+              if (Array.isArray(processed[key])) {
+                processed[key] = processed[key].map((val: string | number) => {
+                  // Check if the value is already an ID
+                  const isAlreadyId = options.some(opt => String(opt.value) === String(val));
+                  
+                  if (!isAlreadyId) {
+                    // Try to find the option by label
+                    const matchingOption = options.find(opt => opt.label === val);
+                    if (matchingOption) {
+                      // Convert label to ID
+                      return matchingOption.value;
+                    }
+                  }
+                  return val;
+                });
+              }
+            }
+          });
+          
+          return processed;
         });
-
+        
         // Preserve the __index and __errors from the original data
-        const newData = result.correctedData.map((item: any, idx: number) => ({
+        const newData = processedData.map((item: any, idx: number) => ({
           ...item,
           __index: data[idx]?.__index,
           __errors: data[idx]?.__errors,
         }));
         
-        // Update the data and run validations
-        await updateData(newData);
+        console.log('About to update data with AI corrections:', {
+          originalDataSample: data.slice(0, 2),
+          newDataSample: newData.slice(0, 2),
+          correctionCount: result.changes?.length || 0
+        });
+        
+        // First validate the new data to ensure all validation rules are applied
+        try {
+          // Use the current fields with updated options for validation
+          const currentFields = fieldsWithUpdatedOptions as unknown as Fields<T>;
+          
+          // Validate the data with the hooks
+          const validatedData = await addErrorsAndRunHooks<T>(
+            newData,
+            currentFields,
+            rowHook,
+            tableHook
+          );
+          
+          // Update the component state with the validated data
+          setData(validatedData as (Data<T> & ExtendedMeta)[]);
+          
+          // Force a small delay to ensure React updates the state before showing the results
+          await new Promise(resolve => setTimeout(resolve, 50));
+          
+          // Log a sample of the state after update, focus on line/subline fields
+          console.log('State after AI validation:', {
+            sampleLineValues: validatedData.slice(0, 3).map(row => ({ 
+              line: row.line, 
+              subline: row.subline,
+              lineDisplay: getFieldDisplayValue('line', row.line),
+              sublineDisplay: getFieldDisplayValue('subline', row.subline)
+            })),
+            fieldsWithOptions: fieldsWithUpdatedOptions
+              .filter(f => f.fieldType.type === 'select' && ['line', 'subline'].includes(f.key as string))
+              .map(f => ({ 
+                key: f.key, 
+                options: 'options' in f.fieldType ? f.fieldType.options?.length || 0 : 0 
+              }))
+          });
+          
+          console.log('Data updated after AI validation:', {
+            dataLength: validatedData.length,
+            hasErrors: validatedData.some(row => row.__errors && Object.keys(row.__errors).length > 0)
+          });
+          
+          // Show changes and warnings in dialog after data is updated
+          setAiValidationDetails({
+            changes: result.changes || [],
+            warnings: result.warnings || [],
+            changeDetails: result.changeDetails || [],
+            isOpen: true,
+            originalData: originalDataCopy  // Use the stored original data
+          });
+        } catch (error) {
+          console.error('Error validating AI corrections:', error);
+          // Fall back to basic update without validation
+          setData(newData);
+          
+          // Still show the result dialog even if validation failed
+          setAiValidationDetails({
+            changes: result.changes || [],
+            warnings: result.warnings || [],
+            changeDetails: result.changeDetails || [],
+            isOpen: true,
+            originalData: originalDataCopy,  // Use the stored original data
+          });
+        }
       }
 
-      // Show changes and warnings in dialog
-      setAiValidationDetails({
-        changes: result.changes || [],
-        warnings: result.warnings || [],
-        isOpen: true,
-      });
-
       setAiValidationProgress(prev => ({
         ...prev,
         status: "Validation complete!",
@@ -1640,7 +1939,6 @@ export const ValidationStep = <T extends string>({
       setTimeout(() => {
         setAiValidationProgress(prev => ({ ...prev, isOpen: false }));
       }, 1000);
-
     } catch (error) {
       console.error('AI Validation Error:', error);
       toast({
@@ -1654,7 +1952,18 @@ export const ValidationStep = <T extends string>({
         step: -1
       }));
     } finally {
+      // Clear the interval when we're done (success or error)
+      if (window.aiValidationTimer) {
+        clearInterval(window.aiValidationTimer);
+        window.aiValidationTimer = undefined;
+      }
       setIsAiValidating(false);
+      
+      // Only set to 100% when actually complete (or in error state)
+      setAiValidationProgress(prev => ({
+        ...prev,
+        progressPercent: prev.step === -1 ? prev.progressPercent : 100 // Only show 100% if successful completion
+      }));
     }
   };
 
@@ -1705,7 +2014,8 @@ export const ValidationStep = <T extends string>({
       // Log the response stats
       console.log('Debug response stats:', {
         promptLength: debugData.promptLength,
-        taxonomyStats: debugData.taxonomyStats
+        taxonomyStats: debugData.taxonomyStats,
+        estimatedProcessingTime: debugData.estimatedProcessingTime
       });
       
       setCurrentPrompt(prev => ({ 
@@ -1713,6 +2023,15 @@ export const ValidationStep = <T extends string>({
         prompt: debugData.sampleFullPrompt,
         isLoading: false 
       }));
+
+      // Store the estimated processing time for later use
+      if (debugData.estimatedProcessingTime?.seconds) {
+        setAiValidationProgress(prev => ({
+          ...prev,
+          estimatedSeconds: debugData.estimatedProcessingTime.seconds,
+          promptLength: debugData.promptLength
+        }));
+      }
     } catch (error) {
       console.error('Error fetching prompt:', error);
       toast({
@@ -1724,6 +2043,83 @@ export const ValidationStep = <T extends string>({
     }
   };
 
+  // Function to get display value for field values (including IDs)
+  const getFieldDisplayValue = (fieldKey: string, value: any): string => {
+    const field = fields.find(f => f.key === fieldKey);
+    if (!field) return String(value);
+
+    // Handle different field types
+    if (field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') {
+      const options = field.fieldType.options || [];
+      
+      // Handle array of values (multi-select)
+      if (Array.isArray(value)) {
+        return value.map(v => {
+          const option = options.find(opt => String(opt.value) === String(v));
+          return option ? option.label : v;
+        }).join(', ');
+      }
+      
+      // Handle single value
+      const option = options.find(opt => String(opt.value) === String(value));
+      return option ? option.label : String(value);
+    }
+    
+    return String(value);
+  };
+
+  // Add a function to revert a specific AI validation change
+  const revertAiChange = (productIndex: number, fieldKey: string) => {
+    // Ensure we have the original data
+    if (!aiValidationDetails.originalData) {
+      console.error('Cannot revert: original data not available');
+      return;
+    }
+
+    // Find the original product and current product
+    const originalProduct = aiValidationDetails.originalData[productIndex];
+    if (!originalProduct) {
+      console.error(`Cannot revert: original product at index ${productIndex} not found`);
+      return;
+    }
+    
+    const currentProduct = data[productIndex];
+    if (!currentProduct) {
+      console.error(`Cannot revert: current product at index ${productIndex} not found`);
+      return;
+    }
+
+    // Get the original value in a type-safe way
+    const originalValue = fieldKey in originalProduct ? 
+      (originalProduct as Record<string, any>)[fieldKey] : undefined;
+    
+    console.log(`Reverting change to field "${fieldKey}" for product at index ${productIndex}`, {
+      original: originalValue,
+      current: fieldKey in currentProduct ? (currentProduct as Record<string, any>)[fieldKey] : undefined
+    });
+
+    // Create a new data array with the reverted field
+    const newData = [...data];
+    
+    // Create a new product object with the reverted field
+    newData[productIndex] = {
+      ...newData[productIndex],
+      [fieldKey]: originalValue
+    } as Data<T> & ExtendedMeta; // Cast to ensure type safety
+
+    // Update the data state
+    setData(newData);
+
+    // Re-validate to update error states
+    updateData(newData, [productIndex]);
+
+    // Show success notification
+    toast({
+      title: "Change reverted",
+      description: `Reverted the change to "${fieldKey}"`,
+    });
+  };
+
   return (
     <div className="flex h-[calc(100vh-9.5rem)] flex-col">
       <CopyDownDialog 
@@ -1795,19 +2191,43 @@ export const ValidationStep = <T extends string>({
                   <div 
                     className="h-full bg-primary transition-all duration-500" 
                     style={{ 
-                      width: `${(aiValidationProgress.step / 5) * 100}%`,
+                      width: `${aiValidationProgress.progressPercent ?? (aiValidationProgress.step / 5) * 100}%`,
                       backgroundColor: aiValidationProgress.step === -1 ? 'var(--destructive)' : undefined
                     }}
                   />
                 </div>
               </div>
               <div className="text-sm text-muted-foreground w-12 text-right">
-                {aiValidationProgress.step === -1 ? '❌' : `${Math.round((aiValidationProgress.step / 5) * 100)}%`}
+                {aiValidationProgress.step === -1 ? '❌' : `${aiValidationProgress.progressPercent ?? Math.round((aiValidationProgress.step / 5) * 100)}%`}
               </div>
             </div>
             <p className="text-center text-sm text-muted-foreground">
               {aiValidationProgress.status}
             </p>
+            {aiValidationProgress.estimatedSeconds && aiValidationProgress.elapsedSeconds !== undefined && aiValidationProgress.step > 0 && aiValidationProgress.step < 5 && (
+              <div className="text-center text-sm">
+                {(() => {
+                  // Calculate time remaining using the elapsed seconds
+                  const elapsedSeconds = aiValidationProgress.elapsedSeconds;
+                  const totalEstimatedSeconds = aiValidationProgress.estimatedSeconds;
+                  const remainingSeconds = Math.max(0, totalEstimatedSeconds - elapsedSeconds);
+                  
+                  // Format time remaining
+                  if (remainingSeconds < 60) {
+                    return `Approximately ${Math.round(remainingSeconds)} seconds remaining`;
+                  } else {
+                    const minutes = Math.floor(remainingSeconds / 60);
+                    const seconds = Math.round(remainingSeconds % 60);
+                    return `Approximately ${minutes}m ${seconds}s remaining`;
+                  }
+                })()}
+                {aiValidationProgress.promptLength && (
+                  <p className="mt-1 text-xs text-muted-foreground">
+                    Prompt length: {aiValidationProgress.promptLength.toLocaleString()} characters
+                  </p>
+                )}
+              </div>
+            )}
           </div>
         </DialogContent>
       </Dialog>
@@ -1815,7 +2235,7 @@ export const ValidationStep = <T extends string>({
         open={aiValidationDetails.isOpen} 
         onOpenChange={(open) => setAiValidationDetails(prev => ({ ...prev, isOpen: open }))}
       >
-        <DialogContent className="max-w-2xl">
+        <DialogContent className="max-w-4xl">
           <DialogHeader>
             <DialogTitle>AI Validation Results</DialogTitle>
             <DialogDescription>
@@ -1823,18 +2243,102 @@ export const ValidationStep = <T extends string>({
             </DialogDescription>
           </DialogHeader>
           <ScrollArea className="max-h-[60vh]">
-            {aiValidationDetails.changes.length > 0 && (
-              <div className="mb-4">
-                <h3 className="font-semibold mb-2">Changes Made:</h3>
-                <ul className="space-y-2">
-                  {aiValidationDetails.changes.map((change, i) => (
-                    <li key={i} className="flex gap-2">
-                      <span className="text-green-500">✓</span>
-                      <span>{change}</span>
-                    </li>
-                  ))}
-                </ul>
+            {aiValidationDetails.changeDetails && aiValidationDetails.changeDetails.length > 0 ? (
+              <div className="mb-6 space-y-6">
+                <h3 className="font-semibold text-lg">Detailed Changes:</h3>
+                {aiValidationDetails.changeDetails.map((product, i) => (
+                  <div key={i} className="border rounded-md p-4">
+                    <h4 className="font-medium text-base mb-3">{product.title}</h4>
+                    <Table>
+                      <TableHeader>
+                        <TableRow>
+                          <TableHead className="w-1/4">Field</TableHead>
+                          <TableHead className="w-3/8">Original Value</TableHead>
+                          <TableHead className="w-3/8">Corrected Value</TableHead>
+                          <TableHead className="w-1/8 text-right">Action</TableHead>
+                        </TableRow>
+                      </TableHeader>
+                      <TableBody>
+                        {product.changes.map((change, j) => {
+                          const field = fields.find(f => f.key === change.field);
+                          return (
+                            <TableRow key={j}>
+                              <TableCell className="font-medium">{field?.label || change.field}</TableCell>
+                              <TableCell>
+                                <div className="space-y-1">
+                                  <div>{getFieldDisplayValue(change.field, change.original)}</div>
+                                  {/* Show raw value if it's an ID */}
+                                  {change.original && typeof change.original === 'string' && 
+                                   !isNaN(Number(change.original)) && 
+                                   getFieldDisplayValue(change.field, change.original) !== change.original && (
+                                    <div className="text-xs text-muted-foreground">ID: {change.original}</div>
+                                  )}
+                                </div>
+                              </TableCell>
+                              <TableCell>
+                                <div className="space-y-1">
+                                  <div>{getFieldDisplayValue(change.field, change.corrected)}</div>
+                                  {/* Show raw value if it's an ID */}
+                                  {change.corrected && typeof change.corrected === 'string' && 
+                                   !isNaN(Number(change.corrected)) && 
+                                   getFieldDisplayValue(change.field, change.corrected) !== change.corrected && (
+                                    <div className="text-xs text-muted-foreground">ID: {change.corrected}</div>
+                                  )}
+                                </div>
+                              </TableCell>
+                              <TableCell className="text-right">
+                                <div className="mt-2">
+                                  <Button
+                                    variant="outline"
+                                    size="sm"
+                                    onClick={() => {
+                                      // Find the product in the current data
+                                      const currentDataIndex = data.findIndex(
+                                        d => d.__index === data[product.productIndex]?.__index
+                                      );
+                                      if (currentDataIndex >= 0) {
+                                        // Create new data array with the original value for this field
+                                        const newData = [...data];
+                                        newData[currentDataIndex] = {
+                                          ...newData[currentDataIndex],
+                                          [change.field]: change.original
+                                        };
+                                        // Update the data
+                                        updateData(newData, [currentDataIndex]);
+                                        // Show toast
+                                        toast({
+                                          title: "Change reverted",
+                                          description: `Reverted ${change.field} to original value`,
+                                        });
+                                      }
+                                    }}
+                                  >
+                                    Revert Change
+                                  </Button>
+                                </div>
+                              </TableCell>
+                            </TableRow>
+                          );
+                        })}
+                      </TableBody>
+                    </Table>
+                  </div>
+                ))}
               </div>
+            ) : (
+              aiValidationDetails.changes.length > 0 && (
+                <div className="mb-4">
+                  <h3 className="font-semibold mb-2">Changes Made:</h3>
+                  <ul className="space-y-2">
+                    {aiValidationDetails.changes.map((change, i) => (
+                      <li key={i} className="flex gap-2">
+                        <span className="text-green-500">✓</span>
+                        <span>{change}</span>
+                      </li>
+                    ))}
+                  </ul>
+                </div>
+              )
             )}
             {aiValidationDetails.warnings.length > 0 && (
               <div>
@@ -2063,3 +2567,10 @@ export const ValidationStep = <T extends string>({
     </div>
   )
 }
+
+// Add TypeScript declaration for our global timer variable
+declare global {
+  interface Window {
+    aiValidationTimer?: NodeJS.Timeout;
+  }
+}

inventory/src/pages/AiValidationDebug.tsx

@@ -7,26 +7,26 @@ import { useToast } from "@/hooks/use-toast"
 import { Loader2 } from "lucide-react"
 import config from "@/config"
 
-interface CacheStatus {
-  isCacheValid: boolean
-  lastUpdated: string | null
-  timeUntilExpiry: string
-}
-
 interface TaxonomyStats {
   categories: number
   themes: number
   colors: number
   taxCodes: number
   sizeCategories: number
+  suppliers: number
+  companies: number
+  artists: number
 }
 
 interface DebugData {
-  cacheStatus: CacheStatus
   taxonomyStats: TaxonomyStats | null
   basePrompt: string
   sampleFullPrompt: string
   promptLength: number
+  estimatedProcessingTime?: {
+    seconds: number | null
+    sampleCount: number
+  }
 }
 
 export function AiValidationDebug() {
@@ -72,39 +72,6 @@ export function AiValidationDebug() {
     }
   }
 
-  const refreshCache = async () => {
-    if (!confirm('Are you sure you want to refresh the cache?')) return
-
-    setIsLoading(true)
-    try {
-      const response = await fetch(`${config.apiUrl}/ai-validation/refresh-cache`, {
-        method: 'POST'
-      })
-      if (!response.ok) {
-        throw new Error('Failed to refresh cache')
-      }
-      const data = await response.json()
-      if (data.success) {
-        toast({
-          title: "Success",
-          description: "Cache refreshed successfully"
-        })
-        fetchDebugData()
-      } else {
-        throw new Error(data.error || 'Failed to refresh cache')
-      }
-    } catch (error) {
-      console.error('Error refreshing cache:', error)
-      toast({
-        variant: "destructive",
-        title: "Error",
-        description: error instanceof Error ? error.message : "Failed to refresh cache"
-      })
-    } finally {
-      setIsLoading(false)
-    }
-  }
-
   useEffect(() => {
     fetchDebugData()
   }, [])
@@ -122,32 +89,11 @@ export function AiValidationDebug() {
             {isLoading && <Loader2 className="mr-2 h-4 w-4 animate-spin" />}
             Refresh Data
           </Button>
-          <Button
-            variant="outline"
-            onClick={refreshCache}
-            disabled={isLoading}
-          >
-            {isLoading && <Loader2 className="mr-2 h-4 w-4 animate-spin" />}
-            Force Cache Refresh
-          </Button>
         </div>
       </div>
 
       {debugData && (
-        <div className="grid grid-cols-1 md:grid-cols-3 gap-6">
-          <Card>
-            <CardHeader>
-              <CardTitle>Cache Status</CardTitle>
-            </CardHeader>
-            <CardContent>
-              <div className="space-y-2">
-                <div>Valid: {debugData.cacheStatus.isCacheValid ? "Yes" : "No"}</div>
-                <div>Last Updated: {debugData.cacheStatus.lastUpdated || "never"}</div>
-                <div>Expires in: {debugData.cacheStatus.timeUntilExpiry}</div>
-              </div>
-            </CardContent>
-          </Card>
-
+        <div className="grid grid-cols-1 md:grid-cols-2 gap-6">
           <Card>
             <CardHeader>
               <CardTitle>Taxonomy Stats</CardTitle>
@@ -160,6 +106,9 @@ export function AiValidationDebug() {
                   <div>Colors: {debugData.taxonomyStats.colors}</div>
                   <div>Tax Codes: {debugData.taxonomyStats.taxCodes}</div>
                   <div>Size Categories: {debugData.taxonomyStats.sizeCategories}</div>
+                  <div>Suppliers: {debugData.taxonomyStats.suppliers}</div>
+                  <div>Companies: {debugData.taxonomyStats.companies}</div>
+                  <div>Artists: {debugData.taxonomyStats.artists}</div>
                 </div>
               ) : (
                 <div>No taxonomy data available</div>
@@ -202,6 +151,23 @@ export function AiValidationDebug() {
                     Cost: <span id="tokenCost">{((Math.round(debugData.promptLength / 4) / 1_000_000) * 3 * 100).toFixed(1)}</span>¢
                   </div>
                 </div>
+                {debugData.estimatedProcessingTime && (
+                  <div className="mt-4 p-3 bg-muted rounded-md">
+                    <h3 className="text-sm font-medium mb-2">Processing Time Estimate</h3>
+                    {debugData.estimatedProcessingTime.seconds ? (
+                      <div className="space-y-1">
+                        <div className="text-sm">
+                          Estimated time: {formatTime(debugData.estimatedProcessingTime.seconds)}
+                        </div>
+                        <div className="text-xs text-muted-foreground">
+                          Based on {debugData.estimatedProcessingTime.sampleCount} similar validation{debugData.estimatedProcessingTime.sampleCount !== 1 ? 's' : ''}
+                        </div>
+                      </div>
+                    ) : (
+                      <div className="text-sm text-muted-foreground">No historical data available for this prompt size</div>
+                    )}
+                  </div>
+                )}
               </div>
             </CardContent>
           </Card>
@@ -220,4 +186,15 @@ export function AiValidationDebug() {
       )}
     </div>
   )
+}
+
+// Helper function to format time in a human-readable way
+function formatTime(seconds: number): string {
+  if (seconds < 60) {
+    return `${Math.round(seconds)} seconds`;
+  } else {
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = Math.round(seconds % 60);
+    return `${minutes}m ${remainingSeconds}s`;
+  }
 }