Fix line/subline regressions, add in AI validation tracking and improve AI results dialog

inventory-server/db/setup-schema.sql

@@ -23,6 +23,20 @@ CREATE TABLE IF NOT EXISTS templates (
     UNIQUE(company, product_type)
 );
 
+-- AI Validation Performance Tracking
+CREATE TABLE IF NOT EXISTS ai_validation_performance (
+    id SERIAL PRIMARY KEY,
+    prompt_length INTEGER NOT NULL,
+    product_count INTEGER NOT NULL,
+    start_time TIMESTAMP WITH TIME ZONE NOT NULL,
+    end_time TIMESTAMP WITH TIME ZONE NOT NULL,
+    duration_seconds DECIMAL(10,2) GENERATED ALWAYS AS (EXTRACT(EPOCH FROM (end_time - start_time))) STORED,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Create index on prompt_length for efficient querying
+CREATE INDEX IF NOT EXISTS idx_ai_validation_prompt_length ON ai_validation_performance(prompt_length);
+
 -- Function to update the updated_at timestamp
 CREATE OR REPLACE FUNCTION update_updated_at_column()
 RETURNS TRIGGER AS $$

inventory-server/src/prompts/product-validation.txt

@@ -7,6 +7,8 @@ Your response should be a JSON object with the following structure:
   "warnings": [] // Array of strings with warnings or suggestions for manual review (see below for details)
 }
 
+IMPORTANT: For all fields that use IDs (categories, supplier, company, line, subline, ship_restrictions, tax_cat, artist, themes, etc.), you MUST return the ID values, not the display names. The system will handle converting IDs to display names.
+
 Using the provided guidelines, focus on:
 1. Correcting typos and any incorrect spelling or grammar
 2. Standardizing product names
@@ -93,7 +95,7 @@ Instructions: Always return a valid numerical tax code ID from the Available Tax
 Fields: size_cat
 Changes: Allowed to correct obvious errors or inconsistencies or to add missing values
 Required: Return if present in the original data or if not present and applicable. Do not return if not applicable (e.g. if no size categories apply based on what you know about the product).
-Instructions: If present or if applicable, return one valid numerical size category ID from the Available Size Categories array below. Give preference to the value provided, but correct it if another value is more accurate. A value is not required if none of the size categories apply, but it's important to include if one clearly applies, such as if the name contains 12x12, 6x8, 2oz, etc.
+Instructions: If present or if applicable, return one valid numerical size category ID from the Available Size Categories array below. Give preference to the value provided, but correct it if another value is more accurate. If the product name contains a match for one of the size categories (such as 12x12, 6x6, 2oz, etc) you MUST return that size category with the results. A value is not required if none of the size categories apply.
 
 Fields: themes
 Changes: Allowed to correct obvious errors or inconsistencies or to add missing values

inventory-server/src/routes/ai-validation.js

@@ -98,7 +98,51 @@ router.post("/debug", async (req, res) => {
     });
 
     try {
-      return await generateDebugResponse(cleanedProducts, res);
+      const debugResponse = await generateDebugResponse(cleanedProducts, res);
+      
+      // Get estimated processing time based on prompt length
+      if (debugResponse && debugResponse.promptLength) {
+        try {
+          // Use the pool from the app
+          const pool = req.app.locals.pool;
+          if (!pool) {
+            console.warn("⚠️ Local database pool not available for time estimates");
+            return;
+          }
+          
+          try {
+            const avgTimeResults = await pool.query(
+              `SELECT AVG(duration_seconds) as avg_duration, 
+                      COUNT(*) as sample_count
+               FROM ai_validation_performance 
+               WHERE prompt_length BETWEEN $1 * 0.8 AND $1 * 1.2`,
+              [debugResponse.promptLength]
+            );
+            
+            // Add estimated time to the response
+            if (avgTimeResults.rows && avgTimeResults.rows[0]) {
+              debugResponse.estimatedProcessingTime = {
+                seconds: avgTimeResults.rows[0].avg_duration || null,
+                sampleCount: avgTimeResults.rows[0].sample_count || 0
+              };
+              console.log("📊 Retrieved processing time estimate:", debugResponse.estimatedProcessingTime);
+            } else {
+              console.log("📊 No processing time estimates available for prompt length:", debugResponse.promptLength);
+            }
+          } catch (queryError) {
+            console.error("⚠️ Failed to query performance metrics:", queryError);
+            // Check if table doesn't exist and log a more helpful message
+            if (queryError.code === '42P01') {
+              console.error("Table 'ai_validation_performance' does not exist. Make sure to run the setup-schema.sql script.");
+            }
+          }
+        } catch (timeEstimateError) {
+          console.error("Error getting time estimate:", timeEstimateError);
+          // Don't fail the request if time estimate fails
+        }
+      }
+      
+      return res.json(debugResponse);
     } catch (generateError) {
       console.error("Error generating debug response:", generateError);
       return res.status(500).json({
@@ -271,7 +315,7 @@ async function generateDebugResponse(productsToUse, res) {
       };
 
       console.log("Sending response with taxonomy stats:", response.taxonomyStats);
-      return res.json(response);
+      return response;
     } finally {
       if (promptConnection) await promptConnection.end();
       if (promptTunnel.ssh) promptTunnel.ssh.end();
@@ -463,9 +507,7 @@ async function loadPrompt(connection, productsToValidate = null) {
     const taxonomy = await getTaxonomyData(connection);
 
     // Add system instructions to the prompt
-    const systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone.
+    const systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`;
-
-`;
 
     // If we have products to validate, create a filtered prompt
     if (productsToValidate) {
@@ -634,6 +676,7 @@ Here is the product data to validate:`;
 router.post("/validate", async (req, res) => {
   try {
     const { products } = req.body;
+    const startTime = new Date(); // Track start time for performance metrics
 
     console.log("🔍 Received products for validation:", {
       isArray: Array.isArray(products),
@@ -654,6 +697,7 @@ router.post("/validate", async (req, res) => {
 
     let ssh = null;
     let connection = null;
+    let promptLength = 0; // Track prompt length for performance metrics
     
     try {
       // Setup MySQL connection via SSH tunnel
@@ -672,7 +716,8 @@ router.post("/validate", async (req, res) => {
       console.log("🔄 Loading prompt with filtered taxonomy...");
       const prompt = await loadPrompt(connection, products);
       const fullPrompt = prompt + "\n" + JSON.stringify(products);
-      console.log("📝 Generated prompt length:", fullPrompt.length);
+      promptLength = fullPrompt.length; // Store prompt length for performance metrics
+      console.log("📝 Generated prompt length:", promptLength);
 
       console.log("🤖 Sending request to OpenAI...");
       const completion = await openai.chat.completions.create({
@@ -698,17 +743,27 @@ router.post("/validate", async (req, res) => {
           Object.keys(aiResponse)
         );
 
+        // Create a detailed comparison between original and corrected data
+        const changeDetails = [];
+        
         // Compare original and corrected data
         if (aiResponse.correctedData) {
           console.log("📊 Changes summary:");
           products.forEach((original, index) => {
             const corrected = aiResponse.correctedData[index];
             if (corrected) {
+              const productChanges = {
+                productIndex: index,
+                title: original.title || `Product ${index + 1}`,
+                changes: []
+              };
+              
               const changes = Object.keys(corrected).filter(
                 (key) =>
                   JSON.stringify(original[key]) !==
                   JSON.stringify(corrected[key])
               );
+              
               if (changes.length > 0) {
                 console.log(`\nProduct ${index + 1} changes:`);
                 changes.forEach((key) => {
@@ -719,14 +774,87 @@ router.post("/validate", async (req, res) => {
                   console.log(
                     `    - Corrected: ${JSON.stringify(corrected[key])}`
                   );
+                  
+                  // Add to our detailed changes array
+                  productChanges.changes.push({
+                    field: key,
+                    original: original[key],
+                    corrected: corrected[key]
                   });
+                });
+                
+                // Only add products that have changes
+                if (productChanges.changes.length > 0) {
+                  changeDetails.push(productChanges);
+                }
               }
             }
           });
         }
 
+        // Record performance metrics after successful validation
+        const endTime = new Date();
+        let performanceMetrics = {
+          promptLength,
+          productCount: products.length,
+          processingTimeSeconds: (endTime - startTime) / 1000
+        };
+        
+        try {
+          // Use the local PostgreSQL pool from the app instead of the MySQL connection
+          const pool = req.app.locals.pool;
+          if (!pool) {
+            console.warn("⚠️ Local database pool not available for recording metrics");
+            return;
+          }
+          
+          try {
+            // Insert performance data into the local PostgreSQL database
+            await pool.query(
+              `INSERT INTO ai_validation_performance 
+               (prompt_length, product_count, start_time, end_time) 
+               VALUES ($1, $2, $3, $4)`,
+              [promptLength, products.length, startTime, endTime]
+            );
+            
+            console.log("📊 Performance metrics inserted into database");
+            
+            // Query for average processing time based on similar prompt lengths
+            try {
+              const avgTimeResults = await pool.query(
+                `SELECT AVG(duration_seconds) as avg_duration, 
+                        COUNT(*) as sample_count
+                 FROM ai_validation_performance 
+                 WHERE prompt_length BETWEEN $1 * 0.8 AND $1 * 1.2`,
+                [promptLength]
+              );
+              
+              if (avgTimeResults.rows && avgTimeResults.rows[0]) {
+                performanceMetrics.avgDuration = avgTimeResults.rows[0].avg_duration;
+                performanceMetrics.sampleCount = avgTimeResults.rows[0].sample_count;
+              }
+              
+              console.log("📊 Performance metrics retrieved:", performanceMetrics);
+            } catch (queryError) {
+              console.error("⚠️ Failed to query performance metrics:", queryError);
+            }
+          } catch (insertError) {
+            console.error("⚠️ Failed to insert performance metrics:", insertError);
+            // Check if table doesn't exist and log a more helpful message
+            if (insertError.code === '42P01') {
+              console.error("Table 'ai_validation_performance' does not exist. Make sure to run the setup-schema.sql script.");
+            }
+          }
+        } catch (metricError) {
+          // Don't fail the request if metrics recording fails
+          console.error("⚠️ Failed to record performance metrics:", metricError);
+        }
+        
+        // Include performance metrics in the response
         res.json({
           success: true,
+          changeDetails: changeDetails,
+          performanceMetrics,
           ...aiResponse,
         });
       } catch (parseError) {

inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStep/ValidationStep.tsx

@@ -171,16 +171,34 @@ const EditableCell = memo(({ value, onChange, error, field, productLines, sublin
 
   // Determine if the field should be disabled based on its key and context
   const isFieldDisabled = useMemo(() => {
+    // If the field is already disabled by the parent component, respect that
+    if (field.disabled) return true;
+    
+    // Special handling for line and subline fields
     if (field.key === 'line') {
-      // Enable the line field if we have product lines available
+      // Never disable line field if it already has a value
-      return !productLines || productLines.length === 0;
+      if (value) return false;
+      
+      // The line field should be enabled if:
+      // 1. We have a company selected (even if product lines are still loading)
+      // 2. We have product lines available
+      return !((field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') && 
+        field.fieldType.options?.length) && !productLines?.length;
     }
     if (field.key === 'subline') {
-      // Enable subline field if we have sublines available
+      // Never disable subline field if it already has a value
-      return !sublines || sublines.length === 0;
+      if (value) return false;
+      
+      // The subline field should be enabled if:
+      // 1. We have a line selected (even if sublines are still loading)
+      // 2. We have sublines available
+      return !((field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') && 
+        field.fieldType.options?.length) && !sublines?.length;
     }
+    
+    // For other fields, use the disabled property
     return field.disabled;
-  }, [field.key, field.disabled, productLines, sublines]);
+  }, [field.key, field.disabled, field.fieldType, productLines, sublines, value]);
 
   // For debugging
   useEffect(() => {
@@ -269,21 +287,71 @@ const EditableCell = memo(({ value, onChange, error, field, productLines, sublin
     if (fieldType.type === "select" || fieldType.type === "multi-select") {
       if (fieldType.type === "select") {
         // For line and subline fields, ensure we're using the latest options
-        if (field.key === 'line' && productLines?.length) {
+        if (field.key === 'line') {
-          const option = productLines.find((opt: SelectOption) => opt.value === value);
+          // Log current state for debugging
-          return option?.label || value;
+          console.log('Getting display value for line:', {
+            value,
+            productLines: productLines?.length ?? 0,
+            options: fieldType.options?.length ?? 0
+          });
+          
+          // First try to find in productLines if available
+          if (productLines?.length) {
+            const matchingOption = productLines.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (matchingOption) {
+              console.log('Found line in productLines:', value, '->', matchingOption.label);
+              return matchingOption.label;
             }
-        if (field.key === 'subline' && sublines?.length) {
-          const option = sublines.find((opt: SelectOption) => opt.value === value);
-          return option?.label || value;
           }
-        return fieldType.options.find((opt: SelectOption) => opt.value === value)?.label || value;
+          // Fall back to fieldType options if productLines not available yet
+          if (fieldType.options?.length) {
+            const fallbackOptionLine = fieldType.options.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (fallbackOptionLine) {
+              console.log('Found line in fallback options:', value, '->', fallbackOptionLine.label);
+              return fallbackOptionLine.label;
+            }
+          }
+          console.log('Unable to find display value for line:', value);
+          return value;
+        }
+        if (field.key === 'subline') {
+          // Log current state for debugging
+          console.log('Getting display value for subline:', {
+            value,
+            sublines: sublines?.length ?? 0,
+            options: fieldType.options?.length ?? 0
+          });
+          
+          // First try to find in sublines if available
+          if (sublines?.length) {
+            const matchingOption = sublines.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (matchingOption) {
+              console.log('Found subline in sublines:', value, '->', matchingOption.label);
+              return matchingOption.label;
+            }
+          }
+          // Fall back to fieldType options if sublines not available yet
+          if (fieldType.options?.length) {
+            const fallbackOptionSubline = fieldType.options.find((opt: SelectOption) => 
+              String(opt.value) === String(value));
+            if (fallbackOptionSubline) {
+              console.log('Found subline in fallback options:', value, '->', fallbackOptionSubline.label);
+              return fallbackOptionSubline.label;
+            }
+          }
+          console.log('Unable to find display value for subline:', value);
+          return value;
+        }
+        return fieldType.options?.find((opt: SelectOption) => String(opt.value) === String(value))?.label || value;
       }
       if (Array.isArray(value)) {
         const options = field.key === 'line' && productLines?.length ? productLines :
                        field.key === 'subline' && sublines?.length ? sublines :
                        fieldType.options;
-        return value.map(v => options.find((opt: SelectOption) => opt.value === v)?.label || v).join(", ");
+        return value.map(v => options.find((opt: SelectOption) => String(opt.value) === String(v))?.label || v).join(", ");
       }
       return value;
     }
@@ -881,21 +949,23 @@ function useTemplates<T extends string>(
       const { __index, __errors, __template, ...templateData } = selectedRow;
 
       // Clean numeric values and prepare template data
-      const cleanedData = Object.entries(templateData).reduce((acc, [key, value]) => {
+      const cleanedData: Record<string, any> = {};
+      
+      // Process each key-value pair
+      Object.entries(templateData).forEach(([key, value]) => {
         // Handle numeric values with dollar signs
         if (typeof value === 'string' && value.includes('$')) {
-          acc[key] = value.replace(/[$,\s]/g, '').trim();
+          cleanedData[key] = value.replace(/[$,\s]/g, '').trim();
         } 
         // Handle array values (like categories or ship_restrictions)
         else if (Array.isArray(value)) {
-          acc[key] = value;
+          cleanedData[key] = value;
         }
         // Handle other values
         else {
-          acc[key] = value;
+          cleanedData[key] = value;
         }
-        return acc;
+      });
-      }, {} as Record<string, any>);
 
       // Log the cleaned data before sending
       console.log('Saving template with cleaned data:', {
@@ -1049,6 +1119,19 @@ const SaveTemplateDialog = memo(({
   );
 });
 
+// Add a new interface to handle the AI validation details response
+interface ChangeDetail {
+  field: string;
+  original: any;
+  corrected: any;
+}
+
+interface ProductChangeDetail {
+  productIndex: number;
+  title: string;
+  changes: ChangeDetail[];
+}
+
 export const ValidationStep = <T extends string>({ 
   initialData,
   file,
@@ -1083,13 +1166,18 @@ export const ValidationStep = <T extends string>({
     queryKey: ["sublines", globalSelections?.line],
     queryFn: async () => {
       if (!globalSelections?.line) return [];
+      console.log('Fetching sublines for line:', globalSelections.line);
       const response = await fetch(`${config.apiUrl}/import/sublines/${globalSelections.line}`);
       if (!response.ok) {
+        console.error('Failed to fetch sublines:', response.status, response.statusText);
         throw new Error("Failed to fetch sublines");
       }
-      return response.json();
+      const data = await response.json();
+      console.log('Received sublines:', data);
+      return data;
     },
     enabled: !!globalSelections?.line,
+    staleTime: 30000, // Cache for 30 seconds
   });
 
   // Apply global selections to initial data and validate
@@ -1163,7 +1251,9 @@ export const ValidationStep = <T extends string>({
             ...field.fieldType,
             options: productLines || (field.fieldType.type === 'select' ? field.fieldType.options : []),
           },
-          disabled: (!productLines || productLines.length === 0) && !globalSelections?.line
+          // Only disable if no company is selected or if product lines failed to load
+          // when a company is selected
+          disabled: !globalSelections?.company || (globalSelections?.company && productLines !== undefined && productLines.length === 0)
         } as Field<T>;
       }
       if (field.key === 'subline') {
@@ -1179,7 +1269,7 @@ export const ValidationStep = <T extends string>({
       }
       return field;
     });
-  }, [fields, productLines, sublines, globalSelections?.line]);
+  }, [fields, productLines, sublines, globalSelections?.company, globalSelections?.line]);
 
   const [data, setData] = useState<RowData<T>[]>(initialDataWithGlobals);
 
@@ -1208,10 +1298,13 @@ export const ValidationStep = <T extends string>({
   const [aiValidationDetails, setAiValidationDetails] = useState<{
     changes: string[];
     warnings: string[];
+    changeDetails: ProductChangeDetail[];
     isOpen: boolean;
+    originalData?: (Data<T> & ExtendedMeta)[];  // Store original data for reverting changes
   }>({
     changes: [],
     warnings: [],
+    changeDetails: [],
     isOpen: false,
   });
 
@@ -1219,6 +1312,11 @@ export const ValidationStep = <T extends string>({
     isOpen: boolean;
     status: string;
     step: number;
+    estimatedSeconds?: number;
+    startTime?: Date;
+    promptLength?: number;
+    elapsedSeconds?: number;
+    progressPercent?: number;
   }>({
     isOpen: false,
     status: "",
@@ -1259,15 +1357,18 @@ export const ValidationStep = <T extends string>({
     async (rows: (Data<T> & ExtendedMeta)[], indexes?: number[]) => {
       setData(rows);
       
+      // Use fieldsWithUpdatedOptions to ensure we have the latest field definitions with proper options
+      const currentFields = fieldsWithUpdatedOptions as unknown as Fields<T>;
+      
       if (rowHook?.constructor.name === "AsyncFunction" || tableHook?.constructor.name === "AsyncFunction") {
-        const updatedData = await addErrorsAndRunHooks<T>(rows, fields, rowHook, tableHook, indexes);
+        const updatedData = await addErrorsAndRunHooks<T>(rows, currentFields, rowHook, tableHook, indexes);
         setData(updatedData as (Data<T> & ExtendedMeta)[]);
       } else {
-        const result = await addErrorsAndRunHooks<T>(rows, fields, rowHook, tableHook, indexes);
+        const result = await addErrorsAndRunHooks<T>(rows, currentFields, rowHook, tableHook, indexes);
         setData(result as (Data<T> & ExtendedMeta)[]);
       }
     },
-    [rowHook, tableHook, fields],
+    [rowHook, tableHook, fieldsWithUpdatedOptions],
   );
 
   const updateRows = useCallback(
@@ -1544,17 +1645,59 @@ export const ValidationStep = <T extends string>({
     }
   }, [data, submitData]);
 
-  // Add AI validation function
+  // Update the AI validation function
   const handleAiValidation = async () => {
     try {
+      if (isAiValidating) return;
+      
+      // Store the original data before any changes
+      const originalDataCopy = [...data];
+      
       setIsAiValidating(true);
+      const startTime = new Date();
       setAiValidationProgress({
         isOpen: true,
         status: "Preparing data for validation...",
-        step: 1
+        step: 1,
+        startTime,
+        elapsedSeconds: 0,
+        progressPercent: 0,
+        ...(aiValidationProgress.estimatedSeconds ? {
+          estimatedSeconds: aiValidationProgress.estimatedSeconds,
+          promptLength: aiValidationProgress.promptLength
+        } : {})
       });
       console.log('Sending data for validation:', data);
 
+      // Set up an interval to update progress
+      window.aiValidationTimer = setInterval(() => {
+        const now = new Date();
+        const elapsedSeconds = Math.floor((now.getTime() - startTime.getTime()) / 1000);
+        
+        setAiValidationProgress(prev => {
+          // Calculate progress percentage
+          let progressPercent = 0;
+          if (prev.estimatedSeconds && prev.estimatedSeconds > 0) {
+            // Cap at 99% if we exceed estimated time but aren't done yet
+            progressPercent = Math.min(99, Math.floor((elapsedSeconds / prev.estimatedSeconds) * 100));
+          } else {
+            // If no estimate, use step-based progress (25% per step), also capped at 99%
+            progressPercent = Math.min(99, (prev.step * 25) + Math.min(24, Math.floor((elapsedSeconds % 10) / 10 * 25)));
+          }
+          
+          // Extract the base status message without any time information
+          const baseStatus = prev.status.replace(/\s\(\d+[ms].+\)$/, '').replace(/\s\(\d+m \d+s.+\)$/, '');
+          
+          return {
+            ...prev,
+            elapsedSeconds,
+            progressPercent,
+            // Just use the base status message without time information
+            status: baseStatus
+          };
+        });
+      }, 1000);
+
       // Clean the data to ensure we only send what's needed
       const cleanedData = data.map(item => {
         const { __errors, __index, ...cleanProduct } = item;
@@ -1563,9 +1706,36 @@ export const ValidationStep = <T extends string>({
 
       console.log('Cleaned data for validation:', cleanedData);
 
+      // If we don't have an estimated time yet, try to get one
+      if (!aiValidationProgress.estimatedSeconds) {
+        try {
+          const debugResponse = await fetch(`${config.apiUrl}/ai-validation/debug`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ products: cleanedData })
+          });
+          
+          if (debugResponse.ok) {
+            const debugData = await debugResponse.json();
+            if (debugData.estimatedProcessingTime?.seconds) {
               setAiValidationProgress(prev => ({
                 ...prev,
-        status: "Sending data to AI service and awaiting response...",
+                estimatedSeconds: debugData.estimatedProcessingTime.seconds,
+                promptLength: debugData.promptLength
+              }));
+            }
+          }
+        } catch (estimateError) {
+          console.error('Error getting time estimate:', estimateError);
+          // Continue without an estimate
+        }
+      }
+
+      setAiValidationProgress(prev => ({
+        ...prev,
+        status: "Sending data to AI service...",
         step: 2
       }));
 
@@ -1587,12 +1757,6 @@ export const ValidationStep = <T extends string>({
         throw new Error(`AI validation failed: ${response.status} ${response.statusText}`);
       }
 
-      setAiValidationProgress(prev => ({
-        ...prev,
-        status: "Processing AI response...",
-        step: 3
-      }));
-
       const result = await response.json();
       console.log('AI validation response:', result);
       
@@ -1600,50 +1764,171 @@ export const ValidationStep = <T extends string>({
         throw new Error(result.error || 'AI validation failed');
       }
 
+      // Update progress with actual processing time if available
+      if (result.performanceMetrics) {
+        console.log('Performance metrics:', result.performanceMetrics);
+        setAiValidationProgress(prev => ({
+          ...prev,
+          status: "Processing AI response...",
+          step: 3,
+          // Update with actual metrics from the server
+          estimatedSeconds: result.performanceMetrics.processingTimeSeconds || prev.estimatedSeconds,
+          promptLength: result.performanceMetrics.promptLength || prev.promptLength,
+          progressPercent: 75 // 75% complete when we're processing the AI response
+        }));
+      } else {
+        setAiValidationProgress(prev => ({
+          ...prev,
+          status: "Processing AI response...",
+          step: 3,
+          progressPercent: 75
+        }));
+      }
+
       setAiValidationProgress(prev => ({
         ...prev,
         status: "Applying corrections...",
-        step: 4
+        step: 4,
+        progressPercent: 90 // 90% complete when applying corrections
       }));
 
       // Update the data with AI suggestions
       if (result.correctedData && Array.isArray(result.correctedData)) {
-        // Log the differences
+        // Process data to properly handle comma-separated values for multi-select fields
-        data.forEach((original, index) => {
+        const processedData = result.correctedData.map((corrected: any) => {
-          const corrected = result.correctedData[index];
+          const processed = { ...corrected };
-          if (corrected) {
+          
-            const changes = Object.keys(corrected).filter(key => {
+          // Process each field
-              const originalValue = original[key as keyof typeof original];
+          Object.keys(processed).forEach(key => {
-              const correctedValue = corrected[key as keyof typeof corrected];
+            const fieldConfig = fields.find(f => f.key === key);
-              return JSON.stringify(originalValue) !== JSON.stringify(correctedValue);
+            
+            // Handle multi-select fields (comma-separated values)
+            if (fieldConfig?.fieldType.type === 'multi-select' && typeof processed[key] === 'string') {
+              // Split comma-separated values and trim each value
+              processed[key] = processed[key].split(',').map((v: string) => v.trim()).filter(Boolean);
+            }
+            
+            // For select and multi-select fields, ensure we're working with IDs
+            // We don't convert IDs to display names here because we want to preserve IDs in the data
+            if (fieldConfig?.fieldType.type === 'select' || fieldConfig?.fieldType.type === 'multi-select') {
+              const options = fieldConfig.fieldType.options || [];
+              
+              // If the value is a string that matches a label but not a value, convert it to the corresponding ID
+              if (!Array.isArray(processed[key]) && typeof processed[key] === 'string') {
+                // Check if the value is already an ID
+                const isAlreadyId = options.some(opt => String(opt.value) === String(processed[key]));
+                
+                if (!isAlreadyId) {
+                  // Try to find the option by label
+                  const matchingOption = options.find(opt => opt.label === processed[key]);
+                  if (matchingOption) {
+                    // Convert label to ID
+                    const originalValue = processed[key];
+                    processed[key] = matchingOption.value;
+                    console.log(`Converted label "${originalValue}" to ID "${matchingOption.value}" for field "${key}"`);
+                  }
+                }
+              }
+              
+              // Handle array of values (multi-select)
+              if (Array.isArray(processed[key])) {
+                processed[key] = processed[key].map((val: string | number) => {
+                  // Check if the value is already an ID
+                  const isAlreadyId = options.some(opt => String(opt.value) === String(val));
+                  
+                  if (!isAlreadyId) {
+                    // Try to find the option by label
+                    const matchingOption = options.find(opt => opt.label === val);
+                    if (matchingOption) {
+                      // Convert label to ID
+                      return matchingOption.value;
+                    }
+                  }
+                  return val;
                 });
-            if (changes.length > 0) {
-              console.log(`Changes for row ${index + 1}:`, changes.map(key => ({
-                field: key,
-                original: original[key as keyof typeof original],
-                corrected: corrected[key as keyof typeof corrected]
-              })));
               }
             }
           });
           
+          return processed;
+        });
+        
         // Preserve the __index and __errors from the original data
-        const newData = result.correctedData.map((item: any, idx: number) => ({
+        const newData = processedData.map((item: any, idx: number) => ({
           ...item,
           __index: data[idx]?.__index,
           __errors: data[idx]?.__errors,
         }));
         
-        // Update the data and run validations
+        console.log('About to update data with AI corrections:', {
-        await updateData(newData);
+          originalDataSample: data.slice(0, 2),
-      }
+          newDataSample: newData.slice(0, 2),
+          correctionCount: result.changes?.length || 0
+        });
         
-      // Show changes and warnings in dialog
+        // First validate the new data to ensure all validation rules are applied
+        try {
+          // Use the current fields with updated options for validation
+          const currentFields = fieldsWithUpdatedOptions as unknown as Fields<T>;
+          
+          // Validate the data with the hooks
+          const validatedData = await addErrorsAndRunHooks<T>(
+            newData,
+            currentFields,
+            rowHook,
+            tableHook
+          );
+          
+          // Update the component state with the validated data
+          setData(validatedData as (Data<T> & ExtendedMeta)[]);
+          
+          // Force a small delay to ensure React updates the state before showing the results
+          await new Promise(resolve => setTimeout(resolve, 50));
+          
+          // Log a sample of the state after update, focus on line/subline fields
+          console.log('State after AI validation:', {
+            sampleLineValues: validatedData.slice(0, 3).map(row => ({ 
+              line: row.line, 
+              subline: row.subline,
+              lineDisplay: getFieldDisplayValue('line', row.line),
+              sublineDisplay: getFieldDisplayValue('subline', row.subline)
+            })),
+            fieldsWithOptions: fieldsWithUpdatedOptions
+              .filter(f => f.fieldType.type === 'select' && ['line', 'subline'].includes(f.key as string))
+              .map(f => ({ 
+                key: f.key, 
+                options: 'options' in f.fieldType ? f.fieldType.options?.length || 0 : 0 
+              }))
+          });
+          
+          console.log('Data updated after AI validation:', {
+            dataLength: validatedData.length,
+            hasErrors: validatedData.some(row => row.__errors && Object.keys(row.__errors).length > 0)
+          });
+          
+          // Show changes and warnings in dialog after data is updated
           setAiValidationDetails({
             changes: result.changes || [],
             warnings: result.warnings || [],
+            changeDetails: result.changeDetails || [],
             isOpen: true,
+            originalData: originalDataCopy  // Use the stored original data
           });
+        } catch (error) {
+          console.error('Error validating AI corrections:', error);
+          // Fall back to basic update without validation
+          setData(newData);
+          
+          // Still show the result dialog even if validation failed
+          setAiValidationDetails({
+            changes: result.changes || [],
+            warnings: result.warnings || [],
+            changeDetails: result.changeDetails || [],
+            isOpen: true,
+            originalData: originalDataCopy,  // Use the stored original data
+          });
+        }
+      }
 
       setAiValidationProgress(prev => ({
         ...prev,
@@ -1654,7 +1939,6 @@ export const ValidationStep = <T extends string>({
       setTimeout(() => {
         setAiValidationProgress(prev => ({ ...prev, isOpen: false }));
       }, 1000);
-
     } catch (error) {
       console.error('AI Validation Error:', error);
       toast({
@@ -1668,7 +1952,18 @@ export const ValidationStep = <T extends string>({
         step: -1
       }));
     } finally {
+      // Clear the interval when we're done (success or error)
+      if (window.aiValidationTimer) {
+        clearInterval(window.aiValidationTimer);
+        window.aiValidationTimer = undefined;
+      }
       setIsAiValidating(false);
+      
+      // Only set to 100% when actually complete (or in error state)
+      setAiValidationProgress(prev => ({
+        ...prev,
+        progressPercent: prev.step === -1 ? prev.progressPercent : 100 // Only show 100% if successful completion
+      }));
     }
   };
 
@@ -1719,7 +2014,8 @@ export const ValidationStep = <T extends string>({
       // Log the response stats
       console.log('Debug response stats:', {
         promptLength: debugData.promptLength,
-        taxonomyStats: debugData.taxonomyStats
+        taxonomyStats: debugData.taxonomyStats,
+        estimatedProcessingTime: debugData.estimatedProcessingTime
       });
       
       setCurrentPrompt(prev => ({ 
@@ -1727,6 +2023,15 @@ export const ValidationStep = <T extends string>({
         prompt: debugData.sampleFullPrompt,
         isLoading: false 
       }));
+
+      // Store the estimated processing time for later use
+      if (debugData.estimatedProcessingTime?.seconds) {
+        setAiValidationProgress(prev => ({
+          ...prev,
+          estimatedSeconds: debugData.estimatedProcessingTime.seconds,
+          promptLength: debugData.promptLength
+        }));
+      }
     } catch (error) {
       console.error('Error fetching prompt:', error);
       toast({
@@ -1738,6 +2043,83 @@ export const ValidationStep = <T extends string>({
     }
   };
 
+  // Function to get display value for field values (including IDs)
+  const getFieldDisplayValue = (fieldKey: string, value: any): string => {
+    const field = fields.find(f => f.key === fieldKey);
+    if (!field) return String(value);
+
+    // Handle different field types
+    if (field.fieldType.type === 'select' || field.fieldType.type === 'multi-select') {
+      const options = field.fieldType.options || [];
+      
+      // Handle array of values (multi-select)
+      if (Array.isArray(value)) {
+        return value.map(v => {
+          const option = options.find(opt => String(opt.value) === String(v));
+          return option ? option.label : v;
+        }).join(', ');
+      }
+      
+      // Handle single value
+      const option = options.find(opt => String(opt.value) === String(value));
+      return option ? option.label : String(value);
+    }
+    
+    return String(value);
+  };
+
+  // Add a function to revert a specific AI validation change
+  const revertAiChange = (productIndex: number, fieldKey: string) => {
+    // Ensure we have the original data
+    if (!aiValidationDetails.originalData) {
+      console.error('Cannot revert: original data not available');
+      return;
+    }
+
+    // Find the original product and current product
+    const originalProduct = aiValidationDetails.originalData[productIndex];
+    if (!originalProduct) {
+      console.error(`Cannot revert: original product at index ${productIndex} not found`);
+      return;
+    }
+    
+    const currentProduct = data[productIndex];
+    if (!currentProduct) {
+      console.error(`Cannot revert: current product at index ${productIndex} not found`);
+      return;
+    }
+
+    // Get the original value in a type-safe way
+    const originalValue = fieldKey in originalProduct ? 
+      (originalProduct as Record<string, any>)[fieldKey] : undefined;
+    
+    console.log(`Reverting change to field "${fieldKey}" for product at index ${productIndex}`, {
+      original: originalValue,
+      current: fieldKey in currentProduct ? (currentProduct as Record<string, any>)[fieldKey] : undefined
+    });
+
+    // Create a new data array with the reverted field
+    const newData = [...data];
+    
+    // Create a new product object with the reverted field
+    newData[productIndex] = {
+      ...newData[productIndex],
+      [fieldKey]: originalValue
+    } as Data<T> & ExtendedMeta; // Cast to ensure type safety
+
+    // Update the data state
+    setData(newData);
+
+    // Re-validate to update error states
+    updateData(newData, [productIndex]);
+
+    // Show success notification
+    toast({
+      title: "Change reverted",
+      description: `Reverted the change to "${fieldKey}"`,
+    });
+  };
+
   return (
     <div className="flex h-[calc(100vh-9.5rem)] flex-col">
       <CopyDownDialog 
@@ -1809,19 +2191,43 @@ export const ValidationStep = <T extends string>({
                   <div 
                     className="h-full bg-primary transition-all duration-500" 
                     style={{ 
-                      width: `${(aiValidationProgress.step / 5) * 100}%`,
+                      width: `${aiValidationProgress.progressPercent ?? (aiValidationProgress.step / 5) * 100}%`,
                       backgroundColor: aiValidationProgress.step === -1 ? 'var(--destructive)' : undefined
                     }}
                   />
                 </div>
               </div>
               <div className="text-sm text-muted-foreground w-12 text-right">
-                {aiValidationProgress.step === -1 ? '❌' : `${Math.round((aiValidationProgress.step / 5) * 100)}%`}
+                {aiValidationProgress.step === -1 ? '❌' : `${aiValidationProgress.progressPercent ?? Math.round((aiValidationProgress.step / 5) * 100)}%`}
               </div>
             </div>
             <p className="text-center text-sm text-muted-foreground">
               {aiValidationProgress.status}
             </p>
+            {aiValidationProgress.estimatedSeconds && aiValidationProgress.elapsedSeconds !== undefined && aiValidationProgress.step > 0 && aiValidationProgress.step < 5 && (
+              <div className="text-center text-sm">
+                {(() => {
+                  // Calculate time remaining using the elapsed seconds
+                  const elapsedSeconds = aiValidationProgress.elapsedSeconds;
+                  const totalEstimatedSeconds = aiValidationProgress.estimatedSeconds;
+                  const remainingSeconds = Math.max(0, totalEstimatedSeconds - elapsedSeconds);
+                  
+                  // Format time remaining
+                  if (remainingSeconds < 60) {
+                    return `Approximately ${Math.round(remainingSeconds)} seconds remaining`;
+                  } else {
+                    const minutes = Math.floor(remainingSeconds / 60);
+                    const seconds = Math.round(remainingSeconds % 60);
+                    return `Approximately ${minutes}m ${seconds}s remaining`;
+                  }
+                })()}
+                {aiValidationProgress.promptLength && (
+                  <p className="mt-1 text-xs text-muted-foreground">
+                    Prompt length: {aiValidationProgress.promptLength.toLocaleString()} characters
+                  </p>
+                )}
+              </div>
+            )}
           </div>
         </DialogContent>
       </Dialog>
@@ -1829,7 +2235,7 @@ export const ValidationStep = <T extends string>({
         open={aiValidationDetails.isOpen} 
         onOpenChange={(open) => setAiValidationDetails(prev => ({ ...prev, isOpen: open }))}
       >
-        <DialogContent className="max-w-2xl">
+        <DialogContent className="max-w-4xl">
           <DialogHeader>
             <DialogTitle>AI Validation Results</DialogTitle>
             <DialogDescription>
@@ -1837,7 +2243,90 @@ export const ValidationStep = <T extends string>({
             </DialogDescription>
           </DialogHeader>
           <ScrollArea className="max-h-[60vh]">
-            {aiValidationDetails.changes.length > 0 && (
+            {aiValidationDetails.changeDetails && aiValidationDetails.changeDetails.length > 0 ? (
+              <div className="mb-6 space-y-6">
+                <h3 className="font-semibold text-lg">Detailed Changes:</h3>
+                {aiValidationDetails.changeDetails.map((product, i) => (
+                  <div key={i} className="border rounded-md p-4">
+                    <h4 className="font-medium text-base mb-3">{product.title}</h4>
+                    <Table>
+                      <TableHeader>
+                        <TableRow>
+                          <TableHead className="w-1/4">Field</TableHead>
+                          <TableHead className="w-3/8">Original Value</TableHead>
+                          <TableHead className="w-3/8">Corrected Value</TableHead>
+                          <TableHead className="w-1/8 text-right">Action</TableHead>
+                        </TableRow>
+                      </TableHeader>
+                      <TableBody>
+                        {product.changes.map((change, j) => {
+                          const field = fields.find(f => f.key === change.field);
+                          return (
+                            <TableRow key={j}>
+                              <TableCell className="font-medium">{field?.label || change.field}</TableCell>
+                              <TableCell>
+                                <div className="space-y-1">
+                                  <div>{getFieldDisplayValue(change.field, change.original)}</div>
+                                  {/* Show raw value if it's an ID */}
+                                  {change.original && typeof change.original === 'string' && 
+                                   !isNaN(Number(change.original)) && 
+                                   getFieldDisplayValue(change.field, change.original) !== change.original && (
+                                    <div className="text-xs text-muted-foreground">ID: {change.original}</div>
+                                  )}
+                                </div>
+                              </TableCell>
+                              <TableCell>
+                                <div className="space-y-1">
+                                  <div>{getFieldDisplayValue(change.field, change.corrected)}</div>
+                                  {/* Show raw value if it's an ID */}
+                                  {change.corrected && typeof change.corrected === 'string' && 
+                                   !isNaN(Number(change.corrected)) && 
+                                   getFieldDisplayValue(change.field, change.corrected) !== change.corrected && (
+                                    <div className="text-xs text-muted-foreground">ID: {change.corrected}</div>
+                                  )}
+                                </div>
+                              </TableCell>
+                              <TableCell className="text-right">
+                                <div className="mt-2">
+                                  <Button
+                                    variant="outline"
+                                    size="sm"
+                                    onClick={() => {
+                                      // Find the product in the current data
+                                      const currentDataIndex = data.findIndex(
+                                        d => d.__index === data[product.productIndex]?.__index
+                                      );
+                                      if (currentDataIndex >= 0) {
+                                        // Create new data array with the original value for this field
+                                        const newData = [...data];
+                                        newData[currentDataIndex] = {
+                                          ...newData[currentDataIndex],
+                                          [change.field]: change.original
+                                        };
+                                        // Update the data
+                                        updateData(newData, [currentDataIndex]);
+                                        // Show toast
+                                        toast({
+                                          title: "Change reverted",
+                                          description: `Reverted ${change.field} to original value`,
+                                        });
+                                      }
+                                    }}
+                                  >
+                                    Revert Change
+                                  </Button>
+                                </div>
+                              </TableCell>
+                            </TableRow>
+                          );
+                        })}
+                      </TableBody>
+                    </Table>
+                  </div>
+                ))}
+              </div>
+            ) : (
+              aiValidationDetails.changes.length > 0 && (
                 <div className="mb-4">
                   <h3 className="font-semibold mb-2">Changes Made:</h3>
                   <ul className="space-y-2">
@@ -1849,6 +2338,7 @@ export const ValidationStep = <T extends string>({
                     ))}
                   </ul>
                 </div>
+              )
             )}
             {aiValidationDetails.warnings.length > 0 && (
               <div>
@@ -2077,3 +2567,10 @@ export const ValidationStep = <T extends string>({
     </div>
   )
 }
+
+// Add TypeScript declaration for our global timer variable
+declare global {
+  interface Window {
+    aiValidationTimer?: NodeJS.Timeout;
+  }
+}

inventory/src/pages/AiValidationDebug.tsx

@@ -23,6 +23,10 @@ interface DebugData {
   basePrompt: string
   sampleFullPrompt: string
   promptLength: number
+  estimatedProcessingTime?: {
+    seconds: number | null
+    sampleCount: number
+  }
 }
 
 export function AiValidationDebug() {
@@ -147,6 +151,23 @@ export function AiValidationDebug() {
                     Cost: <span id="tokenCost">{((Math.round(debugData.promptLength / 4) / 1_000_000) * 3 * 100).toFixed(1)}</span>¢
                   </div>
                 </div>
+                {debugData.estimatedProcessingTime && (
+                  <div className="mt-4 p-3 bg-muted rounded-md">
+                    <h3 className="text-sm font-medium mb-2">Processing Time Estimate</h3>
+                    {debugData.estimatedProcessingTime.seconds ? (
+                      <div className="space-y-1">
+                        <div className="text-sm">
+                          Estimated time: {formatTime(debugData.estimatedProcessingTime.seconds)}
+                        </div>
+                        <div className="text-xs text-muted-foreground">
+                          Based on {debugData.estimatedProcessingTime.sampleCount} similar validation{debugData.estimatedProcessingTime.sampleCount !== 1 ? 's' : ''}
+                        </div>
+                      </div>
+                    ) : (
+                      <div className="text-sm text-muted-foreground">No historical data available for this prompt size</div>
+                    )}
+                  </div>
+                )}
               </div>
             </CardContent>
           </Card>
@@ -166,3 +187,14 @@ export function AiValidationDebug() {
     </div>
   )
 }
+
+// Helper function to format time in a human-readable way
+function formatTime(seconds: number): string {
+  if (seconds < 60) {
+    return `${Math.round(seconds)} seconds`;
+  } else {
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = Math.round(seconds % 60);
+    return `${minutes}m ${remainingSeconds}s`;
+  }
+}