import React, { useCallback, useEffect, useMemo, useState, memo, useRef, useLayoutEffect } from "react"
import { apiFetch } from '@/utils/api';
import { useRsi } from "../../hooks/useRsi"
import { setColumn } from "./utils/setColumn"
import { setIgnoreColumn } from "./utils/setIgnoreColumn"
import { setSubColumn } from "./utils/setSubColumn"
import { normalizeTableData } from "./utils/normalizeTableData"
import type { Field, Fields } from "../../types"
import { getMatchedColumns } from "./utils/getMatchedColumns"
import { findUnmatchedRequiredFields } from "./utils/findUnmatchedRequiredFields"
import { toast } from "sonner"
import { DeepReadonly as TsDeepReadonly } from "ts-essentials"
import type {
  MatchColumnsProps,
  GlobalSelections,
  MatchedSelectColumn,
  MatchedSelectOptionsColumn,
  MatchedMultiSelectColumn,
  Column,
  Columns,
} from "./types"
import { ColumnType } from "./types"
import {
  Select,
  SelectContent,
  SelectItem,
  SelectTrigger,
  SelectValue,
} from "@/components/ui/select"
import { useQuery, useQueryClient } from "@tanstack/react-query"
import config from "@/config"
import { Button } from "@/components/ui/button"
import { CheckCircle2, AlertCircle, EyeIcon, EyeOffIcon, ArrowRightIcon, XIcon, FileSpreadsheetIcon, LinkIcon, CheckIcon, ChevronsUpDown, Bot, RefreshCw, Plus } from "lucide-react"
import { Separator } from "@/components/ui/separator"
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { Badge } from "@/components/ui/badge"
import { ScrollArea } from "@/components/ui/scroll-area"
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"
import {
  Command,
  CommandEmpty,
  CommandGroup,
  CommandInput,
  CommandItem,
  CommandList,
} from "@/components/ui/command"
import { cn } from "@/lib/utils"
import { CreateProductCategoryDialog, type CreatedCategoryInfo } from "@/components/product-import/CreateProductCategoryDialog"

// Extract components to reduce re-renders
const ColumnActions = memo(function ColumnActions({ 
  column, 
  onIgnore, 
  onToggleAiSupplemental,
  toggleValueMapping, 
  isExpanded, 
  canExpandValues,
  isAiSupplemental,
}: { 
  column: any, 
  onIgnore: (index: number) => void, 
  onToggleAiSupplemental: (index: number) => void,
  toggleValueMapping: (index: number) => void, 
  isExpanded: boolean, 
  canExpandValues: boolean,
  isAiSupplemental: boolean,
}) {
  // Create stable callback references to prevent unnecessary re-renders
  const handleIgnore = useCallback(() => {
    onIgnore(column.index);
  }, [onIgnore, column.index]);

  const handleToggleMapping = useCallback(() => {
    toggleValueMapping(column.index);
  }, [toggleValueMapping, column.index]);

  const handleToggleAiSupplemental = useCallback(() => {
    onToggleAiSupplemental(column.index);
  }, [onToggleAiSupplemental, column.index]);

  return (
    <div className="flex justify-end items-center gap-6">
      {canExpandValues && (
        <Button 
          variant="ghost" 
          size="sm"
          onClick={handleToggleMapping}
          className="h-7 px-2"
        >
          {isExpanded ? 
            <>
              <EyeOffIcon className="h-3.5 w-3.5 mr-1" />
              Hide values
            </> : 
            <>
              <LinkIcon className="h-3.5 w-3.5 mr-1" />
              Map values
            </>
          }
        </Button>
      )}
      <TooltipProvider>
        <Tooltip>
          <TooltipTrigger asChild>
            <Button
              variant={isAiSupplemental ? "default" : "ghost"}
              size="icon"
              onClick={handleToggleAiSupplemental}
              className=""
            >
              <Bot className={cn("h-6 w-6", isAiSupplemental && "text-primary-foreground")} />
            </Button>
          </TooltipTrigger>
          <TooltipContent className='max-w-[200px]'>
            <p>{isAiSupplemental ? "Will be sent to AI model for additional context - click to disable" : "Send this column to AI model for additional context without mapping to a field directly"}</p>
          </TooltipContent>
        </Tooltip>
      </TooltipProvider>
      <TooltipProvider>
        <Tooltip>
          <TooltipTrigger asChild>
            <Button   
              variant="ghost" 
              size="icon"
              onClick={handleIgnore}
              className=""
            >
              <XIcon className="h-6 w-6" />
            </Button>
          </TooltipTrigger>
          <TooltipContent>
            <p>Don't use this column</p>
          </TooltipContent>
        </Tooltip>
      </TooltipProvider>
    </div>
  );
});

// Additional performance optimization - use a memoized row renderer for column sample data
const MemoizedColumnSamplePreview = React.memo(function ColumnSamplePreview({ samples }: { samples: any[] }) {
  return (
    <div className="flex items-center justify-center">
      <Popover>
        <PopoverTrigger asChild>
          <Button variant="ghost" size="icon" className="h-8 w-8">
            <FileSpreadsheetIcon className="h-4 w-4 text-muted-foreground" />
          </Button>
        </PopoverTrigger>
        <PopoverContent side="right" align="start" className="w-[280px] p-0" onWheel={(e) => e.stopPropagation()}>
          <div className="max-h-[300px] overflow-y-auto overscroll-contain" style={{ overscrollBehavior: 'contain' }}>
            <div className="p-3 space-y-2">
              {samples.map((sample, i) => (
                <div key={i} className="text-sm break-words">
                  <span className="font-medium">{String(sample || '(empty)')}</span>
                  {i < samples.length - 1 && <Separator className="w-full my-2" />}
                </div>
              ))}
            </div>
          </div>
        </PopoverContent>
      </Popover>
    </div>
  );
});

// Replace the original ColumnSamplePreview with more optimized version
const ColumnSamplePreview = MemoizedColumnSamplePreview;

// Add a memoized component for value mappings
const ValueMappings = memo(function ValueMappings({ 
  column, 
  fieldOptions, 
  onSubChange 
}: { 
  column: any, 
  fieldOptions: any[], 
  onSubChange: (value: string, columnIndex: number, entry: string) => void 
}) {
  // Use a React.useMemo for expensive calculations
  const matchedOptions = useMemo(() => column.matchedOptions || [], [column.matchedOptions]);
  const columnIndex = useMemo(() => column.index, [column.index]);
  
  if (!fieldOptions || fieldOptions.length === 0) {
    return (
      <div className="pl-10 pr-2 py-4 bg-muted/30 border-t border-b">
        <p className="text-sm text-muted-foreground">
          No options available for this field. Options mapping is not required.
        </p>
      </div>
    );
  }
  
  return (
    <div className="pl-10 pr-2 py-3 bg-muted/30 border-t border-b">
      <div className="mb-2">
        <h4 className="text-sm font-medium">Match spreadsheet values with corresponding backend options</h4>
      </div>
      <div className="space-y-2">
        {matchedOptions.map((matched: any, i: number) => {
          // Set default value if none exists
          const currentValue = (matched.value as string) || "";
          // Ensure entry is a string
          const entryValue = matched.entry || "";
          const isUnmapped = !currentValue;
          
          return (
            <div key={i} className="grid grid-cols-3 gap-2 items-center">
              <div className={`text-sm truncate ${isUnmapped ? "font-medium text-amber-600" : ""}`}>
                {entryValue || '(empty)'}
              </div>
              <div className="flex-grow flex justify-center">
                <ArrowRightIcon className={`h-4 w-4 ${isUnmapped ? "text-amber-600" : "text-muted-foreground"}`} />
              </div>
              <Select
                value={currentValue}
                onValueChange={(value: string) => onSubChange(value, columnIndex, entryValue)}
              >
                <SelectTrigger className={`h-8 ${isUnmapped ? "border-amber-600 text-amber-600" : ""}`}>
                  <SelectValue placeholder="Select option..." />
                </SelectTrigger>
                <SelectContent>
                  {fieldOptions.map((option: any) => (
                    <SelectItem key={option.value} value={option.value}>
                      {option.label}
                    </SelectItem>
                  ))}
                </SelectContent>
              </Select>
            </div>
          );
        })}
      </div>
    </div>
  );
});

// Add these new components before the MatchColumnsStep component
const SupplierSelector = React.memo(function SupplierSelector({ 
  value, 
  onChange, 
  suppliers,
  disabled 
}: { 
  value?: string; 
  onChange: (value: string) => void; 
  suppliers: any[];
  disabled?: boolean;
}) {
  const [open, setOpen] = useState(false);
  const handleCommandListWheel = (e: React.WheelEvent) => {
    e.currentTarget.scrollTop += e.deltaY;
    e.stopPropagation();
  };

  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          role="combobox"
          className="w-full justify-between"
          disabled={disabled}
        >
          {value
            ? suppliers?.find((supplier: any) => 
                supplier.value === value)?.label || "Select supplier..."
            : "Select supplier..."}
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0">
        <Command>
          <CommandInput placeholder="Search suppliers..." />
          <CommandList className="max-h-[200px] overflow-y-auto" onWheel={handleCommandListWheel}>
            <CommandEmpty>No suppliers found.</CommandEmpty>
            <CommandGroup>
              {suppliers?.map((supplier: any) => (
                <CommandItem
                  key={supplier.value}
                  value={`${supplier.label} ${supplier.value}`}
                  onSelect={() => {
                    onChange(supplier.value);
                    setOpen(false); // Close popover after selection
                  }}
                >
                  {supplier.label}
                  <CheckIcon
                    className={cn(
                      "ml-auto h-4 w-4",
                      value === supplier.value 
                        ? "opacity-100" 
                        : "opacity-0"
                    )}
                  />
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
});

const CompanySelector = React.memo(function CompanySelector({ 
  value, 
  onChange, 
  companies,
  disabled 
}: { 
  value?: string; 
  onChange: (value: string) => void; 
  companies: any[];
  disabled?: boolean;
}) {
  const [open, setOpen] = useState(false);
  const [query, setQuery] = useState("");
  const handleCommandListWheel = (e: React.WheelEvent) => {
    e.currentTarget.scrollTop += e.deltaY;
    e.stopPropagation();
  };

  // Filtered and sliced list to prevent UI freezes with very large lists
  const filteredCompanies = React.useMemo(() => {
    if (!query.trim()) {
      // When no search, show a capped subset for performance
      return (companies || []).slice(0, 200);
    }
    const q = query.toLowerCase();
    return (companies || []).filter((c: any) => (
      String(c.label || '').toLowerCase().includes(q) ||
      String(c.value || '').toLowerCase().includes(q)
    ));
  }, [companies, query]);

  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          role="combobox"
          className="w-full justify-between"
          disabled={disabled}
        >
          {value
            ? companies?.find((company: any) => 
                company.value === value)?.label || "Select company..."
            : "Select company..."}
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0">
        <Command>
          <CommandInput placeholder="Search companies..." value={query} onValueChange={setQuery} />
          <CommandList className="max-h-[200px] overflow-y-auto" onWheel={handleCommandListWheel}>
            <CommandEmpty>No companies found.</CommandEmpty>
            <CommandGroup>
              {filteredCompanies.map((company: any) => (
                <CommandItem
                  key={company.value}
                  value={`${company.label} ${company.value}`}
                  onSelect={() => {
                    onChange(company.value);
                    setOpen(false); // Close popover after selection
                  }}
                >
                  {company.label}
                  <CheckIcon
                    className={cn(
                      "ml-auto h-4 w-4",
                      value === company.value 
                        ? "opacity-100" 
                        : "opacity-0"
                    )}
                  />
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
});

const LineSelector = React.memo(function LineSelector({ 
  value, 
  onChange, 
  lines,
  disabled
}: { 
  value?: string; 
  onChange: (value: string) => void; 
  lines: any[];
  disabled: boolean;
}) {
  const [open, setOpen] = useState(false);
  const handleCommandListWheel = (e: React.WheelEvent) => {
    e.currentTarget.scrollTop += e.deltaY;
    e.stopPropagation();
  };

  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          role="combobox"
          className="w-full justify-between"
          disabled={disabled}
        >
          {value
            ? lines?.find((line: any) => 
                line.value === value)?.label || "Select line..."
            : "Select line..."}
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0">
        <Command>
          <CommandInput placeholder="Search lines..." />
          <CommandList className="max-h-[200px] overflow-y-auto" onWheel={handleCommandListWheel}>
            <CommandEmpty>No lines found.</CommandEmpty>
            <CommandGroup>
              {lines?.map((line: any) => (
                <CommandItem
                  key={line.value}
                  value={`${line.label} ${line.value}`}
                  onSelect={() => {
                    onChange(line.value);
                    setOpen(false); // Close popover after selection
                  }}
                >
                  {line.label}
                  <CheckIcon
                    className={cn(
                      "ml-auto h-4 w-4",
                      value === line.value 
                        ? "opacity-100" 
                        : "opacity-0"
                    )}
                  />
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
});

const SubLineSelector = React.memo(function SubLineSelector({ 
  value, 
  onChange, 
  sublines,
  disabled
}: { 
  value?: string; 
  onChange: (value: string) => void; 
  sublines: any[];
  disabled: boolean;
}) {
  const [open, setOpen] = useState(false);
  const handleCommandListWheel = (e: React.WheelEvent) => {
    e.currentTarget.scrollTop += e.deltaY;
    e.stopPropagation();
  };

  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          role="combobox"
          className="w-full justify-between"
          disabled={disabled}
        >
          {value
            ? sublines?.find((subline: any) => 
                subline.value === value)?.label || "Select sub line..."
            : "Select sub line..."}
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0">
        <Command>
          <CommandInput placeholder="Search sub lines..." />
          <CommandList className="max-h-[200px] overflow-y-auto" onWheel={handleCommandListWheel}>
            <CommandEmpty>No sub lines found.</CommandEmpty>
            <CommandGroup>
              {sublines?.map((subline: any) => (
                <CommandItem
                  key={subline.value}
                  value={`${subline.label} ${subline.value}`}
                  onSelect={() => {
                    onChange(subline.value);
                    setOpen(false); // Close popover after selection
                  }}
                >
                  {subline.label}
                  <CheckIcon
                    className={cn(
                      "ml-auto h-4 w-4",
                      value === subline.value 
                        ? "opacity-100" 
                        : "opacity-0"
                    )}
                  />
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
});

// Add this new component before the MatchColumnsStep component
const FieldSelector = React.memo(function FieldSelector({ 
  column, 
  fieldCategories,
  allFields,
  onChange,
  isFieldMappedToOtherColumn,
  handleCommandListWheel
}: { 
  column: any;
  isUnmapped?: boolean;
  fieldCategories: any[];
  allFields: any[];
  onChange: (value: string) => void;
  isFieldMappedToOtherColumn: (fieldKey: string, currentColumnIndex: number) => { isMapped: boolean, columnHeader?: string };
  handleCommandListWheel: (e: React.WheelEvent) => void;
}) {
  const [open, setOpen] = useState(false);
  
  // For ignored columns, show a badge
  if (column.type === ColumnType.ignored) {
    return <Badge variant="outline">Ignored</Badge>;
  }
  
  // Get the current value if this is a mapped column
  const currentValue = "value" in column ? column.value as string : undefined;
  
  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button 
          variant="outline" 
          role="combobox" 
          className="w-full justify-between"
        >
          {currentValue
            ? allFields.find(f => f.key === currentValue)?.label || "Select field..."
            : "Select field..."}
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[var(--radix-popover-trigger-width)] p-0">
        <Command>
          <CommandInput placeholder="Search fields..." />
          <CommandList className="max-h-[300px] overflow-y-auto" onWheel={handleCommandListWheel}>
            <CommandEmpty>No fields found.</CommandEmpty>
            {fieldCategories.map(category => (
              <CommandGroup key={category.name} heading={category.name}>
                {category.fields.map((field: { key: string; label: string }) => {
                  const { isMapped, columnHeader } = isFieldMappedToOtherColumn(field.key as string, column.index);
                  return (
                    <CommandItem
                      key={field.key as string}
                      value={field.key as string}
                      onSelect={(value: string) => {
                        onChange(value);
                        setOpen(false); // Close the popover after selection
                      }}
                      className={isMapped ? "opacity-70" : ""}
                    >
                      <div className="flex-1 flex items-center justify-between">
                        <span>{field.label}</span>
                        {isMapped ? (
                          <span className="text-xs text-muted-foreground ml-2">
                            (mapped to "{columnHeader}")
                          </span>
                        ) : null}
                      </div>
                      {currentValue === field.key && (
                        <CheckIcon className="ml-2 h-4 w-4" />
                      )}
                    </CommandItem>
                  );
                })}
              </CommandGroup>
            ))}
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
});

const MatchColumnsStepComponent = <T extends string>({
  data,
  headerValues,
  onContinue,
  onBack,
  initialGlobalSelections,
  initialColumns
}: MatchColumnsProps<T>): JSX.Element => {
  const { fields, autoMapHeaders, autoMapSelectValues, autoMapDistance, translations } = useRsi<T>()
  const queryClient = useQueryClient()
  const [isLoading, setIsLoading] = useState(false)
  const [isRefreshing, setIsRefreshing] = useState(false)

  const [columns, setColumns] = useState<Columns<T>>(() => {
    // Restoring from a previous visit (back-nav): reuse the prior mappings verbatim
    // instead of re-deriving empty columns + auto-mapping.
    if (initialColumns && initialColumns.length > 0) {
      return initialColumns;
    }

    // Helper function to check if a column is completely empty
    const isColumnEmpty = (columnIndex: number) => {
      return data.every(row => {
        const value = row[columnIndex];
        return value === undefined || value === null || (typeof value === 'string' && value.trim() === '');
      });
    };

    // Do not remove spread, it indexes empty array elements, otherwise map() skips over them
    const allColumns = ([...headerValues] as string[]).map((value, index) => ({
      type: ColumnType.empty as ColumnType.empty,
      index,
      header: value ?? ""
    }));

    // Filter out completely empty columns
    return allColumns.filter(col => !isColumnEmpty(col.index)) as Columns<T>;
  })
  const [globalSelections, setGlobalSelections] = useState<GlobalSelections>(initialGlobalSelections || {})
  const [showAllColumns, setShowAllColumns] = useState(true)
  const [expandedValues, setExpandedValues] = useState<number[]>([])
  const [userCollapsedColumns, setUserCollapsedColumns] = useState<number[]>([])
  // When restoring prior columns, suppress the initial header auto-map so it doesn't
  // overwrite the restored mappings (the ref guards within a mount; seeding it true
  // covers the fresh-mount-on-back-nav case).
  const hasAutoMappedRef = useRef(!!(initialColumns && initialColumns.length > 0))

  // Toggle with immediate visual feedback
  const toggleValueMappingOptimized = useCallback((columnIndex: number) => {
    if (expandedValues.includes(columnIndex)) {
      // User is collapsing this column - add to userCollapsedColumns
      setUserCollapsedColumns(prev => [...prev, columnIndex]);
      setExpandedValues(prev => prev.filter(idx => idx !== columnIndex));
    } else {
      // User is expanding this column - remove from userCollapsedColumns
      setUserCollapsedColumns(prev => prev.filter(idx => idx !== columnIndex));
      setExpandedValues(prev => [...prev, columnIndex]);
    }
  }, [expandedValues]);

  // Check if column is expandable (has value mappings)
  const isExpandable = useCallback((column: Column<T>) => {
    return (
      column.type === ColumnType.matchedSelect ||
      column.type === ColumnType.matchedSelectOptions ||
      column.type === ColumnType.matchedMultiSelect
    );
  }, []);

  // Initialize with any provided global selections
  useEffect(() => {
    if (initialGlobalSelections) {
      setGlobalSelections(initialGlobalSelections)
    }
  }, [initialGlobalSelections])

  // Fetch product lines when company is selected
  const { data: productLines } = useQuery({
    queryKey: ["product-lines", globalSelections.company],
    queryFn: async () => {
      if (!globalSelections.company) return [];
      const response = await apiFetch(`${config.apiUrl}/import/product-lines/${globalSelections.company}`);
      if (!response.ok) {
        throw new Error("Failed to fetch product lines");
      }
      return response.json();
    },
    enabled: !!globalSelections.company,
    staleTime: 600000, // 10 minutes (increased from 60 seconds)
  });

  // Fetch sublines when line is selected
  const { data: sublines } = useQuery({
    queryKey: ["sublines", globalSelections.line],
    queryFn: async () => {
      if (!globalSelections.line) return [];
      const response = await apiFetch(`${config.apiUrl}/import/sublines/${globalSelections.line}`);
      if (!response.ok) {
        throw new Error("Failed to fetch sublines");
      }
      return response.json();
    },
    enabled: !!globalSelections.line,
    staleTime: 600000, // 10 minutes (increased from 60 seconds)
  });

  // Find mapped column for a specific field
  const findMappedColumnForField = useCallback((fieldKey: string) => {
    return columns.find(col => {
      // Check if it's a matched column with value property
      return "value" in col && col.value === fieldKey && 
        (col.type === ColumnType.matched || 
         col.type === ColumnType.matchedCheckbox || 
         col.type === ColumnType.matchedSelect || 
         col.type === ColumnType.matchedSelectOptions || 
         col.type === ColumnType.matchedMultiInput ||
         col.type === ColumnType.matchedMultiSelect);
    });
  }, [columns]);

  // Check if global setting fields are mapped to columns
  const isSupplierMappedToColumn = useMemo(() => !!findMappedColumnForField('supplier'), [findMappedColumnForField]);
  const isCompanyMappedToColumn = useMemo(() => !!findMappedColumnForField('company'), [findMappedColumnForField]);
  const isLineMappedToColumn = useMemo(() => !!findMappedColumnForField('line'), [findMappedColumnForField]);
  const isSublineMappedToColumn = useMemo(() => !!findMappedColumnForField('subline'), [findMappedColumnForField]);

  // Get mapped company value (if company is mapped to a column)
  const mappedCompanyColumn = useMemo(() => findMappedColumnForField('company'), [findMappedColumnForField]);
  const mappedCompanyValue = useMemo(() => {
    // If using global selection, return that
    if (globalSelections.company) return globalSelections.company;

    // If company is mapped to a column, get the first value
    if (mappedCompanyColumn && "matchedOptions" in mappedCompanyColumn) {
      const firstEntry = data[0]?.[mappedCompanyColumn.index];
      // Find the mapped value for this entry
      const mappedOption = mappedCompanyColumn.matchedOptions.find(opt => opt.entry === firstEntry);
      return mappedOption?.value as string || null;
    }
    
    return null;
  }, [globalSelections.company, mappedCompanyColumn, data]);

  // Get mapped line value (if line is mapped to a column)
  const mappedLineColumn = useMemo(() => findMappedColumnForField('line'), [findMappedColumnForField]);
  const mappedLineValue = useMemo(() => {
    // If using global selection, return that
    if (globalSelections.line) return globalSelections.line;

    // If line is mapped to a column, get the first value
    if (mappedLineColumn && "matchedOptions" in mappedLineColumn) {
      const firstEntry = data[0]?.[mappedLineColumn.index];
      // Find the mapped value for this entry
      const mappedOption = mappedLineColumn.matchedOptions.find(opt => opt.entry === firstEntry);
      return mappedOption?.value as string || null;
    }
    
    return null;
  }, [globalSelections.line, mappedLineColumn, data]);

  // Fetch product lines for mapped company
  const { data: mappedProductLines } = useQuery({
    queryKey: ["product-lines-mapped", mappedCompanyValue],
    queryFn: async () => {
      if (!mappedCompanyValue) return [];
      const response = await apiFetch(`${config.apiUrl}/import/product-lines/${mappedCompanyValue}`);
      if (!response.ok) {
        throw new Error("Failed to fetch product lines for mapped company");
      }
      return response.json();
    },
    enabled: !!mappedCompanyValue && mappedCompanyValue !== globalSelections.company,
    staleTime: 600000, // 10 minutes (increased from 60 seconds)
  });

  // Fetch sublines for mapped line
  const { data: mappedSublines } = useQuery({
    queryKey: ["sublines-mapped", mappedLineValue],
    queryFn: async () => {
      if (!mappedLineValue) return [];
      const response = await apiFetch(`${config.apiUrl}/import/sublines/${mappedLineValue}`);
      if (!response.ok) {
        throw new Error("Failed to fetch sublines for mapped line");
      }
      return response.json();
    },
    enabled: !!mappedLineValue && mappedLineValue !== globalSelections.line,
    staleTime: 600000, // 10 minutes (increased from 60 seconds)
  });

  // Get field options for suppliers and companies
  const { data: fieldOptionsData } = useQuery({
    queryKey: ["field-options"],
    queryFn: async () => {
      const response = await apiFetch(`${config.apiUrl}/import/field-options`);
      if (!response.ok) {
        throw new Error("Failed to fetch field options");
      }
      return response.json();
    },
    staleTime: 600000, // 10 minutes (increased from 60 seconds)
  });
  
  // Safely access field options
  const fieldOptions = fieldOptionsData || { suppliers: [], companies: [] };

  // Create a stable identity for these queries to avoid re-renders
  const stableProductLines = useMemo(() => productLines || [], [productLines]);
  const stableSublines = useMemo(() => sublines || [], [sublines]);
  const stableMappedProductLines = useMemo(() => mappedProductLines || [], [mappedProductLines]);
  const stableMappedSublines = useMemo(() => mappedSublines || [], [mappedSublines]);

  const handleCategoryCreated = useCallback(
    async ({ type, parentId, id }: CreatedCategoryInfo) => {
      const refreshTasks: Array<Promise<unknown>> = [];

      if (type === "line") {
        refreshTasks.push(
          queryClient.invalidateQueries({ queryKey: ["product-lines", parentId] }),
          queryClient.invalidateQueries({ queryKey: ["product-lines-mapped", parentId] }),
        );
      } else {
        refreshTasks.push(
          queryClient.invalidateQueries({ queryKey: ["sublines", parentId] }),
          queryClient.invalidateQueries({ queryKey: ["sublines-mapped", parentId] }),
        );
      }

      if (refreshTasks.length) {
        try {
          await Promise.all(refreshTasks);
        } catch (error) {
          console.error("Failed to refresh category lists:", error);
        }
      }

      setGlobalSelections((prev) => {
        if (type === "line") {
          return {
            ...prev,
            company: parentId,
            line: id ?? prev.line,
            subline: undefined,
          };
        }

        return {
          ...prev,
          line: parentId,
          subline: id ?? prev.subline,
        };
      });
    },
    [queryClient, setGlobalSelections],
  );

  // Handle manual cache refresh
  const handleRefreshTaxonomy = useCallback(async () => {
    setIsRefreshing(true);
    try {
      // Clear backend cache
      const response = await apiFetch(`${config.apiUrl}/import/clear-taxonomy-cache`, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
      });

      if (!response.ok) {
        throw new Error('Failed to clear backend cache');
      }

      // Clear frontend React Query cache
      await Promise.all([
        queryClient.invalidateQueries({ queryKey: ['field-options'] }),
        queryClient.invalidateQueries({ queryKey: ['product-lines'] }),
        queryClient.invalidateQueries({ queryKey: ['product-lines-mapped'] }),
        queryClient.invalidateQueries({ queryKey: ['sublines'] }),
        queryClient.invalidateQueries({ queryKey: ['sublines-mapped'] }),
      ]);

      // Refetch field options immediately
      await queryClient.refetchQueries({ queryKey: ['field-options'] });

    } catch (error) {
      console.error('Error refreshing taxonomy:', error);
      toast.error('Failed to refresh taxonomy data', {
      });
    } finally {
      setIsRefreshing(false);
    }
  }, [queryClient]);

  // Check if a field is covered by global selections
  const isFieldCoveredByGlobalSelections = useCallback((key: string) => {
    return (key === 'supplier' && !!globalSelections.supplier) ||
           (key === 'company' && !!globalSelections.company) ||
           (key === 'line' && !!globalSelections.line) ||
           (key === 'subline' && !!globalSelections.subline);
  }, [
    globalSelections.supplier,
    globalSelections.company,
    globalSelections.line,
    globalSelections.subline
  ]);

  // Create a stable version of getFieldOptions with improved memoization
  const getFieldOptions = useCallback((fieldKey: string) => {
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    const field = fieldsArray.find(f => f.key === fieldKey);
    
    if (!field) {
      return [];
    }
    
    // Handle special hierarchical fields
    if (fieldKey === 'line') {
      // For line, return the appropriate product lines based on the company
      if (globalSelections.company) {
        return stableProductLines;
      } else if (mappedCompanyValue) {
        return stableMappedProductLines;
      }
    } else if (fieldKey === 'subline') {
      // For subline, return the appropriate sublines based on the line
      if (globalSelections.line) {
        return stableSublines;
      } else if (mappedLineValue) {
        return stableMappedSublines;
      }
    }
    
    // For other fields, check in both places - directly on the field or in fieldType.options
    let options = field.options || [];
    
    // If no options at the root level, check in fieldType
    if ((!options || options.length === 0) && field.fieldType && field.fieldType.options) {
      options = field.fieldType.options;
    }
    
    return options || [];
  }, [
    fields, 
    globalSelections.company, 
    globalSelections.line, 
    stableProductLines, 
    stableSublines, 
    mappedCompanyValue, 
    mappedLineValue, 
    stableMappedProductLines, 
    stableMappedSublines
  ]);

  // Check if a column has unmapped values (for select fields)
  const hasUnmappedValues = useCallback((column: Column<T>) => {
    if (!isExpandable(column) || !("matchedOptions" in column) || !("value" in column)) {
      return false;
    }
    
    const fieldOptions = getFieldOptions(column.value as string);
    
    // If there are options available but some values aren't mapped, consider it unmapped
    return fieldOptions.length > 0 && 
           column.matchedOptions.some(option => !option.value);
  }, [isExpandable, getFieldOptions]);
  
  // Optimize columnsWithUnmappedValuesMap calculation - prevent deep comparisons
  const columnsWithUnmappedValuesMap = useMemo(() => {
    const map = new Map<number, boolean>();
    // Prevent recalculation by checking if we already have this value mapped
    columns.forEach(col => {
      if (!map.has(col.index)) {
        map.set(col.index, hasUnmappedValues(col));
      }
    });
    return map;
  }, [columns, hasUnmappedValues]);

  // Get matched, unmapped, and columns with unmapped values
  const { matchedColumns, unmatchedColumns, columnsWithUnmappedValues } = useMemo(() => {
    // First identify columns with unmapped values (they need special treatment)
    const withUnmappedValues = columns.filter(col => 
      col.type !== ColumnType.empty && 
      col.type !== ColumnType.ignored && 
      col.type !== ColumnType.aiSupplemental && 
      columnsWithUnmappedValuesMap.get(col.index)
    );
    
    // These are columns that are mapped AND have all their values properly mapped
    const fullyMapped = columns.filter(col => 
      col.type !== ColumnType.empty && 
      col.type !== ColumnType.ignored && 
      col.type !== ColumnType.aiSupplemental && 
      !columnsWithUnmappedValuesMap.get(col.index)
    );
    
    // Unmapped columns
    const unmatched = columns.filter(col => col.type === ColumnType.empty);
    
    return { 
      matchedColumns: fullyMapped, 
      unmatchedColumns: unmatched,
      columnsWithUnmappedValues: withUnmappedValues
    };
  }, [columns, columnsWithUnmappedValuesMap]);

  // Get ignored columns
  const ignoredColumns = useMemo(() => {
    return columns.filter(col => col.type === ColumnType.ignored);
  }, [columns]);

  const aiSupplementalColumns = useMemo(() => {
    return columns.filter(col => col.type === ColumnType.aiSupplemental);
  }, [columns]);

  // Get mapping information for required fields
  const requiredFieldMappings = useMemo(() => {
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    const mappings = new Map<string, { isGlobal: boolean, columnHeader?: string, columnIndex?: number }>();
    
    // Check global selections
    fieldsArray.forEach(field => {
      const key = field.key as string;
      if (isFieldCoveredByGlobalSelections(key)) {
        mappings.set(key, { isGlobal: true });
      }
    });
    
    // Check column mappings
    columns.forEach(column => {
      if ("value" in column) {
        const key = column.value as string;
        if (!mappings.has(key) || !mappings.get(key)?.isGlobal) {
          mappings.set(key, { 
            isGlobal: false, 
            columnHeader: column.header,
            columnIndex: column.index
          });
        }
      }
    });
    
    return mappings;
  }, [columns, fields, isFieldCoveredByGlobalSelections]);

  // Available fields for mapping (including already mapped fields)
  const availableFields = useMemo(() => {
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    // Don't filter out mapped fields, only filter global selections
    return fieldsArray.filter(field => 
      !isFieldCoveredByGlobalSelections(field.key)
    );
  }, [fields, isFieldCoveredByGlobalSelections]);

  // All available fields including already mapped ones (for editing mapped columns)
  const allFields = useMemo(() => {
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    return fieldsArray.filter(field => !isFieldCoveredByGlobalSelections(field.key));
  }, [fields, isFieldCoveredByGlobalSelections]);

  // For filtering fields by type
  const getFieldsByKeyPrefix = useCallback((prefix: string, fieldsToUse = availableFields) => {
    return fieldsToUse.filter(field => 
      typeof field.key === 'string' && field.key.startsWith(prefix)
    );
  }, [availableFields]);

  // Group all fields by category (for editing mapped columns)

  // Group available fields by category (for unmapped columns)
  const availableFieldCategories = useMemo(() => {
    return [
      { name: "Basic Info", fields: getFieldsByKeyPrefix('basic') },
      { name: "Product", fields: getFieldsByKeyPrefix('product') },
      { name: "Inventory", fields: getFieldsByKeyPrefix('inventory') },
      { name: "Pricing", fields: getFieldsByKeyPrefix('pricing') },
      { name: "Other", fields: availableFields.filter(f => 
        !f.key.startsWith('basic') && 
        !f.key.startsWith('product') && 
        !f.key.startsWith('inventory') && 
        !f.key.startsWith('pricing')
      ) }
    ].filter(category => category.fields.length > 0);
  }, [availableFields, getFieldsByKeyPrefix]);

  // Override onChange to handle the new interface
  const onChange = useCallback(
    (value: T, columnIndex: number) => {
      const field = (fields as Fields<T>).find((f) => f.key === value)
      if (!field) return

      const existingFieldIndex = columns.findIndex((column) => "value" in column && column.value === field.key)
      
      // Clear global selection if mapping to a global field
      if (field.key === 'supplier' || field.key === 'company' || field.key === 'line' || field.key === 'subline') {
        setGlobalSelections(prev => {
          const newSelections = { ...prev };
          if (field.key === 'supplier') delete newSelections.supplier;
          if (field.key === 'company') {
            delete newSelections.company;
            delete newSelections.line;
            delete newSelections.subline;
          }
          if (field.key === 'line') {
            delete newSelections.line;
            delete newSelections.subline;
          }
          if (field.key === 'subline') delete newSelections.subline;
          return newSelections;
        });
      }
      
      setColumns(
        columns.map<Column<T>>((column, index) => {
          if (column.index === columnIndex) {
            // Set the new column value
            const updatedColumn = setColumn(column, field as Field<T>, data, autoMapSelectValues);
            
            // Auto-map values if this is a field with options
            if (isExpandable(updatedColumn) && "matchedOptions" in updatedColumn) {
              // Get available options for this field
              const fieldOptions = getFieldOptions(field.key as string);
              
              if (fieldOptions && fieldOptions.length > 0) {
                // Try to auto-map each value
                updatedColumn.matchedOptions = updatedColumn.matchedOptions.map(option => {
                  // If already mapped, keep it
                  if (option.value) return option;
                  
                  const entryValue = String(option.entry || '').trim();
                  
                  // Try to find a case-insensitive match
                  const matchingOption = fieldOptions.find((opt: { label: string, value: string }) => 
                    String(opt.label).toLowerCase() === entryValue.toLowerCase() || 
                    String(opt.value).toLowerCase() === entryValue.toLowerCase()
                  );
                  
                  if (matchingOption) {
                    return {
                      ...option,
                      value: matchingOption.value
                    };
                  }
                  
                  return option;
                });
              }
            }
            
            return updatedColumn;
          } else if (index === existingFieldIndex) {
            // Clear the old column that had this field
            toast.warning(translations.matchColumnsStep.duplicateColumnWarningTitle, {
              description: translations.matchColumnsStep.duplicateColumnWarningDescription,
            })
            return setColumn(column)
          } else {
            // Leave other columns unchanged
            return column
          }
        }),
      )
    },
    [
      autoMapSelectValues,
      columns,
      data,
      fields,
      getFieldOptions,
      isExpandable,
      translations.matchColumnsStep.duplicateColumnWarningDescription,
      translations.matchColumnsStep.duplicateColumnWarningTitle,
    ],
  )

  // Auto-map values for all columns
  const autoMapAllValues = useCallback(() => {
    setColumns(
      columns.map<Column<T>>((column) => {
        // Only process expandable columns with unmatched options
        if (isExpandable(column) && "matchedOptions" in column && "value" in column) {
          const fieldKey = column.value as string;
          const fieldOptions = getFieldOptions(fieldKey);
          
          if (fieldOptions && fieldOptions.length > 0) {
            // Create a copy of the column
            const updatedColumn = { ...column };
            
            // Try to auto-map each value
            updatedColumn.matchedOptions = updatedColumn.matchedOptions.map(option => {
              // If already mapped, keep it
              if (option.value) return option;
              
              const entryValue = String(option.entry || '').trim();
              
              // Try to find a case-insensitive match
              const matchingOption = fieldOptions.find((opt: { label: string, value: string }) => 
                String(opt.label).toLowerCase() === entryValue.toLowerCase() || 
                String(opt.value).toLowerCase() === entryValue.toLowerCase()
              );
              
              if (matchingOption) {
                return {
                  ...option,
                  value: matchingOption.value
                };
              }
              
              return option;
            });
            
            return updatedColumn;
          }
        }
        
        return column;
      }),
    )
  }, [columns, getFieldOptions, isExpandable]);

  // Run auto-mapping only when needed - when columns change or field options change
  useEffect(() => {
    // Check if we have columns that need mapping
    const needsMapping = columns.some(column => 
      isExpandable(column) && 
      "matchedOptions" in column && 
      column.matchedOptions.some(option => !option.value)
    );
    
    if (needsMapping) {
      autoMapAllValues();
    }
  }, [autoMapAllValues, columns, fields, productLines, sublines]);

  const onIgnore = useCallback(
    (columnIndex: number) => {
      setColumns(columns.map((column) => (column.index === columnIndex ? setIgnoreColumn<T>(column) : column)))
    },
    [columns, setColumns],
  )

  const onToggleAiSupplemental = useCallback(
    (columnIndex: number) => {
      setColumns(columns.map((column) => {
        if (column.index !== columnIndex) return column;

        if (column.type === ColumnType.aiSupplemental) {
          return { type: ColumnType.empty, index: column.index, header: column.header } as Column<T>;
        }

        return { type: ColumnType.aiSupplemental, index: column.index, header: column.header } as Column<T>;
      }));

      setExpandedValues(prev => prev.filter(idx => idx !== columnIndex));
      setUserCollapsedColumns(prev => prev.filter(idx => idx !== columnIndex));
    },
    [columns, setColumns, setExpandedValues, setUserCollapsedColumns],
  )

  const onRevertIgnore = useCallback(
    (columnIndex: number) => {
      setColumns(columns.map((column) => (column.index === columnIndex ? setColumn(column) : column)))
    },
    [columns, setColumns],
  )

  const onSubChange = useCallback(
    (value: string, columnIndex: number, entry: string) => {
      setColumns(
        columns.map((column) =>
          column.index === columnIndex && "matchedOptions" in column
            ? setSubColumn(column as MatchedSelectColumn<T> | MatchedSelectOptionsColumn<T> | MatchedMultiSelectColumn<T>, entry, value)
            : column,
        ),
      )
    },
    [columns, setColumns],
  )

  // Get all required fields
  const requiredFields = useMemo(() => {
    // Convert the fields to the expected type
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    
    // Check for required fields based on validations
    const required = fieldsArray.filter(field => {
      // Check if the field has validations
      if (!field.validations || !Array.isArray(field.validations)) {
        return false;
      }
      
      // Check for any validation with rule: 'required' or required: true
      const isRequired = field.validations.some(
        (v: any) => v.rule === 'required' || v.required === true
      );
      
      
      return isRequired;
    });
    
    return required;
  }, [fields]);

  // Get unmatched required fields
  const unmatchedRequiredFields = useMemo(() => {
    // Convert the fields to the expected type
    const fieldsArray = Array.isArray(fields) ? fields : [fields]
    const typedFields = fieldsArray.map(field => ({
      ...field,
      key: field.key as TsDeepReadonly<T>
    })) as unknown as Fields<T>
    
    const unmatched = findUnmatchedRequiredFields(typedFields, columns);
    return unmatched;
  }, [fields, columns])

  // Get matched required fields
  const matchedRequiredFields = useMemo(() => {
    const matched = requiredFields
      .map(field => field.key)
      .filter(key => {
        // Type assertion to handle the DeepReadonly<T> vs string type mismatch
        return !unmatchedRequiredFields.includes(key as any);
      });
    return matched;
  }, [requiredFields, unmatchedRequiredFields]);

  // Get field label by key
  const getFieldLabel = useCallback((key: string) => {
    const fieldsArray = Array.isArray(fields) ? fields : [fields];
    const field = fieldsArray.find(f => f.key === key);
    return field?.label || key;
  }, [fields]);

  // Fix handleOnContinue - it should be useCallback, not useEffect
  const handleOnContinue = useCallback(async (useNewValidation: boolean = false) => {
    setIsLoading(true)

    try {
      // Normalize the data with global selections before continuing
      const normalizedData = normalizeTableData(columns, data, fields)
      const hasAiSupplementalColumns = aiSupplementalColumns.length > 0

      const enhancedData = hasAiSupplementalColumns
        ? normalizedData.map((row, rowIndex) => {
            const supplementalEntries = aiSupplementalColumns.reduce<Record<string, string>>((acc, column) => {
              const rawValue = data[rowIndex]?.[column.index]
              if (rawValue === undefined || rawValue === null) {
                return acc
              }

              const trimmedValue = String(rawValue).trim()
              if (!trimmedValue) {
                return acc
              }

              const headerLabel = column.header?.trim() || `Column ${column.index + 1}`
              acc[headerLabel] = trimmedValue
              return acc
            }, {})

            if (Object.keys(supplementalEntries).length === 0) {
              return row
            }

            return {
              ...row,
              __aiSupplemental: supplementalEntries,
            } as typeof row & { __aiSupplemental: Record<string, string> }
          })
        : normalizedData

      await onContinue(enhancedData, data, columns, globalSelections, useNewValidation)
    } finally {
      setIsLoading(false)
    }
  }, [onContinue, columns, data, fields, globalSelections, aiSupplementalColumns])

  useEffect(
    () => {
      if (autoMapHeaders && !hasAutoMappedRef.current) {
        setColumns(getMatchedColumns(columns, fields, data, autoMapDistance, autoMapSelectValues))
        hasAutoMappedRef.current = true
      }
    },
    // eslint-disable-next-line react-hooks/exhaustive-deps
    [],
  )

  // Missing dataExample - let's create a data sample from the actual data
  const dataSample = useMemo(() => {
    // Take first 5 rows as sample data
    return data.slice(0, 5);
  }, [data]);

  // Optimize expensive getColumnSamples calls with memoization and ref
  const columnSamplesCache = useRef(new Map<number, any[]>());

  // Enhanced version with caching and a more stable identity
  const getColumnSamples = useCallback((columnIndex: number) => {
    if (!columnSamplesCache.current.has(columnIndex)) {
      // Only build samples from data when needed
      const samples = dataSample.slice(0, 5).map(row => row[columnIndex] || '');
      columnSamplesCache.current.set(columnIndex, samples);
    }
    return columnSamplesCache.current.get(columnIndex) || [];
  }, [dataSample]);  // Only depends on dataSample, not on data directly

  // Clear the cache when the data changes significantly
  useEffect(() => {
    columnSamplesCache.current.clear();
  }, [data.length]);  // Only clear when data length changes, not on every data reference change

  // Optimize the renderSamplePreview function
  const renderSamplePreview = useCallback((columnIndex: number) => {
    const samples = getColumnSamples(columnIndex);
    return <ColumnSamplePreview samples={samples} />;
  }, [getColumnSamples]);

  // Automatically expand columns with unmapped values - use layoutEffect to avoid flashing
  useLayoutEffect(() => {
    // Track the current unmapped column indexes
    const currentUnmappedIndexes = columnsWithUnmappedValues.map(col => col.index);
    
    // Find columns that are newly unmapped (weren't in expandedValues or userCollapsedColumns)
    const newlyUnmappedColumns = currentUnmappedIndexes.filter(index => 
      !expandedValues.includes(index) && !userCollapsedColumns.includes(index)
    );
    
    if (newlyUnmappedColumns.length > 0) {
      setExpandedValues(prev => [...prev, ...newlyUnmappedColumns]);
    }
  }, [columnsWithUnmappedValues, expandedValues, userCollapsedColumns]);

  // Create a stable mapping of column index to change handlers
  const columnChangeHandlers = useMemo(() => {
    const handlers = new Map<number, (value: string) => void>();
    
    columns.forEach(column => {
      handlers.set(column.index, (value: string) => {
        onChange(value as T, column.index);
      });
    });
    
    return handlers;
  }, [columns, onChange]);

  // Add a function to check if a field is already mapped to another column
  const isFieldMappedToOtherColumn = useCallback((fieldKey: string, currentColumnIndex: number) => {
    const matchedColumnForField = columns.find(col => 
      col.type !== ColumnType.empty &&
      col.type !== ColumnType.ignored &&
      col.type !== ColumnType.aiSupplemental &&
      "value" in col && 
      col.value === fieldKey &&
      col.index !== currentColumnIndex
    );
    return matchedColumnForField ? { isMapped: true, columnHeader: matchedColumnForField.header } : { isMapped: false };
  }, [columns]);

  // Add a wheel handler function for command lists
  const handleCommandListWheel = useCallback((e: React.WheelEvent) => {
    e.currentTarget.scrollTop += e.deltaY;
    e.stopPropagation();
  }, []);

  // Replace the renderFieldSelector function with a more stable version
  const renderFieldSelector = useCallback((column: Column<T>, isUnmapped: boolean = false) => {
    // For ignored columns, show a badge
    if (column.type === ColumnType.ignored) {
      return <Badge variant="outline">Ignored</Badge>;
    }

    if (column.type === ColumnType.aiSupplemental) {
      return (
        <Badge variant="outline" className="bg-sky-100 text-sky-800 border-sky-200">
          AI supplemental
        </Badge>
      );
    }
    
    // Get the pre-created onChange handler for this column
    const handleChange = columnChangeHandlers.get(column.index);
    
    const isCostEach = "value" in column && column.value === "cost_each";

    return (
      <div className="flex items-center gap-3">
        <FieldSelector
          column={column}
          isUnmapped={isUnmapped}
          fieldCategories={availableFieldCategories}
          allFields={allFields}
          onChange={(value: string) => {
            if (handleChange) handleChange(value);
          }}
          isFieldMappedToOtherColumn={isFieldMappedToOtherColumn}
          handleCommandListWheel={handleCommandListWheel}
        />
        {isCostEach && (
          <TooltipProvider>
            <Tooltip>
              <TooltipTrigger asChild>
                <label className="flex items-center gap-1.5 text-xs text-muted-foreground whitespace-nowrap cursor-pointer">
                  <input
                    type="checkbox"
                    checked={!!globalSelections.costIsTotalCost}
                    onChange={(e) => setGlobalSelections(prev => ({ ...prev, costIsTotalCost: e.target.checked }))}
                    className="h-3.5 w-3.5 rounded border-gray-300"
                  />
                  Divide by min qty
                </label>
              </TooltipTrigger>
              <TooltipContent className="max-w-[250px]">
                <p>Enable if the spreadsheet lists total cost per pack rather than cost per individual item. The value will be divided by Min Qty to calculate the actual cost each.</p>
              </TooltipContent>
            </Tooltip>
          </TooltipProvider>
        )}
      </div>
    );
  }, [availableFieldCategories, allFields, columnChangeHandlers, isFieldMappedToOtherColumn, handleCommandListWheel, globalSelections.costIsTotalCost, setGlobalSelections]);

  // Replace the renderValueMappings function with a memoized version
  const renderValueMappings = useCallback((column: Column<T>) => {
    if (!isExpandable(column) || !("matchedOptions" in column) || !("value" in column)) {
      return null;
    }
    
    const fieldOptions = getFieldOptions(column.value as string);
    return <ValueMappings column={column} fieldOptions={fieldOptions} onSubChange={onSubChange} />;
  }, [isExpandable, getFieldOptions, onSubChange]);

  // Remove virtualization references and helpers
  const renderTable = useMemo(() => {
    return (
      <ScrollArea className="h-full">
        <Table>
          <TableHeader className="sticky top-0 bg-muted z-10">
            <TableRow>
              <TableHead className="w-1/4">Imported Column</TableHead>
              <TableHead className="w-15 text-center">Sample Data</TableHead>
              <TableHead className="w-12"></TableHead>
              <TableHead>Map To Field</TableHead>
              <TableHead className="w-32 text-right">
                <div className="flex items-center justify-end gap-6">
                  <span>AI only</span>
                  <span>Ignore</span>
                </div>
              </TableHead>
            </TableRow>
          </TableHeader>
          <TableBody>
            {/* Always show columns with unmapped values */}
            {columnsWithUnmappedValues.map((column) => {
              const isExpanded = expandedValues.includes(column.index);
              
              return (
                <React.Fragment key={`unmapped-values-${column.index}`}>
                  <TableRow className="bg-amber-50 dark:bg-amber-950/20">
                    <TableCell className="font-medium">{column.header}</TableCell>
                    <TableCell className="text-center">
                      {renderSamplePreview(column.index)}
                    </TableCell>
                    <TableCell>
                      <ArrowRightIcon className="h-4 w-4 mx-auto text-amber-600" />
                    </TableCell>
                    <TableCell>
                      {renderFieldSelector(column)}
                    </TableCell>
                    <TableCell className="flex justify-end items-center gap-1">
                      <ColumnActions
                        column={column}
                        onIgnore={onIgnore}
                        onToggleAiSupplemental={onToggleAiSupplemental}
                        toggleValueMapping={toggleValueMappingOptimized}
                        isExpanded={isExpanded}
                        canExpandValues={isExpandable(column)}
                        isAiSupplemental={column.type === ColumnType.aiSupplemental}
                      />
                    </TableCell>
                  </TableRow>
                  
                  {/* Value mappings row */}
                  {isExpanded && (
                    <TableRow>
                      <TableCell colSpan={5} className="p-0">
                        {renderValueMappings(column)}
                      </TableCell>
                    </TableRow>
                  )}
                </React.Fragment>
              );
            })}
            
            {/* Always show unmapped columns */}
            {unmatchedColumns.map((column) => (
              <TableRow key={`unmatched-${column.index}`} className="bg-amber-50 hover:bg-amber-100">
                <TableCell className="font-medium">{column.header}</TableCell>
                <TableCell className="text-center">
                  {renderSamplePreview(column.index)}
                </TableCell>
                <TableCell>
                  <ArrowRightIcon className="h-4 w-4 mx-auto text-muted-foreground" />
                </TableCell>
                <TableCell>
                  {renderFieldSelector(column, true)}
                </TableCell>
                <TableCell className="flex justify-end items-center gap-1">
                  <ColumnActions
                    column={column}
                    onIgnore={onIgnore}
                    onToggleAiSupplemental={onToggleAiSupplemental}
                    toggleValueMapping={toggleValueMappingOptimized}
                    isExpanded={expandedValues.includes(column.index)}
                    canExpandValues={isExpandable(column)}
                    isAiSupplemental={false}
                  />
                </TableCell>
              </TableRow>
            ))}
            
            {/* Show matched columns if showAllColumns is true */}
            {showAllColumns && matchedColumns.map((column) => {
              const isExpanded = expandedValues.includes(column.index);
              const canExpandValues = isExpandable(column);
              
              return (
                <React.Fragment key={`matched-${column.index}`}>
                  <TableRow className="group bg-green-50 hover:bg-green-100">
                    <TableCell className="font-medium">{column.header}</TableCell>
                    <TableCell className="text-center">
                      {renderSamplePreview(column.index)}
                    </TableCell>
                    <TableCell>
                      <ArrowRightIcon className="h-4 w-4 mx-auto text-muted-foreground" />
                    </TableCell>
                    <TableCell>
                      {renderFieldSelector(column)}
                    </TableCell>
                    <TableCell className="flex justify-end items-center gap-1">
                      <ColumnActions
                        column={column}
                        onIgnore={onIgnore}
                        onToggleAiSupplemental={onToggleAiSupplemental}
                        toggleValueMapping={toggleValueMappingOptimized}
                        isExpanded={isExpanded}
                        canExpandValues={canExpandValues}
                        isAiSupplemental={column.type === ColumnType.aiSupplemental}
                      />
                    </TableCell>
                  </TableRow>
                  
                  {/* Value mappings row */}
                  {isExpanded && canExpandValues && (
                    <TableRow>
                      <TableCell colSpan={5} className="p-0">
                        {renderValueMappings(column)}
                      </TableCell>
                    </TableRow>
                  )}
                </React.Fragment>
              );
            })}

            {/* Show AI supplemental columns if showAllColumns is true */}
            {showAllColumns && aiSupplementalColumns.map((column) => (
              <TableRow key={`ai-${column.index}`} className="bg-sky-50 hover:bg-sky-100">
                <TableCell className="font-medium">{column.header}</TableCell>
                <TableCell className="text-center">
                  {renderSamplePreview(column.index)}
                </TableCell>
                <TableCell>
                  <ArrowRightIcon className="h-4 w-4 mx-auto text-sky-600" />
                </TableCell>
                <TableCell>
                  <Badge variant="outline" className="bg-sky-100 text-sky-800 border-sky-200">
                    AI supplemental
                  </Badge>
                </TableCell>
                <TableCell className="flex justify-end items-center gap-1">
                  <ColumnActions
                    column={column}
                    onIgnore={onIgnore}
                    onToggleAiSupplemental={onToggleAiSupplemental}
                    toggleValueMapping={toggleValueMappingOptimized}
                    isExpanded={false}
                    canExpandValues={false}
                    isAiSupplemental
                  />
                </TableCell>
              </TableRow>
            ))}
            
            {/* Show ignored columns if showAllColumns is true */}
            {showAllColumns && ignoredColumns.map((column) => (
              <TableRow key={`ignored-${column.index}`} className="text-muted-foreground bg-red-50 hover:bg-red-100">
                <TableCell className="font-medium">{column.header}</TableCell>
                <TableCell className="text-center">
                  {renderSamplePreview(column.index)}
                </TableCell>
                <TableCell>
                  <ArrowRightIcon className="h-4 w-4 mx-auto" />
                </TableCell>
                <TableCell>
                  <Badge variant="outline" className="bg-card">Ignored</Badge>
                </TableCell>
                <TableCell className="text-right">
                  <Button 
                    variant="ghost" 
                    size="sm"
                    onClick={() => onRevertIgnore(column.index)}
                    className="h-7 px-2"
                    title="Unmatch this column"
                  >
                    <CheckCircle2 className="h-4 w-4" />
                  </Button>
                </TableCell>
              </TableRow>
            ))}
            
            {/* Show a message if all columns are mapped/ignored */}
            {unmatchedColumns.length === 0 && columnsWithUnmappedValues.length === 0 && !showAllColumns && (
              <TableRow>
                <TableCell colSpan={5} className="text-center py-8 text-muted-foreground">
                  All columns have been mapped or ignored. 
                  <Button 
                    variant="link" 
                    onClick={() => setShowAllColumns(true)} 
                    className="px-1 h-auto"
                  >
                    Show all columns
                  </Button>
                </TableCell>
              </TableRow>
            )}
          </TableBody>
        </Table>
      </ScrollArea>
    );
  }, [
    columnsWithUnmappedValues, 
    unmatchedColumns,
    matchedColumns,
    ignoredColumns,
    aiSupplementalColumns,
    showAllColumns,
    expandedValues,
    renderSamplePreview,
    renderFieldSelector,
    renderValueMappings,
    onIgnore,
    onToggleAiSupplemental,
    onRevertIgnore,
    toggleValueMappingOptimized,
    isExpandable
  ]);

  return (
    <div className="flex h-[calc(100vh-10rem)] flex-col">
      <div className="flex-1 overflow-hidden">
        <div className="h-full flex flex-col">
          <div className="px-6 py-4 flex-1 overflow-auto">
            <div className="grid grid-cols-1 lg:grid-cols-4 gap-6 h-full">
              {/* Left panel - Global selections & Required fields */}
              <div className="lg:col-span-1 space-y-4">
                <div className="bg-muted border rounded-md p-4 shadow-md">
                  <h3 className="text-lg font-medium mb-0">Global Settings</h3>
                  <p className="text-sm text-muted-foreground mb-2">
                    These values will be applied to all imported items
                  </p>
                  
                  <div className="space-y-2">
                    <div className="space-y-1">
                      <label className="text-sm font-medium">Supplier</label>
                      <TooltipProvider>
                        <Tooltip>
                          <TooltipTrigger asChild>
                            <div>
                              <SupplierSelector 
                                value={globalSelections.supplier}
                                onChange={(value) => setGlobalSelections(prev => ({ ...prev, supplier: value }))}
                                suppliers={fieldOptions?.suppliers || []}
                                disabled={isSupplierMappedToColumn}
                              />
                            </div>
                          </TooltipTrigger>
                          {isSupplierMappedToColumn && (
                            <TooltipContent>
                              <p>A spreadsheet column is already mapped to Supplier</p>
                            </TooltipContent>
                          )}
                        </Tooltip>
                      </TooltipProvider>
                    </div>

                    <div className="space-y-1">
                      <label className="text-sm font-medium">Company</label>
                      <TooltipProvider>
                        <Tooltip>
                          <TooltipTrigger asChild>
                            <div>
                              <CompanySelector 
                                value={globalSelections.company}
                                onChange={(value) => setGlobalSelections(prev => ({ 
                                  ...prev, 
                                  company: value,
                                  line: undefined,
                                  subline: undefined
                                }))}
                                companies={fieldOptions?.companies || []}
                                disabled={isCompanyMappedToColumn}
                              />
                            </div>
                          </TooltipTrigger>
                          {isCompanyMappedToColumn && (
                            <TooltipContent>
                              <p>A spreadsheet column is already mapped to Company</p>
                            </TooltipContent>
                          )}
                        </Tooltip>
                      </TooltipProvider>
                    </div>

                    <div className="space-y-1">
                      <label className="text-sm font-medium">Line</label>
                      <TooltipProvider>
                        <Tooltip>
                          <TooltipTrigger asChild>
                            <div>
                              <LineSelector 
                                value={globalSelections.line}
                                onChange={(value) => setGlobalSelections(prev => ({ 
                                  ...prev, 
                                  line: value,
                                  subline: undefined
                                }))}
                                lines={productLines || []}
                                disabled={!globalSelections.company || isLineMappedToColumn}
                              />
                            </div>
                          </TooltipTrigger>
                          {!globalSelections.company ? (
                            <TooltipContent>
                              <p>Select a company first</p>
                            </TooltipContent>
                          ) : isLineMappedToColumn ? (
                            <TooltipContent>
                              <p>A spreadsheet column is already mapped to Line</p>
                            </TooltipContent>
                          ) : null}
                        </Tooltip>
                      </TooltipProvider>
                    </div>

                    <div className="space-y-1">
                      <label className="text-sm font-medium">Sub Line</label>
                      <TooltipProvider>
                        <Tooltip>
                          <TooltipTrigger asChild>
                            <div>
                              <SubLineSelector 
                                value={globalSelections.subline}
                                onChange={(value) => setGlobalSelections(prev => ({ 
                                  ...prev, 
                                  subline: value 
                                }))}
                                sublines={sublines || []}
                                disabled={!globalSelections.line || isSublineMappedToColumn}
                              />
                            </div>
                          </TooltipTrigger>
                          {!globalSelections.line ? (
                            <TooltipContent>
                              <p>Select a line first</p>
                            </TooltipContent>
                          ) : isSublineMappedToColumn ? (
                            <TooltipContent>
                              <p>A spreadsheet column is already mapped to Sub Line</p>
                            </TooltipContent>
                          ) : null}
                        </Tooltip>
                      </TooltipProvider>
                    </div>
                  </div>

                  <div className="pt-4 flex items-center gap-2">
                    <CreateProductCategoryDialog
                      trigger={
                        <Button variant="outline" size="sm" className="w-full">
                          <Plus className="h-3 w-3" /> New line or subline
                        </Button>
                      }
                      companies={fieldOptions?.companies || []}
                      defaultCompanyId={globalSelections.company}
                      defaultLineId={globalSelections.line}
                      onCreated={handleCategoryCreated}
                    />

                    <TooltipProvider>
                      <Tooltip>
                        <TooltipTrigger asChild>
                          <Button
                            variant="outline"
                            size="sm"
                            className="w-full"
                            onClick={handleRefreshTaxonomy}
                            disabled={isRefreshing}
                          >
                            <RefreshCw className={`h-3 w-3 ${isRefreshing ? 'animate-spin' : ''}`} />
                            Refresh data
                          </Button>
                        </TooltipTrigger>
                        <TooltipContent className="max-w-[250px]">
                          <p>Reload all suppliers, companies, lines, categories, and other taxonomy data from the database</p>
                        </TooltipContent>
                      </Tooltip>
                    </TooltipProvider>
                  </div>
                </div>
                
                {/* Required Fields Section - Updated to show source column */}
                <div>
                  <div className="flex items-center gap-2 mb-4">
                    <h3 className="text-lg font-medium">Matched Fields</h3>
                  </div>
                  
                  <div className="space-y-2">
                    {requiredFields.length > 0 ? (
                      requiredFields.map(field => {
                        const isMatched = matchedRequiredFields.includes(field.key);
                        const fieldMapping = requiredFieldMappings.get(field.key);
                        const isCoveredByGlobal = fieldMapping?.isGlobal;
                        const isAccountedFor = isMatched || isCoveredByGlobal;
                        
                        return (
                          <div key={field.key} className="flex items-center gap-2">
                            {isAccountedFor ? (
                              <CheckCircle2 className="h-4 w-4 text-green-500 flex-shrink-0" />
                            ) : (
                              <AlertCircle className="h-4 w-4 text-amber-500 flex-shrink-0" />
                            )}
                            <span className={`text-sm ${isAccountedFor ? "text-green-700 dark:text-green-400" : "text-amber-700 dark:text-amber-400"}`}>
                              {getFieldLabel(field.key)}
                            </span>
                            <span className="text-xs text-muted-foreground ml-auto">
                              {isCoveredByGlobal ? 
                                "(global)" : 
                                fieldMapping?.columnHeader ?
                                  `(from "${fieldMapping.columnHeader}")` :
                                  ""
                              }
                            </span>
                          </div>
                        );
                      })
                    ) : (
                      <div className="text-sm text-muted-foreground">
                        No required fields found.
                      </div>
                    )}
                  </div>
                </div>
              </div>
              
              {/* Right panel - Column mapping interface */}
              <div className="lg:col-span-3 h-full">
                <div className="flex items-center justify-between mb-4">
                  <div className="flex items-center gap-2">
                    <h3 className="text-lg font-medium">Map Spreadsheet Columns</h3>
                    {columnsWithUnmappedValues.length > 0 && (
                      <Badge variant="outline" className="bg-amber-100 text-amber-800 hover:bg-amber-100 border-amber-300">
                        {columnsWithUnmappedValues.length} with unmapped values
                      </Badge>
                    )}
                  </div>
                  
                  <div className="flex items-center gap-3">
                    <Button 
                      variant="outline" 
                      size="sm"
                      onClick={() => setShowAllColumns(!showAllColumns)}
                    >
                      {!showAllColumns ? <EyeIcon className="h-3.5 w-3.5 mr-1" /> : <EyeOffIcon className="h-3.5 w-3.5 mr-1" />}
                      {!showAllColumns ? "Show all" : "Hide mapped"}
                    </Button>
                  </div>
                </div>
                
                <div className="bg-background border rounded-md h-[calc(100%-3rem)] overflow-hidden">
                  {renderTable}
                </div>
              </div>
            </div>
          </div>
        </div>
      </div>
      
      <div className="border-t bg-muted px-6 py-4">
        <div className="flex items-center justify-between">

            {onBack && (
              <Button variant="outline" onClick={onBack}>
                {translations.matchColumnsStep.backButtonTitle}
              </Button>
            )}

            <div className="flex items-center gap-2 ml-auto">
              <Button
                disabled={isLoading}
                onClick={() => handleOnContinue(true)}
              >
                {translations.matchColumnsStep.nextButtonTitle}
              </Button>
            </div>
        </div>
      </div>
    </div>
  )
}

export const MatchColumnsStep = React.memo(MatchColumnsStepComponent) as typeof MatchColumnsStepComponent
