diff --git a/inventory-server/src/routes/import.js b/inventory-server/src/routes/import.js index a65a9a8..22cb6a6 100644 --- a/inventory-server/src/routes/import.js +++ b/inventory-server/src/routes/import.js @@ -13,6 +13,26 @@ fs.mkdirSync(uploadsDir, { recursive: true }); // Create a Map to track image upload times and their scheduled deletion const imageUploadMap = new Map(); +// Connection pooling and cache configuration +const connectionCache = { + ssh: null, + dbConnection: null, + lastUsed: 0, + isConnecting: false, + connectionPromise: null, + // Cache expiration time in milliseconds (5 minutes) + expirationTime: 5 * 60 * 1000, + // Cache for query results (key: query string, value: {data, timestamp}) + queryCache: new Map(), + // Cache duration for different query types in milliseconds + cacheDuration: { + 'field-options': 30 * 60 * 1000, // 30 minutes for field options + 'product-lines': 10 * 60 * 1000, // 10 minutes for product lines + 'sublines': 10 * 60 * 1000, // 10 minutes for sublines + 'default': 60 * 1000 // 1 minute default + } +}; + // Function to schedule image deletion after 24 hours const scheduleImageDeletion = (filename, filePath) => { // Delete any existing timeout for this file @@ -165,7 +185,114 @@ const upload = multer({ } }); -// Helper function to setup SSH tunnel +// Modified function to get a database connection with connection pooling +async function getDbConnection() { + const now = Date.now(); + + // Check if we need to refresh the connection due to inactivity + const needsRefresh = !connectionCache.ssh || + !connectionCache.dbConnection || + (now - connectionCache.lastUsed > connectionCache.expirationTime); + + // If connection is still valid, update last used time and return existing connection + if (!needsRefresh) { + connectionCache.lastUsed = now; + return { + ssh: connectionCache.ssh, + connection: connectionCache.dbConnection + }; + } + + // If another request is already establishing a connection, wait for that promise + if (connectionCache.isConnecting && connectionCache.connectionPromise) { + try { + await connectionCache.connectionPromise; + return { + ssh: connectionCache.ssh, + connection: connectionCache.dbConnection + }; + } catch (error) { + // If that connection attempt failed, we'll try again below + console.error('Error waiting for existing connection:', error); + } + } + + // Close existing connections if they exist + if (connectionCache.dbConnection) { + try { + await connectionCache.dbConnection.end(); + } catch (error) { + console.error('Error closing existing database connection:', error); + } + } + + if (connectionCache.ssh) { + try { + connectionCache.ssh.end(); + } catch (error) { + console.error('Error closing existing SSH connection:', error); + } + } + + // Mark that we're establishing a new connection + connectionCache.isConnecting = true; + + // Create a new promise for this connection attempt + connectionCache.connectionPromise = setupSshTunnel().then(tunnel => { + const { ssh, stream, dbConfig } = tunnel; + + return mysql.createConnection({ + ...dbConfig, + stream + }).then(connection => { + // Store the new connections + connectionCache.ssh = ssh; + connectionCache.dbConnection = connection; + connectionCache.lastUsed = Date.now(); + connectionCache.isConnecting = false; + + return { + ssh, + connection + }; + }); + }).catch(error => { + connectionCache.isConnecting = false; + throw error; + }); + + // Wait for the connection to be established + return connectionCache.connectionPromise; +} + +// Helper function to get cached query results or execute query if not cached +async function getCachedQuery(cacheKey, queryType, queryFn) { + // Get cache duration based on query type + const cacheDuration = connectionCache.cacheDuration[queryType] || connectionCache.cacheDuration.default; + + // Check if we have a valid cached result + const cachedResult = connectionCache.queryCache.get(cacheKey); + const now = Date.now(); + + if (cachedResult && (now - cachedResult.timestamp < cacheDuration)) { + console.log(`Cache hit for ${queryType} query: ${cacheKey}`); + return cachedResult.data; + } + + // No valid cache found, execute the query + console.log(`Cache miss for ${queryType} query: ${cacheKey}`); + const result = await queryFn(); + + // Cache the result + connectionCache.queryCache.set(cacheKey, { + data: result, + timestamp: now + }); + + return result; +} + +// Helper function to setup SSH tunnel - ONLY USED BY getDbConnection NOW async function setupSshTunnel() { const sshConfig = { host: process.env.PROD_SSH_HOST, @@ -303,221 +430,202 @@ router.delete('/delete-image', (req, res) => { // Get all options for import fields router.get('/field-options', async (req, res) => { - let ssh; - let connection; - try { - // Setup SSH tunnel and get database connection - const tunnel = await setupSshTunnel(); - ssh = tunnel.ssh; - - // Create MySQL connection over SSH tunnel - connection = await mysql.createConnection({ - ...tunnel.dbConfig, - stream: tunnel.stream + // Use cached connection + const { connection } = await getDbConnection(); + + const cacheKey = 'field-options'; + const result = await getCachedQuery(cacheKey, 'field-options', async () => { + // Fetch companies (type 1) + const [companies] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 1 + ORDER BY name + `); + + // Fetch artists (type 40) + const [artists] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 40 + ORDER BY name + `); + + // Fetch sizes (type 50) + const [sizes] = await connection.query(` + SELECT cat_id, name + FROM product_categories + WHERE type = 50 + ORDER BY name + `); + + // Fetch themes with subthemes + const [themes] = await connection.query(` + SELECT t.cat_id, t.name AS display_name, t.type, t.name AS sort_theme, + '' AS sort_subtheme, 1 AS level_order + FROM product_categories t + WHERE t.type = 20 + UNION ALL + SELECT ts.cat_id, CONCAT(t.name,' - ',ts.name) AS display_name, ts.type, + t.name AS sort_theme, ts.name AS sort_subtheme, 2 AS level_order + FROM product_categories ts + JOIN product_categories t ON ts.master_cat_id = t.cat_id + WHERE ts.type = 21 AND t.type = 20 + ORDER BY sort_theme, sort_subtheme + `); + + // Fetch categories with all levels + const [categories] = await connection.query(` + SELECT s.cat_id, s.name AS display_name, s.type, s.name AS sort_section, + '' AS sort_category, '' AS sort_subcategory, '' AS sort_subsubcategory, + 1 AS level_order + FROM product_categories s + WHERE s.type = 10 + UNION ALL + SELECT c.cat_id, CONCAT(s.name,' - ',c.name) AS display_name, c.type, + s.name AS sort_section, c.name AS sort_category, '' AS sort_subcategory, + '' AS sort_subsubcategory, 2 AS level_order + FROM product_categories c + JOIN product_categories s ON c.master_cat_id = s.cat_id + WHERE c.type = 11 AND s.type = 10 + UNION ALL + SELECT sc.cat_id, CONCAT(s.name,' - ',c.name,' - ',sc.name) AS display_name, + sc.type, s.name AS sort_section, c.name AS sort_category, + sc.name AS sort_subcategory, '' AS sort_subsubcategory, 3 AS level_order + FROM product_categories sc + JOIN product_categories c ON sc.master_cat_id = c.cat_id + JOIN product_categories s ON c.master_cat_id = s.cat_id + WHERE sc.type = 12 AND c.type = 11 AND s.type = 10 + UNION ALL + SELECT ssc.cat_id, CONCAT(s.name,' - ',c.name,' - ',sc.name,' - ',ssc.name) AS display_name, + ssc.type, s.name AS sort_section, c.name AS sort_category, + sc.name AS sort_subcategory, ssc.name AS sort_subsubcategory, 4 AS level_order + FROM product_categories ssc + JOIN product_categories sc ON ssc.master_cat_id = sc.cat_id + JOIN product_categories c ON sc.master_cat_id = c.cat_id + JOIN product_categories s ON c.master_cat_id = s.cat_id + WHERE ssc.type = 13 AND sc.type = 12 AND c.type = 11 AND s.type = 10 + ORDER BY sort_section, sort_category, sort_subcategory, sort_subsubcategory + `); + + // Fetch colors + const [colors] = await connection.query(` + SELECT color, name, hex_color + FROM product_color_list + ORDER BY \`order\` + `); + + // Fetch suppliers + const [suppliers] = await connection.query(` + SELECT supplierid as value, companyname as label + FROM suppliers + WHERE companyname <> '' + ORDER BY companyname + `); + + // Fetch tax categories + const [taxCategories] = await connection.query(` + SELECT tax_code_id as value, name as label + FROM product_tax_codes + ORDER BY tax_code_id = 0 DESC, name + `); + + // Format and return all options + return { + companies: companies.map(c => ({ label: c.name, value: c.cat_id.toString() })), + artists: artists.map(a => ({ label: a.name, value: a.cat_id.toString() })), + sizes: sizes.map(s => ({ label: s.name, value: s.cat_id.toString() })), + themes: themes.map(t => ({ + label: t.display_name, + value: t.cat_id.toString(), + type: t.type, + level: t.level_order + })), + categories: categories.map(c => ({ + label: c.display_name, + value: c.cat_id.toString(), + type: c.type, + level: c.level_order + })), + colors: colors.map(c => ({ + label: c.name, + value: c.color, + hexColor: c.hex_color + })), + suppliers: suppliers, + taxCategories: taxCategories, + shippingRestrictions: [ + { label: "None", value: "0" }, + { label: "US Only", value: "1" }, + { label: "Limited Quantity", value: "2" }, + { label: "US/CA Only", value: "3" }, + { label: "No FedEx 2 Day", value: "4" }, + { label: "North America Only", value: "5" } + ] + }; }); - // Fetch companies (type 1) - const [companies] = await connection.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 1 - ORDER BY name - `); - - // Fetch artists (type 40) - const [artists] = await connection.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 40 - ORDER BY name - `); - - // Fetch sizes (type 50) - const [sizes] = await connection.query(` - SELECT cat_id, name - FROM product_categories - WHERE type = 50 - ORDER BY name - `); - - // Fetch themes with subthemes - const [themes] = await connection.query(` - SELECT t.cat_id, t.name AS display_name, t.type, t.name AS sort_theme, - '' AS sort_subtheme, 1 AS level_order - FROM product_categories t - WHERE t.type = 20 - UNION ALL - SELECT ts.cat_id, CONCAT(t.name,' - ',ts.name) AS display_name, ts.type, - t.name AS sort_theme, ts.name AS sort_subtheme, 2 AS level_order - FROM product_categories ts - JOIN product_categories t ON ts.master_cat_id = t.cat_id - WHERE ts.type = 21 AND t.type = 20 - ORDER BY sort_theme, sort_subtheme - `); - - // Fetch categories with all levels - const [categories] = await connection.query(` - SELECT s.cat_id, s.name AS display_name, s.type, s.name AS sort_section, - '' AS sort_category, '' AS sort_subcategory, '' AS sort_subsubcategory, - 1 AS level_order - FROM product_categories s - WHERE s.type = 10 - UNION ALL - SELECT c.cat_id, CONCAT(s.name,' - ',c.name) AS display_name, c.type, - s.name AS sort_section, c.name AS sort_category, '' AS sort_subcategory, - '' AS sort_subsubcategory, 2 AS level_order - FROM product_categories c - JOIN product_categories s ON c.master_cat_id = s.cat_id - WHERE c.type = 11 AND s.type = 10 - UNION ALL - SELECT sc.cat_id, CONCAT(s.name,' - ',c.name,' - ',sc.name) AS display_name, - sc.type, s.name AS sort_section, c.name AS sort_category, - sc.name AS sort_subcategory, '' AS sort_subsubcategory, 3 AS level_order - FROM product_categories sc - JOIN product_categories c ON sc.master_cat_id = c.cat_id - JOIN product_categories s ON c.master_cat_id = s.cat_id - WHERE sc.type = 12 AND c.type = 11 AND s.type = 10 - UNION ALL - SELECT ssc.cat_id, CONCAT(s.name,' - ',c.name,' - ',sc.name,' - ',ssc.name) AS display_name, - ssc.type, s.name AS sort_section, c.name AS sort_category, - sc.name AS sort_subcategory, ssc.name AS sort_subsubcategory, 4 AS level_order - FROM product_categories ssc - JOIN product_categories sc ON ssc.master_cat_id = sc.cat_id - JOIN product_categories c ON sc.master_cat_id = c.cat_id - JOIN product_categories s ON c.master_cat_id = s.cat_id - WHERE ssc.type = 13 AND sc.type = 12 AND c.type = 11 AND s.type = 10 - ORDER BY sort_section, sort_category, sort_subcategory, sort_subsubcategory - `); - - // Fetch colors - const [colors] = await connection.query(` - SELECT color, name, hex_color - FROM product_color_list - ORDER BY \`order\` - `); - - // Fetch suppliers - const [suppliers] = await connection.query(` - SELECT supplierid as value, companyname as label - FROM suppliers - WHERE companyname <> '' - ORDER BY companyname - `); - - // Fetch tax categories - const [taxCategories] = await connection.query(` - SELECT tax_code_id as value, name as label - FROM product_tax_codes - ORDER BY tax_code_id = 0 DESC, name - `); - - res.json({ - companies: companies.map(c => ({ label: c.name, value: c.cat_id.toString() })), - artists: artists.map(a => ({ label: a.name, value: a.cat_id.toString() })), - sizes: sizes.map(s => ({ label: s.name, value: s.cat_id.toString() })), - themes: themes.map(t => ({ - label: t.display_name, - value: t.cat_id.toString(), - type: t.type, - level: t.level_order - })), - categories: categories.map(c => ({ - label: c.display_name, - value: c.cat_id.toString(), - type: c.type, - level: c.level_order - })), - colors: colors.map(c => ({ - label: c.name, - value: c.color, - hexColor: c.hex_color - })), - suppliers: suppliers, - taxCategories: taxCategories, - shippingRestrictions: [ - { label: "None", value: "0" }, - { label: "US Only", value: "1" }, - { label: "Limited Quantity", value: "2" }, - { label: "US/CA Only", value: "3" }, - { label: "No FedEx 2 Day", value: "4" }, - { label: "North America Only", value: "5" } - ] - }); + res.json(result); } catch (error) { console.error('Error fetching import field options:', error); res.status(500).json({ error: 'Failed to fetch import field options' }); - } finally { - if (connection) await connection.end(); - if (ssh) ssh.end(); } }); // Get product lines for a specific company router.get('/product-lines/:companyId', async (req, res) => { - let ssh; - let connection; - try { - // Setup SSH tunnel and get database connection - const tunnel = await setupSshTunnel(); - ssh = tunnel.ssh; + // Use cached connection + const { connection } = await getDbConnection(); + + const companyId = req.params.companyId; + const cacheKey = `product-lines-${companyId}`; - // Create MySQL connection over SSH tunnel - connection = await mysql.createConnection({ - ...tunnel.dbConfig, - stream: tunnel.stream + const lines = await getCachedQuery(cacheKey, 'product-lines', async () => { + const [queryResult] = await connection.query(` + SELECT cat_id as value, name as label + FROM product_categories + WHERE type = 2 + AND master_cat_id = ? + ORDER BY name + `, [companyId]); + + return queryResult.map(l => ({ label: l.label, value: l.value.toString() })); }); - const [lines] = await connection.query(` - SELECT cat_id as value, name as label - FROM product_categories - WHERE type = 2 - AND master_cat_id = ? - ORDER BY name - `, [req.params.companyId]); - - res.json(lines.map(l => ({ label: l.label, value: l.value.toString() }))); + res.json(lines); } catch (error) { console.error('Error fetching product lines:', error); res.status(500).json({ error: 'Failed to fetch product lines' }); - } finally { - if (connection) await connection.end(); - if (ssh) ssh.end(); } }); // Get sublines for a specific product line router.get('/sublines/:lineId', async (req, res) => { - let ssh; - let connection; - try { - // Setup SSH tunnel and get database connection - const tunnel = await setupSshTunnel(); - ssh = tunnel.ssh; + // Use cached connection + const { connection } = await getDbConnection(); + + const lineId = req.params.lineId; + const cacheKey = `sublines-${lineId}`; - // Create MySQL connection over SSH tunnel - connection = await mysql.createConnection({ - ...tunnel.dbConfig, - stream: tunnel.stream + const sublines = await getCachedQuery(cacheKey, 'sublines', async () => { + const [queryResult] = await connection.query(` + SELECT cat_id as value, name as label + FROM product_categories + WHERE type = 3 + AND master_cat_id = ? + ORDER BY name + `, [lineId]); + + return queryResult.map(s => ({ label: s.label, value: s.value.toString() })); }); - const [sublines] = await connection.query(` - SELECT cat_id as value, name as label - FROM product_categories - WHERE type = 3 - AND master_cat_id = ? - ORDER BY name - `, [req.params.lineId]); - - res.json(sublines.map(s => ({ label: s.label, value: s.value.toString() }))); + res.json(sublines); } catch (error) { console.error('Error fetching sublines:', error); res.status(500).json({ error: 'Failed to fetch sublines' }); - } finally { - if (connection) await connection.end(); - if (ssh) ssh.end(); } }); diff --git a/inventory/src/lib/react-spreadsheet-import/src/steps/MatchColumnsStep/MatchColumnsStep.tsx b/inventory/src/lib/react-spreadsheet-import/src/steps/MatchColumnsStep/MatchColumnsStep.tsx index 2ef1536..085ba80 100644 --- a/inventory/src/lib/react-spreadsheet-import/src/steps/MatchColumnsStep/MatchColumnsStep.tsx +++ b/inventory/src/lib/react-spreadsheet-import/src/steps/MatchColumnsStep/MatchColumnsStep.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useMemo, useState } from "react" +import React, { useCallback, useEffect, useMemo, useState, memo, useRef, useLayoutEffect } from "react" import { useRsi } from "../../hooks/useRsi" import { setColumn } from "./utils/setColumn" import { setIgnoreColumn } from "./utils/setIgnoreColumn" @@ -25,6 +25,7 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@ import { Badge } from "@/components/ui/badge" import { ScrollArea } from "@/components/ui/scroll-area" import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover" +import { useVirtualizer } from '@tanstack/react-virtual'; export type MatchColumnsProps = { data: RawData[] @@ -101,15 +102,175 @@ export type Column = export type Columns = Column[] +// Extract components to reduce re-renders +const ColumnActions = memo(({ + column, + onIgnore, + toggleValueMapping, + isExpanded, + canExpandValues +}: { + column: any, + onIgnore: (index: number) => void, + toggleValueMapping: (index: number) => void, + isExpanded: boolean, + canExpandValues: boolean +}) => { + // Create stable callback references to prevent unnecessary re-renders + const handleIgnore = useCallback(() => { + onIgnore(column.index); + }, [onIgnore, column.index]); -export const MatchColumnsStep = ({ + const handleToggleMapping = useCallback(() => { + toggleValueMapping(column.index); + }, [toggleValueMapping, column.index]); + + return ( +
+ {canExpandValues && ( + + )} + +
+ ); +}); + +// Additional performance optimization - use a memoized row renderer for column sample data +const MemoizedColumnSamplePreview = React.memo(({ samples }: { samples: any[] }) => { + return ( +
+ + + + + +
+

Sample Data

+
+ +
+ {samples.map((sample, i) => ( +
+ {i + 1}: + {String(sample || '(empty)')} +
+ ))} +
+ + + +
+ ); +}); + +// Replace the original ColumnSamplePreview with more optimized version +const ColumnSamplePreview = MemoizedColumnSamplePreview; + +// Add a memoized component for value mappings +const ValueMappings = memo(({ + column, + fieldOptions, + onSubChange +}: { + column: any, + fieldOptions: any[], + onSubChange: (value: string, columnIndex: number, entry: string) => void +}) => { + // Use a React.useMemo for expensive calculations + const matchedOptions = useMemo(() => column.matchedOptions || [], [column.matchedOptions]); + const columnIndex = useMemo(() => column.index, [column.index]); + + if (!fieldOptions || fieldOptions.length === 0) { + return ( +
+

+ No options available for this field. Options mapping is not required. +

+
+ ); + } + + return ( +
+
+

Map Values from Column to Field Options

+

+ Match values found in your spreadsheet to options available in the system +

+
+
+ {matchedOptions.map((matched: any, i: number) => { + // Set default value if none exists + const currentValue = (matched.value as string) || ""; + // Ensure entry is a string + const entryValue = matched.entry || ""; + const isUnmapped = !currentValue; + + // Use stable callback for value change + const handleValueChange = useCallback((value: string) => { + onSubChange(value, columnIndex, entryValue); + }, [onSubChange, columnIndex, entryValue]); + + return ( +
+
+ {entryValue || '(empty)'} +
+
+ +
+ +
+ ); + })} +
+
+ ); +}); + +export const MatchColumnsStep = React.memo(({ data, headerValues, onContinue, onBack, initialGlobalSelections -}: MatchColumnsProps) => { - const dataExample = useMemo(() => data.slice(0, 5), [data]) // Show 5 sample rows +}: MatchColumnsProps): JSX.Element => { const { fields, autoMapHeaders, autoMapSelectValues, autoMapDistance, translations } = useRsi() const [isLoading, setIsLoading] = useState(false) const [columns, setColumns] = useState>( @@ -120,14 +281,25 @@ export const MatchColumnsStep = ({ const [showAllColumns, setShowAllColumns] = useState(false) const [expandedValueMappings, setExpandedValueMappings] = useState([]) - // Toggle the expanded state for value mappings - const toggleValueMapping = (columnIndex: number) => { - setExpandedValueMappings(prev => + // Use debounce for expensive operations + const [expandedValues, setExpandedValues] = useState([]); + + useEffect(() => { + const timeoutId = setTimeout(() => { + setExpandedValueMappings(expandedValues); + }, 50); + + return () => clearTimeout(timeoutId); + }, [expandedValues]); + + // Toggle with immediate visual feedback but debounced actual state change + const toggleValueMappingOptimized = useCallback((columnIndex: number) => { + setExpandedValues(prev => prev.includes(columnIndex) ? prev.filter(idx => idx !== columnIndex) : [...prev, columnIndex] ); - }; + }, []); // Check if column is expandable (has value mappings) const isExpandable = useCallback((column: Column) => { @@ -145,18 +317,6 @@ export const MatchColumnsStep = ({ } }, [initialGlobalSelections]) - // Fetch field options from the API - const { data: fieldOptions } = useQuery({ - queryKey: ["import-field-options"], - queryFn: async () => { - const response = await fetch(`${config.apiUrl}/import/field-options`); - if (!response.ok) { - throw new Error("Failed to fetch field options"); - } - return response.json(); - }, - }); - // Fetch product lines when company is selected const { data: productLines } = useQuery({ queryKey: ["product-lines", globalSelections.company], @@ -169,6 +329,7 @@ export const MatchColumnsStep = ({ return response.json(); }, enabled: !!globalSelections.company, + staleTime: 600000, // 10 minutes (increased from 60 seconds) }); // Fetch sublines when line is selected @@ -183,6 +344,7 @@ export const MatchColumnsStep = ({ return response.json(); }, enabled: !!globalSelections.line, + staleTime: 600000, // 10 minutes (increased from 60 seconds) }); // Find mapped column for a specific field @@ -245,6 +407,7 @@ export const MatchColumnsStep = ({ return response.json(); }, enabled: !!mappedCompanyValue && mappedCompanyValue !== globalSelections.company, + staleTime: 600000, // 10 minutes (increased from 60 seconds) }); // Fetch sublines for mapped line @@ -259,25 +422,58 @@ export const MatchColumnsStep = ({ return response.json(); }, enabled: !!mappedLineValue && mappedLineValue !== globalSelections.line, + staleTime: 600000, // 10 minutes (increased from 60 seconds) }); + // Get field options for suppliers and companies + const { data: fieldOptionsData } = useQuery({ + queryKey: ["field-options"], + queryFn: async () => { + const response = await fetch(`${config.apiUrl}/import/field-options`); + if (!response.ok) { + throw new Error("Failed to fetch field options"); + } + return response.json(); + }, + staleTime: 600000, // 10 minutes (increased from 60 seconds) + }); + + // Safely access field options + const fieldOptions = fieldOptionsData || { suppliers: [], companies: [] }; + + // Create a stable identity for these queries to avoid re-renders + const stableFieldOptions = useMemo(() => fieldOptionsData || { suppliers: [], companies: [] }, [fieldOptionsData]); + const stableProductLines = useMemo(() => productLines || [], [productLines]); + const stableSublines = useMemo(() => sublines || [], [sublines]); + const stableMappedProductLines = useMemo(() => mappedProductLines || [], [mappedProductLines]); + const stableMappedSublines = useMemo(() => mappedSublines || [], [mappedSublines]); + + // Type guard for suppliers and companies + const hasSuppliers = (options: any): options is { suppliers: any[] } => + options && Array.isArray(options.suppliers); + + const hasCompanies = (options: any): options is { companies: any[] } => + options && Array.isArray(options.companies); + // Check if a field is covered by global selections const isFieldCoveredByGlobalSelections = useCallback((key: string) => { - const isCovered = (key === 'supplier' && globalSelections.supplier) || - (key === 'company' && globalSelections.company) || - (key === 'line' && globalSelections.line) || - (key === 'subline' && globalSelections.subline); - console.log(`Field ${key} covered by global selections:`, isCovered); - return isCovered; - }, [globalSelections]); + return (key === 'supplier' && !!globalSelections.supplier) || + (key === 'company' && !!globalSelections.company) || + (key === 'line' && !!globalSelections.line) || + (key === 'subline' && !!globalSelections.subline); + }, [ + globalSelections.supplier, + globalSelections.company, + globalSelections.line, + globalSelections.subline + ]); - // Get possible mapping values for a field + // Create a stable version of getFieldOptions with improved memoization const getFieldOptions = useCallback((fieldKey: string) => { const fieldsArray = Array.isArray(fields) ? fields : [fields]; const field = fieldsArray.find(f => f.key === fieldKey); if (!field) { - console.log(`Field ${fieldKey} not found`); return []; } @@ -285,20 +481,16 @@ export const MatchColumnsStep = ({ if (fieldKey === 'line') { // For line, return the appropriate product lines based on the company if (globalSelections.company) { - console.log(`Using product lines for global company:`, productLines); - return productLines || []; + return stableProductLines; } else if (mappedCompanyValue) { - console.log(`Using product lines for mapped company:`, mappedProductLines); - return mappedProductLines || []; + return stableMappedProductLines; } } else if (fieldKey === 'subline') { // For subline, return the appropriate sublines based on the line if (globalSelections.line) { - console.log(`Using sublines for global line:`, sublines); - return sublines || []; + return stableSublines; } else if (mappedLineValue) { - console.log(`Using sublines for mapped line:`, mappedSublines); - return mappedSublines || []; + return stableMappedSublines; } } @@ -308,13 +500,6 @@ export const MatchColumnsStep = ({ // If no options at the root level, check in fieldType if ((!options || options.length === 0) && field.fieldType && field.fieldType.options) { options = field.fieldType.options; - console.log(`Found options in fieldType for ${fieldKey}:`, options); - } else { - console.log(`Using options from root level for ${fieldKey}:`, options); - } - - if (!options || options.length === 0) { - console.log(`No options found for field ${fieldKey}`, field); } return options || []; @@ -322,12 +507,12 @@ export const MatchColumnsStep = ({ fields, globalSelections.company, globalSelections.line, - productLines, - sublines, + stableProductLines, + stableSublines, mappedCompanyValue, mappedLineValue, - mappedProductLines, - mappedSublines + stableMappedProductLines, + stableMappedSublines ]); // Check if a column has unmapped values (for select fields) @@ -342,6 +527,18 @@ export const MatchColumnsStep = ({ return fieldOptions.length > 0 && column.matchedOptions.some(option => !option.value); }, [isExpandable, getFieldOptions]); + + // Optimize columnsWithUnmappedValuesMap calculation - prevent deep comparisons + const columnsWithUnmappedValuesMap = useMemo(() => { + const map = new Map(); + // Prevent recalculation by checking if we already have this value mapped + columns.forEach(col => { + if (!map.has(col.index)) { + map.set(col.index, hasUnmappedValues(col)); + } + }); + return map; + }, [columns, hasUnmappedValues]); // Get matched, unmapped, and columns with unmapped values const { matchedColumns, unmatchedColumns, columnsWithUnmappedValues } = useMemo(() => { @@ -349,14 +546,14 @@ export const MatchColumnsStep = ({ const withUnmappedValues = columns.filter(col => col.type !== ColumnType.empty && col.type !== ColumnType.ignored && - hasUnmappedValues(col) + columnsWithUnmappedValuesMap.get(col.index) ); // These are columns that are mapped AND have all their values properly mapped const fullyMapped = columns.filter(col => col.type !== ColumnType.empty && col.type !== ColumnType.ignored && - !hasUnmappedValues(col) + !columnsWithUnmappedValuesMap.get(col.index) ); // Unmapped columns @@ -367,7 +564,7 @@ export const MatchColumnsStep = ({ unmatchedColumns: unmatched, columnsWithUnmappedValues: withUnmappedValues }; - }, [columns, hasUnmappedValues]); + }, [columns, columnsWithUnmappedValuesMap]); // Get ignored columns const ignoredColumns = useMemo(() => { @@ -580,10 +777,19 @@ export const MatchColumnsStep = ({ ) }, [columns, getFieldOptions, isExpandable]); - // Run auto-mapping on component mount + // Run auto-mapping only when needed - when columns change or field options change useEffect(() => { - autoMapAllValues(); - }, [autoMapAllValues]); + // Check if we have columns that need mapping + const needsMapping = columns.some(column => + isExpandable(column) && + "matchedOptions" in column && + column.matchedOptions.some(option => !option.value) + ); + + if (needsMapping) { + autoMapAllValues(); + } + }, [autoMapAllValues, columns, fields, productLines, sublines]); const onIgnore = useCallback( (columnIndex: number) => { @@ -683,6 +889,7 @@ export const MatchColumnsStep = ({ return field?.label || key; }, [fields]); + // Fix handleOnContinue - it should be useCallback, not useEffect const handleOnContinue = useCallback(async () => { setIsLoading(true) // Normalize the data with global selections before continuing @@ -701,53 +908,63 @@ export const MatchColumnsStep = ({ [], ) - // Helper to get sample data for a column - const getColumnSamples = (columnIndex: number) => { - return dataExample.map(row => row[columnIndex]); - }; + // Missing dataExample - let's create a data sample from the actual data + const dataSample = useMemo(() => { + // Take first 5 rows as sample data + return data.slice(0, 5); + }, [data]); - // Automatically expand columns with unmapped values + // Optimize expensive getColumnSamples calls with memoization and ref + const columnSamplesCache = useRef(new Map()); + + // Enhanced version with caching and a more stable identity + const getColumnSamples = useCallback((columnIndex: number) => { + if (!columnSamplesCache.current.has(columnIndex)) { + // Only build samples from data when needed + const samples = dataSample.slice(0, 5).map(row => row[columnIndex] || ''); + columnSamplesCache.current.set(columnIndex, samples); + } + return columnSamplesCache.current.get(columnIndex) || []; + }, [dataSample]); // Only depends on dataSample, not on data directly + + // Clear the cache when the data changes significantly useEffect(() => { - columnsWithUnmappedValues.forEach(column => { - if (!expandedValueMappings.includes(column.index)) { - setExpandedValueMappings(prev => [...prev, column.index]); - } - }); + columnSamplesCache.current.clear(); + }, [data.length]); // Only clear when data length changes, not on every data reference change + + // Optimize the renderSamplePreview function + const renderSamplePreview = useCallback((columnIndex: number) => { + const samples = getColumnSamples(columnIndex); + return ; + }, [getColumnSamples]); + + // Automatically expand columns with unmapped values - use layoutEffect to avoid flashing + useLayoutEffect(() => { + // Only add new columns that need to be expanded without removing existing ones + const columnsToExpand = columnsWithUnmappedValues + .filter(column => !expandedValueMappings.includes(column.index)) + .map(column => column.index); + + if (columnsToExpand.length > 0) { + setExpandedValueMappings(prev => [...prev, ...columnsToExpand]); + } }, [columnsWithUnmappedValues, expandedValueMappings]); - // Render the sample data preview - const renderSamplePreview = (columnIndex: number) => { - const samples = getColumnSamples(columnIndex); - return ( -
- - - - - -
-

Sample Data

-
- -
- {samples.map((sample, i) => ( -
- {i + 1}: - {String(sample || '(empty)')} -
- ))} -
- - - -
- ); - }; + // Create a stable mapping of column index to change handlers + const columnChangeHandlers = useMemo(() => { + const handlers = new Map void>(); + + columns.forEach(column => { + handlers.set(column.index, (value: string) => { + onChange(value as T, column.index); + }); + }); + + return handlers; + }, [columns, onChange]); // Render the field selector for a column - const renderFieldSelector = (column: Column, isUnmapped: boolean = false) => { + const renderFieldSelector = useCallback((column: Column, isUnmapped: boolean = false) => { // For ignored columns, show a badge if (column.type === ColumnType.ignored) { return Ignored; @@ -759,10 +976,13 @@ export const MatchColumnsStep = ({ // Use all fields for mapped columns, and only available fields for unmapped columns const fieldCategoriesForSelector = isUnmapped ? availableFieldCategories : allFieldCategories; + // Get the pre-created onChange handler for this column + const handleChange = columnChangeHandlers.get(column.index); + return ( ); - }; + }, [availableFieldCategories, allFieldCategories, columnChangeHandlers]); - // Render value mappings for select-type fields - const renderValueMappings = (column: Column) => { + // Replace the renderValueMappings function with a memoized version + const renderValueMappings = useCallback((column: Column) => { if (!isExpandable(column) || !("matchedOptions" in column) || !("value" in column)) { return null; } const fieldOptions = getFieldOptions(column.value as string); - console.log(`Rendering value mappings for ${column.header}`, fieldOptions); - - // If no options available, show a message - if (!fieldOptions || fieldOptions.length === 0) { - return ( -
-

- No options available for this field. Options mapping is not required. -

-
- ); - } - + return ; + }, [isExpandable, getFieldOptions, onSubChange]); + + // Remove virtualization references and helpers + const renderTable = useMemo(() => { return ( -
-
-

Map Values from Column to Field Options

-

- Match values found in your spreadsheet to options available in the system -

-
-
- {column.matchedOptions.map((matched, i) => { - // Set default value if none exists - const currentValue = (matched.value as string) || ""; - // Ensure entry is a string - const entryValue = matched.entry || ""; - const isUnmapped = !currentValue; + + + + + Spreadsheet Column + Data + + Map To Field + Ignore + + + + {/* Always show columns with unmapped values */} + {columnsWithUnmappedValues.map((column) => { + const isExpanded = expandedValueMappings.includes(column.index); + + return ( + + + {column.header} + + {renderSamplePreview(column.index)} + + + + + + {renderFieldSelector(column)} + + + + + + + {/* Value mappings row */} + {isExpanded && ( + + + {renderValueMappings(column)} + + + )} + + ); + })} - return ( -
-
- {entryValue || '(empty)'} -
-
- -
- -
- ); - })} - - + {/* Always show unmapped columns */} + {unmatchedColumns.map((column) => ( + + {column.header} + + {renderSamplePreview(column.index)} + + + + + + {renderFieldSelector(column, true)} + + + + + + ))} + + {/* Show matched columns if showAllColumns is true */} + {showAllColumns && matchedColumns.map((column) => { + const isExpanded = expandedValueMappings.includes(column.index); + const canExpandValues = isExpandable(column); + + return ( + + + {column.header} + + {renderSamplePreview(column.index)} + + + + + + {renderFieldSelector(column)} + + + + + + + {/* Value mappings row */} + {isExpanded && canExpandValues && ( + + + {renderValueMappings(column)} + + + )} + + ); + })} + + {/* Show ignored columns if showAllColumns is true */} + {showAllColumns && ignoredColumns.map((column) => ( + + {column.header} + + {renderSamplePreview(column.index)} + + + + + + Ignored + + + + + + ))} + + {/* Show a message if all columns are mapped/ignored */} + {unmatchedColumns.length === 0 && columnsWithUnmappedValues.length === 0 && !showAllColumns && ( + + + All columns have been mapped or ignored. + + + + )} + +
+ ); - }; + }, [ + columnsWithUnmappedValues, + unmatchedColumns, + matchedColumns, + ignoredColumns, + showAllColumns, + expandedValueMappings, + renderSamplePreview, + renderFieldSelector, + renderValueMappings, + onIgnore, + onRevertIgnore, + toggleValueMappingOptimized, + isExpandable + ]); return (
@@ -878,11 +1224,12 @@ export const MatchColumnsStep = ({ - {fieldOptions?.suppliers?.map((supplier: any) => ( - - {supplier.label} - - ))} + {hasSuppliers(stableFieldOptions) ? + stableFieldOptions.suppliers.map((supplier: any) => ( + + {supplier.label} + + )) : null}
@@ -904,11 +1251,12 @@ export const MatchColumnsStep = ({ - {fieldOptions?.companies?.map((company: any) => ( - - {company.label} - - ))} + {hasCompanies(stableFieldOptions) ? + stableFieldOptions.companies.map((company: any) => ( + + {company.label} + + )) : null}
@@ -930,11 +1278,12 @@ export const MatchColumnsStep = ({ - {productLines?.map((line: any) => ( - - {line.label} - - ))} + {Array.isArray(stableProductLines) ? + stableProductLines.map((line: any) => ( + + {line.label} + + )) : null}
@@ -950,11 +1299,12 @@ export const MatchColumnsStep = ({ - {sublines?.map((subline: any) => ( - - {subline.label} - - ))} + {Array.isArray(stableSublines) ? + stableSublines.map((subline: any) => ( + + {subline.label} + + )) : null} @@ -1005,8 +1355,6 @@ export const MatchColumnsStep = ({ )} - - {/* Right panel - Column mapping interface */} @@ -1037,203 +1385,7 @@ export const MatchColumnsStep = ({
- - - - - Spreadsheet Column - Sample Data - - Map To Field - Action - - - - {/* Always show columns with unmapped values */} - {columnsWithUnmappedValues.map((column) => { - const isExpanded = expandedValueMappings.includes(column.index); - - return ( - - - {column.header} - - {renderSamplePreview(column.index)} - - - - - - {renderFieldSelector(column)} - - - - - - - - {/* Value mappings row */} - {isExpanded && ( - - - {renderValueMappings(column)} - - - )} - - ); - })} - - {/* Always show unmapped columns */} - {unmatchedColumns.map((column) => ( - - {column.header} - - {renderSamplePreview(column.index)} - - - - - - {renderFieldSelector(column, true)} - - - - - - ))} - - {/* Show matched columns if showAllColumns is true */} - {showAllColumns && matchedColumns.map((column) => { - const isExpanded = expandedValueMappings.includes(column.index); - const canExpandValues = isExpandable(column); - - return ( - - - {column.header} - - {renderSamplePreview(column.index)} - - - - - - {renderFieldSelector(column)} - - - {canExpandValues && ( - - )} - - - - - {/* Value mappings row */} - {isExpanded && canExpandValues && ( - - - {renderValueMappings(column)} - - - )} - - ); - })} - - {/* Show ignored columns if showAllColumns is true */} - {showAllColumns && ignoredColumns.map((column) => ( - - {column.header} - - {renderSamplePreview(column.index)} - - - - - - Ignored - - - - - - ))} - - {/* Show a message if all columns are mapped/ignored */} - {unmatchedColumns.length === 0 && columnsWithUnmappedValues.length === 0 && !showAllColumns && ( - - - All columns have been mapped or ignored. - - - - )} - -
- + {renderTable}
@@ -1259,4 +1411,4 @@ export const MatchColumnsStep = ({ ) -} +})