const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');
const BATCH_SIZE = 1000; // Smaller batch size for better progress tracking
const MAX_RETRIES = 3;
const RETRY_DELAY = 5000; // 5 seconds
const dotenv = require("dotenv");
const path = require("path");
dotenv.config({ path: path.join(__dirname, "../../.env") });

// Utility functions
const imageUrlBase = process.env.PRODUCT_IMAGE_URL_BASE || 'https://sbing.com/i/products/0000/';
const getImageUrls = (pid, iid = 1) => {
  const paddedPid = pid.toString().padStart(6, '0');
  // Use padded PID only for the first 3 digits
  const prefix = paddedPid.slice(0, 3);
  // Use the actual pid for the rest of the URL
  const basePath = `${imageUrlBase}${prefix}/${pid}`;
  return {
    image: `${basePath}-t-${iid}.jpg`,
    image_175: `${basePath}-175x175-${iid}.jpg`,
    image_full: `${basePath}-o-${iid}.jpg`
  };
};

// Add helper function for retrying operations with exponential backoff
async function withRetry(operation, errorMessage) {
  let lastError;
  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
    try {
      return await operation();
    } catch (error) {
      lastError = error;
      console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error);
      if (attempt < MAX_RETRIES) {
        const backoffTime = RETRY_DELAY * Math.pow(2, attempt - 1);
        await new Promise(resolve => setTimeout(resolve, backoffTime));
      }
    }
  }
  throw lastError;
}

// Add helper function at the top of the file
function validateDate(mysqlDate) {
  if (!mysqlDate || mysqlDate === '0000-00-00' || mysqlDate === '0000-00-00 00:00:00') {
    return null;
  }
  // Check if the date is valid
  const date = new Date(mysqlDate);
  return isNaN(date.getTime()) ? null : mysqlDate;
}

async function setupTemporaryTables(connection) {
  // Drop the table if it exists
  await connection.query('DROP TABLE IF EXISTS temp_products');

  // Create the temporary table
  await connection.query(`
    CREATE TEMP TABLE temp_products (
      pid BIGINT NOT NULL,
      title TEXT,
      description TEXT,
      sku TEXT,
      stock_quantity INTEGER DEFAULT 0,
      preorder_count INTEGER DEFAULT 0,
      notions_inv_count INTEGER DEFAULT 0,
      price NUMERIC(14, 4) NOT NULL DEFAULT 0,
      regular_price NUMERIC(14, 4) NOT NULL DEFAULT 0,
      cost_price NUMERIC(14, 4),
      vendor TEXT,
      vendor_reference TEXT,
      notions_reference TEXT,
      brand TEXT,
      line TEXT,
      subline TEXT,
      artist TEXT,
      categories TEXT,
      created_at TIMESTAMP WITH TIME ZONE,
      date_online TIMESTAMP WITH TIME ZONE,
      first_received TIMESTAMP WITH TIME ZONE,
      barcode TEXT,
      harmonized_tariff_code TEXT,
      updated_at TIMESTAMP WITH TIME ZONE,
      visible BOOLEAN,
      managing_stock BOOLEAN DEFAULT true,
      replenishable BOOLEAN,
      permalink TEXT,
      moq INTEGER DEFAULT 1,
      uom INTEGER DEFAULT 1,
      rating NUMERIC(14, 4),
      reviews INTEGER,
      weight NUMERIC(14, 4),
      length NUMERIC(14, 4),
      width NUMERIC(14, 4),
      height NUMERIC(14, 4),
      country_of_origin TEXT,
      location TEXT,
      total_sold INTEGER,
      baskets INTEGER,
      notifies INTEGER,
      date_last_sold TIMESTAMP WITH TIME ZONE,
      shop_score NUMERIC(10, 2) DEFAULT 0,
      primary_iid INTEGER,
      image TEXT,
      image_175 TEXT,
      image_full TEXT,
      options TEXT,
      tags TEXT,
      needs_update BOOLEAN DEFAULT TRUE,
      PRIMARY KEY (pid)
    )`);

  // Create the index
  await connection.query('CREATE INDEX idx_temp_products_needs_update ON temp_products (needs_update)');
}

async function cleanupTemporaryTables(connection) {
  await connection.query('DROP TABLE IF EXISTS temp_products');
}

async function importMissingProducts(prodConnection, localConnection, missingPids) {
  if (!missingPids || missingPids.length === 0) {
    return {
      status: "complete",
      recordsAdded: 0,
      message: "No missing products to import"
    };
  }

  try {
    // Setup temporary tables
    await setupTemporaryTables(localConnection);

    // Get product data from production - Keep MySQL compatible
    const [prodData] = await prodConnection.query(`
      SELECT 
        p.pid,
        p.description AS title,
        p.notes AS description,
        p.itemnumber AS sku,
        p.date_created,
        p.date_ol,
        p.datein AS first_received,
        p.location,
        p.upc AS barcode,
        p.harmonized_tariff_code,
        p.stamp AS updated_at,
        CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
        CASE
          WHEN p.reorder < 0 THEN 0
          WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
          WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
          WHEN (
            (COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
            AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
            AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
          ) THEN 0
          ELSE 1
        END AS replenishable,
        COALESCE(si.available_local, 0) as stock_quantity,
        0 as pending_qty,
        COALESCE(ci.onpreorder, 0) as preorder_count,
        COALESCE(pnb.inventory, 0) as notions_inv_count,
        COALESCE(pcp.price_each, 0) as price,
        COALESCE(p.sellingprice, 0) AS regular_price,
        CASE
          WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
          THEN (
            SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
            FROM product_inventory
            WHERE pid = p.pid AND count > 0
          )
          ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
        END AS cost_price,
        s.companyname AS vendor,
        CASE
          WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
          ELSE sid.supplier_itemnumber
        END AS vendor_reference,
        sid.notions_itemnumber AS notions_reference,
        CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
        pc1.name AS brand,
        pc2.name AS line,
        pc3.name AS subline,
        pc4.name AS artist,
        COALESCE(CASE
          WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
          ELSE sid.supplier_qty_per_unit
        END, sid.notions_qty_per_unit) AS moq,
        p.rating,
        p.rating_votes AS reviews,
        p.weight,
        p.length,
        p.width,
        p.height,
        p.country_of_origin,
        (SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
        (SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
        (SELECT COALESCE(SUM(oi.qty_ordered), 0)
         FROM order_items oi
         JOIN _order o ON oi.order_id = o.order_id
         WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
        pls.date_sold as date_last_sold,
        COALESCE(p.score, 0) as shop_score,
        (SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
        GROUP_CONCAT(DISTINCT CASE
          WHEN pc.cat_id IS NOT NULL
          AND pc.type IN (10, 20, 11, 21, 12, 13)
          AND pci.cat_id NOT IN (16, 17)
          THEN pci.cat_id
        END) as category_ids
      FROM products p
      LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
      LEFT JOIN current_inventory ci ON p.pid = ci.pid
      LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
      LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
      LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
      LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
      LEFT JOIN product_category_index pci ON p.pid = pci.pid
      LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
      LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
      LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
      LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
      LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
      LEFT JOIN product_last_sold pls ON p.pid = pls.pid
      WHERE p.pid IN (?)
      GROUP BY p.pid
    `, [missingPids]);

    if (!prodData || prodData.length === 0) {
      return {
        status: "complete",
        recordsAdded: 0,
        message: "No products found in production database"
      };
    }

    // Process in batches
    let recordsAdded = 0;
    for (let i = 0; i < prodData.length; i += BATCH_SIZE) {
      const batch = prodData.slice(i, i + BATCH_SIZE);
      
      const placeholders = batch.map((_, idx) => {
        const base = idx * 49; // 49 columns
        return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
      }).join(',');

      const values = batch.flatMap(row => {
        const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
        return [
          row.pid,
          row.title,
          row.description,
          row.sku || '',
          row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
          row.preorder_count,
          row.notions_inv_count,
          row.price,
          row.regular_price,
          row.cost_price,
          row.vendor,
          row.vendor_reference,
          row.notions_reference,
          row.brand,
          row.line,
          row.subline,
          row.artist,
          row.category_ids,
          validateDate(row.date_created),
          validateDate(row.date_ol),
          validateDate(row.first_received),
          row.barcode,
          row.harmonized_tariff_code,
          validateDate(row.updated_at),
          row.visible,
          true,
          row.replenishable,
          row.permalink,
          Math.max(1, Math.round(row.moq || 1)),
          1,
          row.rating,
          row.reviews,
          row.weight,
          row.length,
          row.width,
          row.height,
          row.country_of_origin,
          row.location,
          row.total_sold,
          row.baskets,
          row.notifies,
          validateDate(row.date_last_sold),
          Number(row.shop_score) || 0,
          row.primary_iid,
          imageUrls.image,
          imageUrls.image_175,
          imageUrls.image_full,
          null,
          null
        ];
      });

      const [result] = await localConnection.query(`
        WITH inserted_products AS (
          INSERT INTO products (
            pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
            price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
            brand, line, subline, artist, categories, created_at, date_online, first_received,
            barcode, harmonized_tariff_code, updated_at, visible,
            managing_stock, replenishable, permalink, moq, uom, rating, reviews,
            weight, length, width, height, country_of_origin, location, total_sold,
            baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
          )
          VALUES ${placeholders}
          ON CONFLICT (pid) DO NOTHING
          RETURNING pid
        )
        SELECT COUNT(*) as inserted FROM inserted_products
      `, values);

      recordsAdded += parseInt(result.rows[0].inserted, 10) || 0;
    }

    return {
      status: "complete",
      recordsAdded,
      message: `Successfully imported ${recordsAdded} missing products`
    };
  } catch (error) {
    console.error('Error importing missing products:', error);
    throw error;
  }
}

async function materializeCalculations(prodConnection, localConnection, incrementalUpdate = true, lastSyncTime = '1970-01-01', startTime = Date.now()) {
  outputProgress({
    status: "running",
    operation: "Products import",
    message: "Fetching product data from production"
  });

  // Get all product data in a single optimized query - Keep MySQL compatible
  const [prodData] = await prodConnection.query(`
    SELECT 
      p.pid,
      p.description AS title,
      p.notes AS description,
      p.itemnumber AS sku,
      p.date_created,
      p.date_ol,
      p.datein AS first_received,
      p.location,
      p.upc AS barcode,
      p.harmonized_tariff_code,
      p.stamp AS updated_at,
      CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
      CASE
        WHEN p.reorder < 0 THEN 0
        WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
        WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
        WHEN (
          (COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
          AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
          AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
        ) THEN 0
        ELSE 1
      END AS replenishable,
      COALESCE(si.available_local, 0) as stock_quantity,
      0 as pending_qty,
      COALESCE(ci.onpreorder, 0) as preorder_count,
      COALESCE(pnb.inventory, 0) as notions_inv_count,
      COALESCE(pcp.price_each, 0) as price,
      COALESCE(p.sellingprice, 0) AS regular_price,
      CASE
        WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
        THEN (
          SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
          FROM product_inventory
          WHERE pid = p.pid AND count > 0
        )
        ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
      END AS cost_price,
      s.companyname AS vendor,
      CASE
        WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
        ELSE sid.supplier_itemnumber
      END AS vendor_reference,
      sid.notions_itemnumber AS notions_reference,
      CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
      pc1.name AS brand,
      pc2.name AS line,
      pc3.name AS subline,
      pc4.name AS artist,
      COALESCE(CASE
        WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
        ELSE sid.supplier_qty_per_unit
      END, sid.notions_qty_per_unit) AS moq,
      p.rating,
      p.rating_votes AS reviews,
      p.weight,
      p.length,
      p.width,
      p.height,
      p.country_of_origin,
      (SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
      (SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
      (SELECT COALESCE(SUM(oi.qty_ordered), 0)
       FROM order_items oi
       JOIN _order o ON oi.order_id = o.order_id
       WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
      pls.date_sold as date_last_sold,
      COALESCE(p.score, 0) as shop_score,
      (SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
      GROUP_CONCAT(DISTINCT CASE
        WHEN pc.cat_id IS NOT NULL
        AND pc.type IN (10, 20, 11, 21, 12, 13)
        AND pci.cat_id NOT IN (16, 17)
        THEN pci.cat_id
      END) as category_ids
    FROM products p
    LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
    LEFT JOIN current_inventory ci ON p.pid = ci.pid
    LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
    LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
    LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
    LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
    LEFT JOIN product_category_index pci ON p.pid = pci.pid
    LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
    LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
    LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
    LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
    LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
    LEFT JOIN product_last_sold pls ON p.pid = pls.pid
    WHERE ${incrementalUpdate ? `
      p.date_created >= DATE(?) OR
      p.stamp > ? OR
      ci.stamp > ? OR
      pcp.date_deactive > ? OR
      pcp.date_active > ? OR
      pnb.date_updated > ?
    ` : 'TRUE'}
    GROUP BY p.pid
  `, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);

  outputProgress({
    status: "running",
    operation: "Products import",
    message: `Processing ${prodData.length} product records`
  });

  // Insert all product data into temp table in batches
  for (let i = 0; i < prodData.length; i += BATCH_SIZE) {
    const batch = prodData.slice(i, Math.min(i + BATCH_SIZE, prodData.length));
    
    await withRetry(async () => {
      const placeholders = batch.map((_, idx) => {
        const base = idx * 49; // 49 columns
        return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
      }).join(',');

      const values = batch.flatMap(row => {
        const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
        return [
          row.pid,
          row.title,
          row.description,
          row.sku || '',
          row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
          row.preorder_count,
          row.notions_inv_count,
          row.price,
          row.regular_price,
          row.cost_price,
          row.vendor,
          row.vendor_reference,
          row.notions_reference,
          row.brand,
          row.line,
          row.subline,
          row.artist,
          row.category_ids,
          validateDate(row.date_created),
          validateDate(row.date_ol),
          validateDate(row.first_received),
          row.barcode,
          row.harmonized_tariff_code,
          validateDate(row.updated_at),
          row.visible,
          true,
          row.replenishable,
          row.permalink,
          Math.max(1, Math.round(row.moq || 1)),
          1,
          row.rating,
          row.reviews,
          row.weight,
          row.length,
          row.width,
          row.height,
          row.country_of_origin,
          row.location,
          row.total_sold,
          row.baskets,
          row.notifies,
          validateDate(row.date_last_sold),
          Number(row.shop_score) || 0,
          row.primary_iid,
          imageUrls.image,
          imageUrls.image_175,
          imageUrls.image_full,
          null,
          null
        ];
      });

      await localConnection.query(`
        INSERT INTO temp_products (
          pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
          price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
          brand, line, subline, artist, categories, created_at, date_online, first_received,
          barcode, harmonized_tariff_code, updated_at, visible,
          managing_stock, replenishable, permalink, moq, uom, rating, reviews,
          weight, length, width, height, country_of_origin, location, total_sold,
          baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
        ) VALUES ${placeholders}
        ON CONFLICT (pid) DO UPDATE SET
          title = EXCLUDED.title,
          description = EXCLUDED.description,
          sku = EXCLUDED.sku,
          stock_quantity = EXCLUDED.stock_quantity,
          preorder_count = EXCLUDED.preorder_count,
          notions_inv_count = EXCLUDED.notions_inv_count,
          price = EXCLUDED.price,
          regular_price = EXCLUDED.regular_price,
          cost_price = EXCLUDED.cost_price,
          vendor = EXCLUDED.vendor,
          vendor_reference = EXCLUDED.vendor_reference,
          notions_reference = EXCLUDED.notions_reference,
          brand = EXCLUDED.brand,
          line = EXCLUDED.line,
          subline = EXCLUDED.subline,
          artist = EXCLUDED.artist,
          created_at = EXCLUDED.created_at,
          date_online = EXCLUDED.date_online,
          first_received = EXCLUDED.first_received,
          barcode = EXCLUDED.barcode,
          harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
          updated_at = EXCLUDED.updated_at,
          visible = EXCLUDED.visible,
          managing_stock = EXCLUDED.managing_stock,
          replenishable = EXCLUDED.replenishable,
          permalink = EXCLUDED.permalink,
          moq = EXCLUDED.moq,
          uom = EXCLUDED.uom,
          rating = EXCLUDED.rating,
          reviews = EXCLUDED.reviews,
          weight = EXCLUDED.weight,
          length = EXCLUDED.length,
          width = EXCLUDED.width,
          height = EXCLUDED.height,
          country_of_origin = EXCLUDED.country_of_origin,
          location = EXCLUDED.location,
          total_sold = EXCLUDED.total_sold,
          baskets = EXCLUDED.baskets,
          notifies = EXCLUDED.notifies,
          date_last_sold = EXCLUDED.date_last_sold,
          shop_score = EXCLUDED.shop_score,
          primary_iid = EXCLUDED.primary_iid,
          image = EXCLUDED.image,
          image_175 = EXCLUDED.image_175,
          image_full = EXCLUDED.image_full,
          options = EXCLUDED.options,
          tags = EXCLUDED.tags
        RETURNING
          xmax = 0 as inserted
      `, values);
    }, `Error inserting batch ${i} to ${i + batch.length}`);

    outputProgress({
      status: "running",
      operation: "Products import",
      message: `Imported ${i + batch.length} of ${prodData.length} products`,
      current: i + batch.length,
      total: prodData.length,
              elapsed: formatElapsedTime(startTime),
        remaining: estimateRemaining(startTime, i + batch.length, prodData.length),
      rate: calculateRate(startTime, i + batch.length)
    });
  }

  outputProgress({
    status: "running",
    operation: "Products import",
    message: "Finished materializing calculations"
  });
  
  // Add step to identify which products actually need updating
  outputProgress({
    status: "running",
    operation: "Products import",
    message: "Identifying changed products"
  });
  
  // Mark products that haven't changed as needs_update = false
  await localConnection.query(`
    UPDATE temp_products t
    SET needs_update = FALSE
    FROM products p
    WHERE t.pid = p.pid
      AND t.title IS NOT DISTINCT FROM p.title
      AND t.description IS NOT DISTINCT FROM p.description
      AND t.sku IS NOT DISTINCT FROM p.sku
      AND t.stock_quantity = p.stock_quantity
      AND t.price = p.price
      AND t.regular_price = p.regular_price
      AND t.cost_price IS NOT DISTINCT FROM p.cost_price
      AND t.vendor IS NOT DISTINCT FROM p.vendor
      AND t.brand IS NOT DISTINCT FROM p.brand
      AND t.visible = p.visible
      AND t.replenishable = p.replenishable
      AND t.barcode IS NOT DISTINCT FROM p.barcode
      AND t.updated_at IS NOT DISTINCT FROM p.updated_at
      AND t.total_sold IS NOT DISTINCT FROM p.total_sold
      AND t.date_online IS NOT DISTINCT FROM p.date_online
      AND t.shop_score IS NOT DISTINCT FROM p.shop_score
      AND t.categories IS NOT DISTINCT FROM p.categories
  `);
  
  // Get count of products that need updating
  const [countResult] = await localConnection.query(`
    SELECT 
      COUNT(*) FILTER (WHERE needs_update = true) as update_count,
      COUNT(*) FILTER (WHERE needs_update = false) as skip_count,
      COUNT(*) as total_count
    FROM temp_products
  `);
  
  outputProgress({
    status: "running",
    operation: "Products import",
    message: `Found ${countResult.rows[0].update_count} products that need updating, ${countResult.rows[0].skip_count} unchanged`
  });
  
  // Return the total products processed
  return {
    totalProcessed: prodData.length,
    needsUpdate: parseInt(countResult.rows[0].update_count),
    skipped: parseInt(countResult.rows[0].skip_count)
  };
}

async function importProducts(prodConnection, localConnection, incrementalUpdate = true) {
  const startTime = Date.now();
  let lastSyncTime = '1970-01-01';

  try {
    // Get last sync time if doing incremental update
    if (incrementalUpdate) {
      const [syncResult] = await localConnection.query(
        "SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'products'"
      );
      if (syncResult.rows.length > 0) {
        lastSyncTime = syncResult.rows[0].last_sync_timestamp;
      }
    }

    // Capture the next watermark from MySQL's own clock BEFORE querying any data.
    // Rows modified while the import runs stay above this watermark for the next
    // incremental run (overlap re-imports are harmless upserts).
    const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');

    // Start a transaction to ensure temporary tables persist
    await localConnection.beginTransaction();

    try {
      // Setup temporary tables
      await setupTemporaryTables(localConnection);

      // Adjust sync time for mysql2 driver timezone vs MySQL server timezone mismatch
      const mysqlSyncTime = prodConnection.adjustDateForMySQL
        ? prodConnection.adjustDateForMySQL(lastSyncTime)
        : lastSyncTime;

      // Materialize calculations into temp table
      const materializeResult = await materializeCalculations(prodConnection, localConnection, incrementalUpdate, mysqlSyncTime, startTime);

      // Get the list of products that need updating
      const [products] = await localConnection.query(`
        SELECT 
          t.pid,
          t.title,
          t.description,
          t.sku,
          t.stock_quantity,
          t.preorder_count,
          t.notions_inv_count,
          t.price,
          t.regular_price,
          t.cost_price,
          t.vendor,
          t.vendor_reference,
          t.notions_reference,
          t.brand,
          t.line,
          t.subline,
          t.artist,
          t.categories,
          t.created_at,
          t.date_online,
          t.first_received,
          t.barcode,
          t.harmonized_tariff_code,
          t.updated_at,
          t.visible,
          t.managing_stock,
          t.replenishable,
          t.permalink,
          t.moq,
          t.rating,
          t.reviews,
          t.weight,
          t.length,
          t.width,
          t.height,
          t.country_of_origin,
          t.location,
          t.total_sold,
          t.baskets,
          t.notifies,
          t.date_last_sold,
          t.shop_score,
          t.primary_iid,
          t.image,
          t.image_175,
          t.image_full,
          t.options,
          t.tags
        FROM temp_products t
        WHERE t.needs_update = true
      `);

      // Process products in batches
      let recordsAdded = 0;
      let recordsUpdated = 0;

      for (let i = 0; i < products.rows.length; i += BATCH_SIZE) {
        const batch = products.rows.slice(i, i + BATCH_SIZE);
        
        const placeholders = batch.map((_, idx) => {
          const base = idx * 48; // 48 columns (no primary_iid in this INSERT)
          return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
        }).join(',');

        const values = batch.flatMap(row => {
          const imageUrls = getImageUrls(row.pid, row.primary_iid || 1);
          return [
            row.pid,
            row.title,
            row.description,
            row.sku || '',
            row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
            row.preorder_count,
            row.notions_inv_count,
            row.price,
            row.regular_price,
            row.cost_price,
            row.vendor,
            row.vendor_reference,
            row.notions_reference,
            row.brand,
            row.line,
            row.subline,
            row.artist,
            row.categories,
            validateDate(row.created_at),
            validateDate(row.date_online),
            validateDate(row.first_received),
            row.barcode,
            row.harmonized_tariff_code,
            validateDate(row.updated_at),
            row.visible,
            row.managing_stock,
            row.replenishable,
            row.permalink,
            row.moq,
            1,
            row.rating,
            row.reviews,
            row.weight,
            row.length,
            row.width,
            row.height,
            row.country_of_origin,
            row.location,
            row.total_sold,
            row.baskets,
            row.notifies,
            validateDate(row.date_last_sold),
            Number(row.shop_score) || 0,
            imageUrls.image,
            imageUrls.image_175,
            imageUrls.image_full,
            row.options,
            row.tags
          ];
        });

        const [result] = await localConnection.query(`
          WITH upserted AS (
            INSERT INTO products (
              pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
              price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
              brand, line, subline, artist, categories, created_at, date_online, first_received,
              barcode, harmonized_tariff_code, updated_at, visible,
              managing_stock, replenishable, permalink, moq, uom, rating, reviews,
              weight, length, width, height, country_of_origin, location, total_sold,
              baskets, notifies, date_last_sold, shop_score, image, image_175, image_full, options, tags
            )
            VALUES ${placeholders}
            ON CONFLICT (pid) DO UPDATE SET
              title = EXCLUDED.title,
              description = EXCLUDED.description,
              sku = EXCLUDED.sku,
              stock_quantity = EXCLUDED.stock_quantity,
              preorder_count = EXCLUDED.preorder_count,
              notions_inv_count = EXCLUDED.notions_inv_count,
              price = EXCLUDED.price,
              regular_price = EXCLUDED.regular_price,
              cost_price = EXCLUDED.cost_price,
              vendor = EXCLUDED.vendor,
              vendor_reference = EXCLUDED.vendor_reference,
              notions_reference = EXCLUDED.notions_reference,
              brand = EXCLUDED.brand,
              line = EXCLUDED.line,
              subline = EXCLUDED.subline,
              artist = EXCLUDED.artist,
              created_at = EXCLUDED.created_at,
              date_online = EXCLUDED.date_online,
              first_received = EXCLUDED.first_received,
              barcode = EXCLUDED.barcode,
              harmonized_tariff_code = EXCLUDED.harmonized_tariff_code,
              updated_at = EXCLUDED.updated_at,
              visible = EXCLUDED.visible,
              managing_stock = EXCLUDED.managing_stock,
              replenishable = EXCLUDED.replenishable,
              permalink = EXCLUDED.permalink,
              moq = EXCLUDED.moq,
              uom = EXCLUDED.uom,
              rating = EXCLUDED.rating,
              reviews = EXCLUDED.reviews,
              weight = EXCLUDED.weight,
              length = EXCLUDED.length,
              width = EXCLUDED.width,
              height = EXCLUDED.height,
              country_of_origin = EXCLUDED.country_of_origin,
              location = EXCLUDED.location,
              total_sold = EXCLUDED.total_sold,
              baskets = EXCLUDED.baskets,
              notifies = EXCLUDED.notifies,
              date_last_sold = EXCLUDED.date_last_sold,
              shop_score = EXCLUDED.shop_score,
              image = EXCLUDED.image,
              image_175 = EXCLUDED.image_175,
              image_full = EXCLUDED.image_full,
              options = EXCLUDED.options,
              tags = EXCLUDED.tags
            RETURNING
              xmax = 0 as inserted
          )
          SELECT
            COUNT(*) FILTER (WHERE inserted) as inserted,
            COUNT(*) FILTER (WHERE NOT inserted) as updated
          FROM upserted
        `, values);

        recordsAdded += parseInt(result.rows[0].inserted, 10) || 0;
        recordsUpdated += parseInt(result.rows[0].updated, 10) || 0;

        // Process category relationships in batches
        const allCategories = [];
        for (const row of batch) {
          if (row.categories) {
            const categoryIds = row.categories.split(',').filter(id => id && id.trim());
            if (categoryIds.length > 0) {
              categoryIds.forEach(catId => {
                allCategories.push([row.pid, parseInt(catId.trim(), 10)]);
              });
            }
          }
        }

        // If we have categories to process
        if (allCategories.length > 0) {
          // First get all products in this batch
          const productIds = batch.map(p => p.pid);
          
          // Delete all existing relationships for products in this batch
          await localConnection.query(
            'DELETE FROM product_categories WHERE pid = ANY($1)',
            [productIds]
          );
          
          // Insert all new relationships in one batch
          const catPlaceholders = allCategories.map((_, idx) => 
            `($${idx * 2 + 1}, $${idx * 2 + 2})`
          ).join(',');
          
          const catValues = allCategories.flat();
          
          await localConnection.query(`
            INSERT INTO product_categories (pid, cat_id)
            VALUES ${catPlaceholders}
            ON CONFLICT (pid, cat_id) DO NOTHING
          `, catValues);
        }

        outputProgress({
          status: "running",
          operation: "Products import",
          message: `Processing products: ${i + batch.length} of ${products.rows.length}`,
          current: i + batch.length,
          total: products.rows.length,
                  elapsed: formatElapsedTime(startTime),
        remaining: estimateRemaining(startTime, i + batch.length, products.rows.length),
          rate: calculateRate(startTime, i + batch.length)
        });
      }

      // Cleanup temporary tables
      await cleanupTemporaryTables(localConnection);

      // Sync supplier-quoted cost fields (notions_cost_each / supplier_cost_each).
      // These feed the Create-PO page so the displayed cost matches what the
      // legacy PHP backend will stamp onto the PO line item.
      await syncSupplierCosts(prodConnection, localConnection);

      // Sync category assignments for ALL products. product_category_index has no
      // stamp column, so category-only changes never bump any of the incremental
      // WHERE timestamps — without this pass PG categories go permanently stale.
      await syncProductCategories(prodConnection, localConnection);

      // Commit the transaction
      await localConnection.commit();

      // Update sync status with the watermark captured from MySQL BEFORE the
      // source queries ran (see sourceNow above).
      await localConnection.query(`
        INSERT INTO sync_status (table_name, last_sync_timestamp)
        VALUES ('products', $1)
        ON CONFLICT (table_name) DO UPDATE SET
          last_sync_timestamp = $1
      `, [sourceNow]);

      return {
        status: 'complete',
        recordsAdded,
        recordsUpdated,
        totalRecords: products.rows.length,
        totalProcessed: materializeResult.totalProcessed,
        duration: formatElapsedTime(startTime),
        needsUpdate: materializeResult.needsUpdate,
        skippedUnchanged: materializeResult.skipped
      };
    } catch (error) {
      // Rollback on error
      await localConnection.rollback();
      throw error;
    }
  } catch (error) {
    console.error('Error in importProducts:', error);
    throw error;
  }
}

// Bulk-sync supplier_item_data.notions_cost_each / supplier_cost_each into
// products.{notions_cost_each, supplier_cost_each}. These mirror the supplier-
// quoted "cost each" values the legacy PHP backend writes onto a PO when
// _product_add() runs (see po.class.php:189-209). Kept as a separate, idempotent
// pass so the main 49-column import paths don't need to know about it.
async function syncSupplierCosts(prodConnection, localConnection) {
  outputProgress({
    status: "running",
    operation: "Products import",
    message: "Syncing supplier costs from supplier_item_data"
  });

  const [rows] = await prodConnection.query(`
    SELECT pid, notions_cost_each, supplier_cost_each
    FROM supplier_item_data
  `);

  if (!rows || rows.length === 0) {
    return { updated: 0 };
  }

  // Stage into a temp table, then UPDATE in a single SQL statement.
  await localConnection.query(`
    CREATE TEMP TABLE temp_supplier_costs (
      pid BIGINT PRIMARY KEY,
      notions_cost_each NUMERIC(10,3),
      supplier_cost_each NUMERIC(10,3)
    ) ON COMMIT DROP
  `);

  const CHUNK = 5000;
  for (let i = 0; i < rows.length; i += CHUNK) {
    const batch = rows.slice(i, i + CHUNK);
    const placeholders = batch
      .map((_, idx) => `($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`)
      .join(',');
    const values = batch.flatMap(r => [
      r.pid,
      r.notions_cost_each,
      r.supplier_cost_each
    ]);
    await localConnection.query(
      `INSERT INTO temp_supplier_costs (pid, notions_cost_each, supplier_cost_each)
       VALUES ${placeholders}
       ON CONFLICT (pid) DO NOTHING`,
      values
    );
  }

  const [result] = await localConnection.query(`
    UPDATE products p
    SET notions_cost_each = t.notions_cost_each,
        supplier_cost_each = t.supplier_cost_each
    FROM temp_supplier_costs t
    WHERE p.pid = t.pid
      AND (p.notions_cost_each IS DISTINCT FROM t.notions_cost_each
        OR p.supplier_cost_each IS DISTINCT FROM t.supplier_cost_each)
  `);

  const updated = result.rowCount || 0;
  outputProgress({
    status: "running",
    operation: "Products import",
    message: `Supplier costs synced for ${updated} products`
  });

  return { updated };
}

// Full category-assignment sweep. The incremental product import keys on
// p.stamp / ci.stamp / price / b2b dates — none of which change when a product
// is recategorized in product_category_index (the table has no stamp column).
// This pass compares the canonical GROUP_CONCAT representation against
// products.categories and rewrites product_categories only for changed pids.
// Must run inside the caller's transaction (uses ON COMMIT DROP temp table).
async function syncProductCategories(prodConnection, localConnection) {
  outputProgress({
    status: "running",
    operation: "Products import",
    message: "Syncing category assignments"
  });

  // Same expression as the main import query so representations compare equal
  // (GROUP_CONCAT(DISTINCT int) returns values numerically sorted).
  const [rows] = await prodConnection.query(`
    SELECT
      p.pid,
      GROUP_CONCAT(DISTINCT CASE
        WHEN pc.cat_id IS NOT NULL
        AND pc.type IN (10, 20, 11, 21, 12, 13)
        AND pci.cat_id NOT IN (16, 17)
        THEN pci.cat_id
      END) as category_ids
    FROM products p
    LEFT JOIN product_category_index pci ON p.pid = pci.pid
    LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
    GROUP BY p.pid
  `);

  if (!rows || rows.length === 0) {
    return { updated: 0 };
  }

  await localConnection.query(`
    CREATE TEMP TABLE temp_category_sync (
      pid BIGINT PRIMARY KEY,
      categories TEXT
    ) ON COMMIT DROP
  `);

  const CHUNK = 5000;
  for (let i = 0; i < rows.length; i += CHUNK) {
    const batch = rows.slice(i, i + CHUNK);
    const pids = batch.map(r => r.pid);
    const cats = batch.map(r => r.category_ids);
    await localConnection.query(
      `INSERT INTO temp_category_sync (pid, categories)
       SELECT * FROM UNNEST($1::bigint[], $2::text[])
       ON CONFLICT (pid) DO NOTHING`,
      [pids, cats]
    );
  }

  // Which existing products actually changed?
  const [changed] = await localConnection.query(`
    SELECT t.pid, t.categories
    FROM temp_category_sync t
    JOIN products p ON p.pid = t.pid
    WHERE t.categories IS DISTINCT FROM p.categories
  `);

  if (changed.rows.length === 0) {
    return { updated: 0 };
  }

  await localConnection.query(`
    UPDATE products p
    SET categories = t.categories
    FROM temp_category_sync t
    WHERE p.pid = t.pid
      AND t.categories IS DISTINCT FROM p.categories
  `);

  // Rewrite the relationship rows for changed products only
  const REL_CHUNK = 1000;
  for (let i = 0; i < changed.rows.length; i += REL_CHUNK) {
    const batch = changed.rows.slice(i, i + REL_CHUNK);
    const pids = batch.map(r => r.pid);

    await localConnection.query(
      'DELETE FROM product_categories WHERE pid = ANY($1)',
      [pids]
    );

    const relPids = [];
    const relCats = [];
    for (const row of batch) {
      if (!row.categories) continue;
      for (const catId of row.categories.split(',')) {
        if (catId && catId.trim()) {
          relPids.push(row.pid);
          relCats.push(parseInt(catId.trim(), 10));
        }
      }
    }
    if (relPids.length > 0) {
      await localConnection.query(`
        INSERT INTO product_categories (pid, cat_id)
        SELECT * FROM UNNEST($1::bigint[], $2::int[])
        ON CONFLICT (pid, cat_id) DO NOTHING
      `, [relPids, relCats]);
    }
  }

  outputProgress({
    status: "running",
    operation: "Products import",
    message: `Category assignments updated for ${changed.rows.length} products`
  });

  return { updated: changed.rows.length };
}

module.exports = {
  importProducts,
  importMissingProducts,
  setupTemporaryTables,
  cleanupTemporaryTables,
  materializeCalculations,
  syncSupplierCosts,
  syncProductCategories
};