const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics-new/utils/progress');

/**
 * Imports orders from a production MySQL database to a local PostgreSQL database.
 * It can run in two modes:
 *   1. Incremental update mode (default): Only fetch orders that have changed since the last sync time.
 *   2. Full update mode: Fetch all eligible orders within the last 5 years regardless of timestamp.
 *
 * @param {object} prodConnection - A MySQL connection to production DB (MySQL 5.7).
 * @param {object} localConnection - A MySQL connection to local DB (MySQL 8.0).
 * @param {boolean} incrementalUpdate - Set to false for a full sync; true for incremental.
 *
 * @returns {object} Information about the sync operation.
 */
async function importOrders(prodConnection, localConnection, incrementalUpdate = true) {
  const startTime = Date.now();
  const skippedOrders = new Set();
  const missingProducts = new Set();

  // Map order status codes to text values (consistent with PO status mapping in purchase-orders.js)
  const orderStatusMap = {
    0: 'created',
    10: 'unfinished',
    15: 'canceled',
    16: 'combined',
    20: 'placed',
    22: 'placed_incomplete',
    30: 'canceled',
    40: 'awaiting_payment',
    45: 'payment_pending',
    50: 'awaiting_products',
    55: 'shipping_later',
    56: 'shipping_together',
    60: 'ready',
    61: 'flagged',
    62: 'fix_before_pick',
    65: 'manual_picking',
    67: 'remote_send',
    70: 'in_pt',
    80: 'picked',
    90: 'awaiting_shipment',
    91: 'remote_wait',
    92: 'awaiting_pickup',
    93: 'fix_before_ship',
    95: 'shipped_confirmed',
    100: 'shipped'
  };
  let recordsAdded = 0;
  let recordsUpdated = 0;
  let processedCount = 0;
  let importedCount = 0;
  let totalOrderItems = 0;
  let totalUniqueOrders = 0;
  let cumulativeProcessedOrders = 0;

  try {
    // Get last sync info - NOT in a transaction anymore
    const [syncInfo] = await localConnection.query(
      "SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
    );
    const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
    // Adjust for mysql2 driver timezone vs MySQL server timezone mismatch
    const mysqlSyncTime = prodConnection.adjustDateForMySQL
      ? prodConnection.adjustDateForMySQL(lastSyncTime)
      : lastSyncTime;

    console.log('Orders: Using last sync time:', lastSyncTime, '(adjusted:', mysqlSyncTime, ')');

    // Capture the next watermark from MySQL's own clock BEFORE querying any data.
    // Rows modified while the import runs stay above this watermark for the next
    // incremental run (overlap re-imports are harmless upserts); writing NOW()
    // after the import finishes would permanently skip them.
    const [[{ source_now: sourceNow }]] = await prodConnection.query('SELECT NOW() as source_now');

    // First get count of order items - Keep MySQL compatible for production
    const [[{ total }]] = await prodConnection.query(`
      SELECT COUNT(*) as total
      FROM order_items oi
      JOIN _order o ON oi.order_id = o.order_id
      WHERE o.order_status >= 15
        AND o.date_placed >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
        AND o.date_placed IS NOT NULL
        ${incrementalUpdate ? `
          AND (
            o.stamp > ?
            OR oi.stamp > ?
            OR EXISTS (
              SELECT 1 FROM order_tax_info oti
              JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
              WHERE oti.order_id = o.order_id
              AND otip.pid = oi.prod_pid
              AND oti.stamp > ?
            )
          )
        ` : ''}
    `, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : []);

    totalOrderItems = total;
    console.log('Orders: Found changes:', totalOrderItems);

    // Get order items - Keep MySQL compatible for production
    console.log('Orders: Starting MySQL query...');
    const [orderItems] = await prodConnection.query(`
      SELECT 
        oi.order_id,
        oi.prod_pid,
        COALESCE(NULLIF(TRIM(oi.prod_itemnumber), ''), 'NO-SKU') as SKU,
        oi.prod_price as price,
        oi.qty_ordered as quantity,
        oi.stamp as last_modified
      FROM order_items oi
      JOIN _order o ON oi.order_id = o.order_id
      WHERE o.order_status >= 15
        AND o.date_placed >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
        AND o.date_placed IS NOT NULL
        ${incrementalUpdate ? `
          AND (
            o.stamp > ?
            OR oi.stamp > ?
            OR EXISTS (
              SELECT 1 FROM order_tax_info oti
              JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
              WHERE oti.order_id = o.order_id
              AND otip.pid = oi.prod_pid
              AND oti.stamp > ?
            )
          )
        ` : ''}
    `, incrementalUpdate ? [mysqlSyncTime, mysqlSyncTime, mysqlSyncTime] : []);

    console.log('Orders: Found', orderItems.length, 'order items to process');

    // Create tables in PostgreSQL for data processing
    // Start a transaction just for creating the temp tables
    await localConnection.beginTransaction();
    try {
      await localConnection.query(`
        DROP TABLE IF EXISTS temp_order_items;
        DROP TABLE IF EXISTS temp_order_meta;
        DROP TABLE IF EXISTS temp_order_taxes;
        DROP TABLE IF EXISTS temp_order_costs;
        DROP TABLE IF EXISTS temp_item_discounts;

        CREATE TEMP TABLE temp_order_items (
          order_id INTEGER NOT NULL,
          pid INTEGER NOT NULL,
          sku TEXT NOT NULL,
          price NUMERIC(14, 4) NOT NULL,
          quantity INTEGER NOT NULL,
          PRIMARY KEY (order_id, pid)
        );

        CREATE TEMP TABLE temp_order_meta (
          order_id INTEGER NOT NULL,
          date TIMESTAMP WITH TIME ZONE NOT NULL,
          customer TEXT NOT NULL,
          customer_name TEXT NOT NULL,
          status TEXT,
          canceled BOOLEAN,
          summary_discount NUMERIC(14, 4) DEFAULT 0.0000,
          summary_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
          summary_discount_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
          PRIMARY KEY (order_id)
        );

        CREATE TEMP TABLE temp_item_discounts (
          order_id INTEGER NOT NULL,
          pid INTEGER NOT NULL,
          discount_id INTEGER NOT NULL,
          amount NUMERIC(14, 4) NOT NULL,
          PRIMARY KEY (order_id, pid, discount_id)
        );

        CREATE TEMP TABLE temp_order_taxes (
          order_id INTEGER NOT NULL,
          pid INTEGER NOT NULL,
          tax NUMERIC(14, 4) NOT NULL,
          PRIMARY KEY (order_id, pid)
        );

        CREATE TEMP TABLE temp_order_costs (
          order_id INTEGER NOT NULL,
          pid INTEGER NOT NULL,
          costeach NUMERIC(14, 4) DEFAULT 0.0000,
          PRIMARY KEY (order_id, pid)
        );

        CREATE INDEX idx_temp_order_items_pid ON temp_order_items(pid);
        CREATE INDEX idx_temp_order_meta_order_id ON temp_order_meta(order_id);
        CREATE INDEX idx_temp_order_taxes_order_pid ON temp_order_taxes(order_id, pid);
        CREATE INDEX idx_temp_order_costs_order_pid ON temp_order_costs(order_id, pid);
        CREATE INDEX idx_temp_item_discounts_order_pid ON temp_item_discounts(order_id, pid);
        CREATE INDEX idx_temp_item_discounts_discount_id ON temp_item_discounts(discount_id);
      `);
      await localConnection.commit();
    } catch (error) {
      await localConnection.rollback();
      throw error;
    }

    // Insert order items in batches - each batch gets its own transaction
    for (let i = 0; i < orderItems.length; i += 5000) {
      await localConnection.beginTransaction();
      try {
        const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
        const placeholders = batch.map((_, idx) =>
          `($${idx * 5 + 1}, $${idx * 5 + 2}, $${idx * 5 + 3}, $${idx * 5 + 4}, $${idx * 5 + 5})`
        ).join(",");
        const values = batch.flatMap(item => [
          item.order_id, item.prod_pid, item.SKU, item.price, item.quantity
        ]);

        await localConnection.query(`
          INSERT INTO temp_order_items (order_id, pid, sku, price, quantity)
          VALUES ${placeholders}
          ON CONFLICT (order_id, pid) DO UPDATE SET
            sku = EXCLUDED.sku,
            price = EXCLUDED.price,
            quantity = EXCLUDED.quantity
        `, values);

        await localConnection.commit();
        
        processedCount = i + batch.length;
        outputProgress({
          status: "running",
          operation: "Orders import",
          message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
          current: processedCount,
          total: totalOrderItems,
                      elapsed: formatElapsedTime(startTime),
            remaining: estimateRemaining(startTime, processedCount, totalOrderItems),
          rate: calculateRate(startTime, processedCount)
        });
      } catch (error) {
        await localConnection.rollback();
        throw error;
      }
    }

    // Get unique order IDs
    const orderIds = [...new Set(orderItems.map(item => item.order_id))];
    totalUniqueOrders = orderIds.length;
    console.log('Orders: Processing', totalUniqueOrders, 'unique orders');

    // Reset processed count for order processing phase
    processedCount = 0;

    // Process metadata, discounts, taxes, and costs in parallel
    const METADATA_BATCH_SIZE = 2000;
    const PG_BATCH_SIZE = 200;
    
    // Add a helper function for title case conversion
    function toTitleCase(str) {
      if (!str) return '';
      return str.toLowerCase().split(' ').map(word => {
        return word.charAt(0).toUpperCase() + word.slice(1);
      }).join(' ');
    }

    const processMetadataBatch = async (batchIds) => {
      const [orders] = await prodConnection.query(`
        SELECT 
          o.order_id,
          o.date_placed as date,
          o.order_cid as customer,
          CONCAT(COALESCE(u.firstname, ''), ' ', COALESCE(u.lastname, '')) as customer_name,
          o.order_status as status,
          CASE WHEN o.date_cancelled != '0000-00-00 00:00:00' THEN 1 ELSE 0 END as canceled,
          o.summary_discount,
          o.summary_subtotal,
          o.summary_discount_subtotal
        FROM _order o
        LEFT JOIN users u ON o.order_cid = u.cid
        WHERE o.order_id IN (?)
      `, [batchIds]);

      // Process in sub-batches for PostgreSQL
      await localConnection.beginTransaction();
      try {
        for (let j = 0; j < orders.length; j += PG_BATCH_SIZE) {
          const subBatch = orders.slice(j, j + PG_BATCH_SIZE);
          if (subBatch.length === 0) continue;

          const placeholders = subBatch.map((_, idx) => 
            `($${idx * 9 + 1}, $${idx * 9 + 2}, $${idx * 9 + 3}, $${idx * 9 + 4}, $${idx * 9 + 5}, $${idx * 9 + 6}, $${idx * 9 + 7}, $${idx * 9 + 8}, $${idx * 9 + 9})`
          ).join(",");
          
          const values = subBatch.flatMap(order => [
            order.order_id,
            new Date(order.date), // Convert to TIMESTAMP WITH TIME ZONE
            order.customer,
            toTitleCase(order.customer_name) || '',
            orderStatusMap[order.status] || order.status.toString(), // Map numeric status to text
            order.canceled,
            order.summary_discount || 0,
            order.summary_subtotal || 0,
            order.summary_discount_subtotal || 0
          ]);

          await localConnection.query(`
            INSERT INTO temp_order_meta (
              order_id, date, customer, customer_name, status, canceled,
              summary_discount, summary_subtotal, summary_discount_subtotal
            )
            VALUES ${placeholders}
            ON CONFLICT (order_id) DO UPDATE SET
              date = EXCLUDED.date,
              customer = EXCLUDED.customer,
              customer_name = EXCLUDED.customer_name,
              status = EXCLUDED.status,
              canceled = EXCLUDED.canceled,
              summary_discount = EXCLUDED.summary_discount,
              summary_subtotal = EXCLUDED.summary_subtotal,
              summary_discount_subtotal = EXCLUDED.summary_discount_subtotal
          `, values);
        }
        await localConnection.commit();
      } catch (error) {
        await localConnection.rollback();
        throw error;
      }
    };

    const processDiscountsBatch = async (batchIds) => {
      // Load item-level discount records. Only which = 2 rows are real per-item
      // discount amounts; which = 1 rows store the price of free promo-added
      // items and which = 3 rows are usage records (neither is a discount).
      // These amounts are NOT included in summary_discount_subtotal, so they
      // must be added on top of the prorated subtotal discount unconditionally.
      const [discounts] = await prodConnection.query(`
        SELECT order_id, pid, discount_id, amount
        FROM order_discount_items
        WHERE order_id IN (?) AND which = 2
      `, [batchIds]);

      if (discounts.length === 0) return;

      // Process in memory to handle potential duplicates
      const discountMap = new Map();
      for (const d of discounts) {
        const key = `${d.order_id}-${d.pid}-${d.discount_id}`;
        discountMap.set(key, d);
      }

      const uniqueDiscounts = Array.from(discountMap.values());

      await localConnection.beginTransaction();
      try {
        for (let j = 0; j < uniqueDiscounts.length; j += PG_BATCH_SIZE) {
          const subBatch = uniqueDiscounts.slice(j, j + PG_BATCH_SIZE);
          if (subBatch.length === 0) continue;

          const placeholders = subBatch.map((_, idx) => 
            `($${idx * 4 + 1}, $${idx * 4 + 2}, $${idx * 4 + 3}, $${idx * 4 + 4})`
          ).join(",");
          
          const values = subBatch.flatMap(d => [
            d.order_id,
            d.pid,
            d.discount_id,
            d.amount || 0
          ]);

          await localConnection.query(`
            INSERT INTO temp_item_discounts (order_id, pid, discount_id, amount)
            VALUES ${placeholders}
            ON CONFLICT (order_id, pid, discount_id) DO UPDATE SET
              amount = EXCLUDED.amount
          `, values);
        }

        await localConnection.commit();
      } catch (error) {
        await localConnection.rollback();
        throw error;
      }
    };

    const processTaxesBatch = async (batchIds) => {
      // Optimized tax query to avoid subquery
      const [taxes] = await prodConnection.query(`
        SELECT oti.order_id, otip.pid, otip.item_taxes_to_collect as tax
        FROM (
          SELECT order_id, MAX(taxinfo_id) as latest_taxinfo_id
          FROM order_tax_info
          WHERE order_id IN (?)
          GROUP BY order_id
        ) latest_info
        JOIN order_tax_info oti ON oti.order_id = latest_info.order_id 
          AND oti.taxinfo_id = latest_info.latest_taxinfo_id
        JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
      `, [batchIds]);

      if (taxes.length === 0) return;

      await localConnection.beginTransaction();
      try {
        for (let j = 0; j < taxes.length; j += PG_BATCH_SIZE) {
          const subBatch = taxes.slice(j, j + PG_BATCH_SIZE);
          if (subBatch.length === 0) continue;

          const placeholders = subBatch.map((_, idx) => 
            `($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
          ).join(",");
          
          const values = subBatch.flatMap(t => [
            t.order_id,
            t.pid,
            t.tax || 0
          ]);

          await localConnection.query(`
            INSERT INTO temp_order_taxes (order_id, pid, tax)
            VALUES ${placeholders}
            ON CONFLICT (order_id, pid) DO UPDATE SET
              tax = EXCLUDED.tax
          `, values);
        }
        await localConnection.commit();
      } catch (error) {
        await localConnection.rollback();
        throw error;
      }
    };

    const processCostsBatch = async (batchIds) => {
      // Modified query to ensure one row per order_id/pid by using a subquery
      const [costs] = await prodConnection.query(`
        SELECT 
          oc.orderid as order_id,
          oc.pid,
          oc.costeach
        FROM order_costs oc
        INNER JOIN (
          SELECT 
            orderid,
            pid, 
            MAX(id) as max_id
          FROM order_costs
          WHERE orderid IN (?) 
            AND pending = 0
          GROUP BY orderid, pid
        ) latest ON oc.orderid = latest.orderid AND oc.pid = latest.pid AND oc.id = latest.max_id
      `, [batchIds]);

      if (costs.length === 0) return;

      await localConnection.beginTransaction();
      try {
        for (let j = 0; j < costs.length; j += PG_BATCH_SIZE) {
          const subBatch = costs.slice(j, j + PG_BATCH_SIZE);
          if (subBatch.length === 0) continue;

          const placeholders = subBatch.map((_, idx) => 
            `($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
          ).join(",");
          
          const values = subBatch.flatMap(c => [
            c.order_id,
            c.pid,
            c.costeach || 0
          ]);

          await localConnection.query(`
            INSERT INTO temp_order_costs (order_id, pid, costeach)
            VALUES ${placeholders}
            ON CONFLICT (order_id, pid) DO UPDATE SET
              costeach = EXCLUDED.costeach
          `, values);
        }
        await localConnection.commit();
      } catch (error) {
        await localConnection.rollback();
        throw error;
      }
    };

    // Process all data types for each batch
    // Note: these run sequentially because they share a single PG connection
    // and each manages its own transaction
    for (let i = 0; i < orderIds.length; i += METADATA_BATCH_SIZE) {
      const batchIds = orderIds.slice(i, i + METADATA_BATCH_SIZE);

      await processMetadataBatch(batchIds);
      await processDiscountsBatch(batchIds);
      await processTaxesBatch(batchIds);
      await processCostsBatch(batchIds);

      processedCount = i + batchIds.length;
      outputProgress({
        status: "running",
        operation: "Orders import",
        message: `Loading order data: ${processedCount} of ${totalUniqueOrders}`,
        current: processedCount,
        total: totalUniqueOrders,
                        elapsed: formatElapsedTime(startTime),
                remaining: estimateRemaining(startTime, processedCount, totalUniqueOrders),
        rate: calculateRate(startTime, processedCount)
      });
    }

    // Pre-check all products and preload cost_price into a temp table
    // This avoids joining public.products in every sub-batch query (was causing 2x slowdown)
    const allOrderPids = [...new Set(orderItems.map(item => item.prod_pid))];
    console.log('Orders: Checking', allOrderPids.length, 'unique products');

    const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
      "SELECT pid, cost_price FROM products WHERE pid = ANY($1::bigint[])",
      [allOrderPids]
    ) : [{ rows: [] }];

    const existingPids = new Set(existingProducts.rows.map(p => p.pid));

    // Create temp table with product cost_price for fast lookup in sub-batch queries
    await localConnection.query(`
      DROP TABLE IF EXISTS temp_product_costs;
      CREATE TEMP TABLE temp_product_costs (
        pid BIGINT PRIMARY KEY,
        cost_price NUMERIC(14, 4)
      )
    `);
    if (existingProducts.rows.length > 0) {
      const costPids = existingProducts.rows.filter(p => p.cost_price != null).map(p => p.pid);
      const costPrices = existingProducts.rows.filter(p => p.cost_price != null).map(p => p.cost_price);
      if (costPids.length > 0) {
        await localConnection.query(`
          INSERT INTO temp_product_costs (pid, cost_price)
          SELECT * FROM UNNEST($1::bigint[], $2::numeric[])
        `, [costPids, costPrices]);
      }
    }
    
    // Process in smaller batches
    for (let i = 0; i < orderIds.length; i += 2000) {  // Increased from 1000 to 2000
      const batchIds = orderIds.slice(i, i + 2000);

      // Get combined data for this batch in sub-batches
      const PG_BATCH_SIZE = 200; // Increased from 100 to 200
      for (let j = 0; j < batchIds.length; j += PG_BATCH_SIZE) {
        const subBatchIds = batchIds.slice(j, j + PG_BATCH_SIZE);
        
        // Start a transaction for this sub-batch
        await localConnection.beginTransaction();
        try {
          const [orders] = await localConnection.query(`
            WITH order_totals AS (
              SELECT
                oi.order_id,
                oi.pid,
                -- Item-level promo discounts (which = 2 rows). These live outside
                -- summary_discount_subtotal, so they are summed unconditionally.
                SUM(COALESCE(id.amount, 0)) as promo_discount_sum,
                COALESCE(ot.tax, 0) as total_tax,
                COALESCE(oc.costeach, pc.cost_price, oi.price * 0.5) as costeach
              FROM temp_order_items oi
              LEFT JOIN temp_item_discounts id ON oi.order_id = id.order_id AND oi.pid = id.pid
              LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
              LEFT JOIN temp_order_costs oc ON oi.order_id = oc.order_id AND oi.pid = oc.pid
              LEFT JOIN temp_product_costs pc ON oi.pid = pc.pid
              WHERE oi.order_id = ANY($1)
              GROUP BY oi.order_id, oi.pid, ot.tax, oc.costeach, pc.cost_price
            )
            SELECT
              oi.order_id as order_number,
              oi.pid::bigint as pid,
              oi.sku,
              om.date,
              oi.price,
              oi.quantity,
              -- Discount = prorated order-level subtotal discount + item-level promo
              -- discounts, clamped so a sale line can never be discounted below free.
              (
                CASE WHEN oi.quantity > 0 THEN
                  LEAST(
                    (
                      CASE
                        WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
                          COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
                        ELSE 0
                      END
                      + COALESCE(ot.promo_discount_sum, 0)
                    ),
                    oi.price * oi.quantity
                  )
                ELSE
                  (
                    CASE
                      WHEN om.summary_discount_subtotal > 0 AND om.summary_subtotal > 0 THEN
                        COALESCE(ROUND((om.summary_discount_subtotal * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 4), 0)
                      ELSE 0
                    END
                    + COALESCE(ot.promo_discount_sum, 0)
                  )
                END
              )::NUMERIC(14, 4) as discount,
              COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
              false as tax_included,
              0 as shipping,
              om.customer,
              om.customer_name,
              om.status,
              om.canceled,
              COALESCE(ot.costeach, pc.cost_price, oi.price * 0.5)::NUMERIC(14, 4) as costeach
            FROM temp_order_items oi
            JOIN temp_order_meta om ON oi.order_id = om.order_id
            LEFT JOIN order_totals ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
            LEFT JOIN temp_product_costs pc ON oi.pid = pc.pid
            WHERE oi.order_id = ANY($1)
            ORDER BY oi.order_id, oi.pid
          `, [subBatchIds]);

          // Filter orders and track missing products
          const validOrders = [];
          const processedOrderItems = new Set();
          const processedOrders = new Set();
          
          for (const order of orders.rows) {
            if (!existingPids.has(order.pid)) {
              missingProducts.add(order.pid);
              skippedOrders.add(order.order_number);
              continue;
            }
            validOrders.push(order);
            processedOrderItems.add(`${order.order_number}-${order.pid}`);
            processedOrders.add(order.order_number);
          }

          // Process valid orders in smaller sub-batches
          const FINAL_BATCH_SIZE = 100;  // Increased from 50 to 100
          for (let k = 0; k < validOrders.length; k += FINAL_BATCH_SIZE) {
            const subBatch = validOrders.slice(k, k + FINAL_BATCH_SIZE);
            
            const placeholders = subBatch.map((_, idx) => {
              const base = idx * 15;  // 15 columns including costeach
              return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
            }).join(',');

            const batchValues = subBatch.flatMap(o => [
              o.order_number,
              o.pid,
              o.sku || 'NO-SKU',
              o.date,  // This is now a TIMESTAMP WITH TIME ZONE
              o.price,
              o.quantity,
              o.discount,
              o.tax,
              o.tax_included,
              o.shipping,
              o.customer,
              o.customer_name,
              o.status, // Already mapped to text via orderStatusMap
              o.canceled,
              o.costeach
            ]);

            const [result] = await localConnection.query(`
              WITH inserted_orders AS (
                INSERT INTO orders (
                  order_number, pid, sku, date, price, quantity, discount,
                  tax, tax_included, shipping, customer, customer_name,
                  status, canceled, costeach
                )
                VALUES ${placeholders}
                ON CONFLICT (order_number, pid) DO UPDATE SET
                  sku = EXCLUDED.sku,
                  date = EXCLUDED.date,
                  price = EXCLUDED.price,
                  quantity = EXCLUDED.quantity,
                  discount = EXCLUDED.discount,
                  tax = EXCLUDED.tax,
                  tax_included = EXCLUDED.tax_included,
                  shipping = EXCLUDED.shipping,
                  customer = EXCLUDED.customer,
                  customer_name = EXCLUDED.customer_name,
                  status = EXCLUDED.status,
                  canceled = EXCLUDED.canceled,
                  costeach = EXCLUDED.costeach
                WHERE -- Only update if at least one key field has changed
                  orders.price IS DISTINCT FROM EXCLUDED.price OR
                  orders.quantity IS DISTINCT FROM EXCLUDED.quantity OR
                  orders.discount IS DISTINCT FROM EXCLUDED.discount OR
                  orders.tax IS DISTINCT FROM EXCLUDED.tax OR
                  orders.status IS DISTINCT FROM EXCLUDED.status OR
                  orders.canceled IS DISTINCT FROM EXCLUDED.canceled OR
                  orders.costeach IS DISTINCT FROM EXCLUDED.costeach OR
                  orders.date IS DISTINCT FROM EXCLUDED.date
                RETURNING xmax = 0 as inserted
              )
              SELECT 
                COUNT(*) FILTER (WHERE inserted) as inserted,
                COUNT(*) FILTER (WHERE NOT inserted) as updated
              FROM inserted_orders
            `, batchValues);
            
            const { inserted, updated } = result.rows[0];
            recordsAdded += parseInt(inserted) || 0;
            recordsUpdated += parseInt(updated) || 0;
            importedCount += subBatch.length;
          }

          await localConnection.commit();
          
          cumulativeProcessedOrders += processedOrders.size;
          outputProgress({
            status: "running",
            operation: "Orders import",
            message: `Importing orders: ${cumulativeProcessedOrders} of ${totalUniqueOrders}`,
            current: cumulativeProcessedOrders,
            total: totalUniqueOrders,
            elapsed: formatElapsedTime(startTime),
            remaining: estimateRemaining(startTime, cumulativeProcessedOrders, totalUniqueOrders),
            rate: calculateRate(startTime, cumulativeProcessedOrders)
          });
        } catch (error) {
          await localConnection.rollback();
          throw error;
        }
      }
    }

    // Reconciliation 2 prep: fetch canceled (15) / combined (16) orders from MySQL
    // WITHOUT a date_placed filter — combine_orders zeroes date_placed on the source
    // orders, so the main item query can never re-fetch them. Done before opening
    // the PG transaction so we don't hold it across a MySQL round-trip.
    const [statusSweepRows] = await prodConnection.query(`
      SELECT order_id, order_status
      FROM _order
      WHERE order_status IN (15, 16)
      ${incrementalUpdate ? 'AND stamp > ?' : ''}
    `, incrementalUpdate ? [mysqlSyncTime] : []);

    let staleItemsDeleted = 0;
    let sweepUpdated = 0;

    // Final transaction: reconcile deletions, sweep statuses, update sync status, drop temps
    await localConnection.beginTransaction();
    try {
      // Reconciliation 1: delete PG item rows that no longer exist in MySQL for the
      // orders fetched this run. temp_order_items holds the complete current item
      // set of every fetched order (staff edits and unpicked promo items DELETE
      // order_items rows in MySQL, which an upsert-only import never removes).
      const [reconcileResult] = await localConnection.query(`
        DELETE FROM orders o
        USING (SELECT DISTINCT order_id FROM temp_order_items) fetched
        WHERE o.order_number = fetched.order_id::text -- orders.order_number is TEXT
          AND NOT EXISTS (
            SELECT 1 FROM temp_order_items t
            WHERE t.order_id = fetched.order_id AND t.pid = o.pid
          )
      `);
      staleItemsDeleted = reconcileResult.rowCount || 0;

      // Reconciliation 2: mark canceled/combined orders. 'combined' source orders were
      // merged into a new order that carries the same items — counting both would
      // double-count, so they also get canceled = true (routes filter on canceled).
      for (const [code, statusText] of [[15, 'canceled'], [16, 'combined']]) {
        const ids = statusSweepRows.filter(r => r.order_status === code).map(r => r.order_id);
        for (let i = 0; i < ids.length; i += 5000) {
          const chunk = ids.slice(i, i + 5000);
          const [sweepResult] = await localConnection.query(`
            UPDATE orders
            SET status = $1, canceled = true
            WHERE order_number = ANY($2::text[])
              AND (status IS DISTINCT FROM $1 OR canceled IS DISTINCT FROM true)
          `, [statusText, chunk.map(String)]);
          sweepUpdated += sweepResult.rowCount || 0;
        }
      }

      // Update sync status with the watermark captured from MySQL BEFORE the
      // source queries ran (see sourceNow above).
      await localConnection.query(`
        INSERT INTO sync_status (table_name, last_sync_timestamp)
        VALUES ('orders', $1)
        ON CONFLICT (table_name) DO UPDATE SET
          last_sync_timestamp = $1
      `, [sourceNow]);

      // Cleanup temporary tables
      await localConnection.query(`
        DROP TABLE IF EXISTS temp_order_items;
        DROP TABLE IF EXISTS temp_order_meta;
        DROP TABLE IF EXISTS temp_order_taxes;
        DROP TABLE IF EXISTS temp_order_costs;
        DROP TABLE IF EXISTS temp_item_discounts;
        DROP TABLE IF EXISTS temp_product_costs;
      `);

      // Commit final transaction
      await localConnection.commit();
    } catch (error) {
      await localConnection.rollback();
      throw error;
    }

    if (staleItemsDeleted > 0 || sweepUpdated > 0) {
      console.log(`Orders: reconciliation removed ${staleItemsDeleted} stale item rows, swept ${sweepUpdated} canceled/combined rows`);
    }

    return {
      status: "complete",
      totalImported: Math.floor(importedCount) || 0,
      recordsAdded: parseInt(recordsAdded) || 0,
      recordsUpdated: parseInt(recordsUpdated) || 0,
      recordsDeleted: staleItemsDeleted,
      statusSweepUpdated: sweepUpdated,
      totalSkipped: skippedOrders.size || 0,
      missingProducts: missingProducts.size || 0,
      totalProcessed: orderItems.length,  // Total order items in source
      incrementalUpdate,
      lastSyncTime,
      details: {
        uniqueOrdersProcessed: cumulativeProcessedOrders,
        totalOrderItems: orderItems.length,
        skippedDueToMissingProducts: skippedOrders.size,
        missingProductIds: Array.from(missingProducts).slice(0, 100) // First 100 for debugging
      }
    };
  } catch (error) {
    console.error("Error during orders import:", error);
    throw error;
  }
}

module.exports = importOrders;