Product import fixes, more category import fixes

inventory-server/db/config-schema.sql

@@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS stock_thresholds (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     UNIQUE KEY unique_category_vendor (category_id, vendor)
 );
 
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS lead_time_thresholds (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     UNIQUE KEY unique_category_vendor (category_id, vendor)
 );
 
@@ -43,7 +43,7 @@ CREATE TABLE IF NOT EXISTS sales_velocity_config (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     UNIQUE KEY unique_category_vendor (category_id, vendor)
 );
 
@@ -67,7 +67,7 @@ CREATE TABLE IF NOT EXISTS safety_stock_config (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     UNIQUE KEY unique_category_vendor (category_id, vendor)
 );
 
@@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS turnover_config (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     UNIQUE KEY unique_category_vendor (category_id, vendor)
 );
 
@@ -140,7 +140,7 @@ SELECT
 FROM 
     stock_thresholds st
 LEFT JOIN 
-    categories c ON st.category_id = c.id
+    categories c ON st.category_id = c.cat_id
 ORDER BY 
     CASE 
         WHEN st.category_id IS NULL AND st.vendor IS NULL THEN 1

inventory-server/db/metrics-schema.sql

@@ -3,7 +3,7 @@ SET FOREIGN_KEY_CHECKS = 0;
 
 -- Temporary tables for batch metrics processing
 CREATE TABLE IF NOT EXISTS temp_sales_metrics (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     daily_sales_avg DECIMAL(10,3),
     weekly_sales_avg DECIMAL(10,3),
     monthly_sales_avg DECIMAL(10,3),
@@ -11,21 +11,21 @@ CREATE TABLE IF NOT EXISTS temp_sales_metrics (
     avg_margin_percent DECIMAL(10,3),
     first_sale_date DATE,
     last_sale_date DATE,
-    PRIMARY KEY (product_id)
+    PRIMARY KEY (pid)
 );
 
 CREATE TABLE IF NOT EXISTS temp_purchase_metrics (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     avg_lead_time_days INT,
     last_purchase_date DATE,
     first_received_date DATE,
     last_received_date DATE,
-    PRIMARY KEY (product_id)
+    PRIMARY KEY (pid)
 );
 
 -- New table for product metrics
 CREATE TABLE IF NOT EXISTS product_metrics (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     -- Sales velocity metrics
     daily_sales_avg DECIMAL(10,3),
@@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS product_metrics (
     last_purchase_date DATE,
     first_received_date DATE,
     last_received_date DATE,
-    -- Classification
+    -- Classification metrics
     abc_class CHAR(1),
     stock_status VARCHAR(20),
     -- Turnover metrics
@@ -67,8 +67,8 @@ CREATE TABLE IF NOT EXISTS product_metrics (
     forecast_accuracy DECIMAL(5,2) DEFAULT NULL,
     forecast_bias DECIMAL(5,2) DEFAULT NULL,
     last_forecast_date DATE DEFAULT NULL,
-    PRIMARY KEY (product_id),
+    PRIMARY KEY (pid),
-    FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
+    FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
     INDEX idx_metrics_revenue (total_revenue),
     INDEX idx_metrics_stock_status (stock_status),
     INDEX idx_metrics_lead_time (lead_time_status),
@@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS product_metrics (
 
 -- New table for time-based aggregates
 CREATE TABLE IF NOT EXISTS product_time_aggregates (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     year INT NOT NULL,
     month INT NOT NULL,
     -- Sales metrics
@@ -97,8 +97,8 @@ CREATE TABLE IF NOT EXISTS product_time_aggregates (
     profit_margin DECIMAL(10,3),
     inventory_value DECIMAL(10,3),
     gmroi DECIMAL(10,3),
-    PRIMARY KEY (product_id, year, month),
+    PRIMARY KEY (pid, year, month),
-    FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
+    FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
     INDEX idx_date (year, month)
 );
 
@@ -159,7 +159,7 @@ CREATE TABLE IF NOT EXISTS category_metrics (
     -- Status
     status VARCHAR(20) DEFAULT 'active',
     PRIMARY KEY (category_id),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     INDEX idx_category_status (status),
     INDEX idx_category_growth (growth_rate),
     INDEX idx_metrics_last_calculated (last_calculated_at),
@@ -198,7 +198,7 @@ CREATE TABLE IF NOT EXISTS category_time_metrics (
     avg_margin DECIMAL(5,2),
     turnover_rate DECIMAL(12,3),
     PRIMARY KEY (category_id, year, month),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     INDEX idx_category_date (year, month)
 );
 
@@ -214,7 +214,7 @@ CREATE TABLE IF NOT EXISTS category_sales_metrics (
     avg_price DECIMAL(10,3) DEFAULT 0,
     last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (category_id, brand, period_start, period_end),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     INDEX idx_category_brand (category_id, brand),
     INDEX idx_period (period_start, period_end)
 );
@@ -261,14 +261,14 @@ CREATE TABLE IF NOT EXISTS brand_time_metrics (
 
 -- New table for sales forecasts
 CREATE TABLE IF NOT EXISTS sales_forecasts (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     forecast_date DATE NOT NULL,
     forecast_units DECIMAL(10,2) DEFAULT 0,
     forecast_revenue DECIMAL(10,2) DEFAULT 0,
     confidence_level DECIMAL(5,2) DEFAULT 0,
     last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    PRIMARY KEY (product_id, forecast_date),
+    PRIMARY KEY (pid, forecast_date),
-    FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
+    FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
     INDEX idx_forecast_date (forecast_date),
     INDEX idx_forecast_last_calculated (last_calculated_at)
 );
@@ -282,7 +282,7 @@ CREATE TABLE IF NOT EXISTS category_forecasts (
     confidence_level DECIMAL(5,2) DEFAULT 0,
     last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (category_id, forecast_date),
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
     INDEX idx_category_forecast_date (forecast_date),
     INDEX idx_category_forecast_last_calculated (last_calculated_at)
 );
@@ -311,17 +311,17 @@ SET FOREIGN_KEY_CHECKS = 1;
 CREATE OR REPLACE VIEW inventory_health AS
 WITH product_thresholds AS (
     SELECT 
-        p.product_id,
+        p.pid,
         COALESCE(
             -- Try category+vendor specific
             (SELECT critical_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor = p.vendor LIMIT 1),
             -- Try category specific
             (SELECT critical_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor IS NULL LIMIT 1),
             -- Try vendor specific
             (SELECT critical_days FROM stock_thresholds st 
@@ -336,13 +336,13 @@ WITH product_thresholds AS (
         COALESCE(
             -- Try category+vendor specific
             (SELECT reorder_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor = p.vendor LIMIT 1),
             -- Try category specific
             (SELECT reorder_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor IS NULL LIMIT 1),
             -- Try vendor specific
             (SELECT reorder_days FROM stock_thresholds st 
@@ -357,13 +357,13 @@ WITH product_thresholds AS (
         COALESCE(
             -- Try category+vendor specific
             (SELECT overstock_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor = p.vendor LIMIT 1),
             -- Try category specific
             (SELECT overstock_days FROM stock_thresholds st 
-            JOIN product_categories pc ON st.category_id = pc.category_id
+            JOIN product_categories pc ON st.category_id = pc.cat_id
-            WHERE pc.product_id = p.product_id 
+            WHERE pc.pid = p.pid 
             AND st.vendor IS NULL LIMIT 1),
             -- Try vendor specific
             (SELECT overstock_days FROM stock_thresholds st 
@@ -378,7 +378,7 @@ WITH product_thresholds AS (
     FROM products p
 )
 SELECT 
-    p.product_id,
+    p.pid,
     p.SKU,
     p.title,
     p.stock_quantity,
@@ -396,16 +396,16 @@ SELECT
 FROM 
     products p
 LEFT JOIN 
-    product_metrics pm ON p.product_id = pm.product_id
+    product_metrics pm ON p.pid = pm.pid
 LEFT JOIN
-    product_thresholds pt ON p.product_id = pt.product_id
+    product_thresholds pt ON p.pid = pt.pid
 WHERE 
     p.managing_stock = true;
 
 -- Create view for category performance trends
 CREATE OR REPLACE VIEW category_performance_trends AS
 SELECT 
-    c.id as category_id,
+    c.cat_id as category_id,
     c.name,
     c.description,
     p.name as parent_name,
@@ -425,6 +425,6 @@ SELECT
 FROM 
     categories c
 LEFT JOIN 
-    categories p ON c.parent_id = p.id
+    categories p ON c.parent_id = p.cat_id
 LEFT JOIN 
-    category_metrics cm ON c.id = cm.category_id; 
+    category_metrics cm ON c.cat_id = cm.category_id; 

inventory-server/db/schema.sql

@@ -4,13 +4,15 @@ SET FOREIGN_KEY_CHECKS = 0;
 
 -- Create tables
 CREATE TABLE products (
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     title VARCHAR(255) NOT NULL,
     description TEXT,
     SKU VARCHAR(50) NOT NULL,
     created_at TIMESTAMP NULL,
     first_received TIMESTAMP NULL,
     stock_quantity INT DEFAULT 0,
+    preorder_count INT DEFAULT 0,
+    notions_inv_count INT DEFAULT 0,
     price DECIMAL(10, 3) NOT NULL,
     regular_price DECIMAL(10, 3) NOT NULL,
     cost_price DECIMAL(10, 3),
@@ -49,7 +51,7 @@ CREATE TABLE products (
     baskets INT UNSIGNED DEFAULT 0,
     notifies INT UNSIGNED DEFAULT 0,
     date_last_sold DATE,
-    PRIMARY KEY (product_id),
+    PRIMARY KEY (pid),
     UNIQUE KEY unique_sku (SKU),
     INDEX idx_vendor (vendor),
     INDEX idx_brand (brand),
@@ -60,7 +62,7 @@ CREATE TABLE products (
 
 -- Create categories table with hierarchy support
 CREATE TABLE categories (
-    id BIGINT AUTO_INCREMENT PRIMARY KEY,
+    cat_id BIGINT AUTO_INCREMENT PRIMARY KEY,
     name VARCHAR(100) NOT NULL,
     type SMALLINT NOT NULL COMMENT '10=section, 11=category, 12=subcategory, 13=subsubcategory, 1=company, 2=line, 3=subline, 40=artist',
     parent_id BIGINT,
@@ -68,8 +70,7 @@ CREATE TABLE categories (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     status VARCHAR(20) DEFAULT 'active',
-    UNIQUE KEY unique_category (id),
+    FOREIGN KEY (parent_id) REFERENCES categories(cat_id),
-    FOREIGN KEY (parent_id) REFERENCES categories(id),
     INDEX idx_parent (parent_id),
     INDEX idx_type (type),
     INDEX idx_status (status),
@@ -90,20 +91,20 @@ CREATE TABLE vendor_details (
 
 -- Create product_categories junction table
 CREATE TABLE product_categories (
-    product_id BIGINT NOT NULL,
+    cat_id BIGINT NOT NULL,
-    category_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
-    PRIMARY KEY (product_id, category_id),
+    PRIMARY KEY (pid, cat_id),
-    FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
+    FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
-    FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
+    FOREIGN KEY (cat_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
-    INDEX idx_category (category_id),
+    INDEX idx_category (cat_id),
-    INDEX idx_product (product_id)
+    INDEX idx_product (pid)
 ) ENGINE=InnoDB;
 
 -- Create orders table with its indexes
 CREATE TABLE orders (
     id BIGINT AUTO_INCREMENT PRIMARY KEY,
     order_number VARCHAR(50) NOT NULL,
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     SKU VARCHAR(50) NOT NULL,
     date DATE NOT NULL,
     price DECIMAL(10, 3) NOT NULL,
@@ -120,15 +121,15 @@ CREATE TABLE orders (
     shipping_address TEXT,
     billing_address TEXT,
     canceled BOOLEAN DEFAULT false,
-    FOREIGN KEY (product_id) REFERENCES products(product_id),
+    FOREIGN KEY (pid) REFERENCES products(pid),
     FOREIGN KEY (SKU) REFERENCES products(SKU),
     INDEX idx_order_number (order_number),
     INDEX idx_customer (customer),
     INDEX idx_date (date),
     INDEX idx_status (status),
-    INDEX idx_orders_metrics (product_id, date, canceled, quantity, price),
+    INDEX idx_orders_metrics (pid, date, canceled, quantity, price),
-    INDEX idx_orders_product_date (product_id, date),
+    INDEX idx_orders_product_date (pid, date),
-    UNIQUE KEY unique_order_product (order_number, product_id)
+    UNIQUE KEY unique_order_product (order_number, pid)
 ) ENGINE=InnoDB;
 
 -- Create purchase_orders table with its indexes
@@ -138,7 +139,7 @@ CREATE TABLE purchase_orders (
     vendor VARCHAR(100) NOT NULL,
     date DATE NOT NULL,
     expected_date DATE,
-    product_id BIGINT NOT NULL,
+    pid BIGINT NOT NULL,
     sku VARCHAR(50) NOT NULL,
     cost_price DECIMAL(10, 3) NOT NULL,
     status VARCHAR(20) DEFAULT 'pending' COMMENT 'canceled,created,electronically_ready_send,ordered,preordered,electronically_sent,receiving_started,closed',
@@ -147,15 +148,15 @@ CREATE TABLE purchase_orders (
     received INT DEFAULT 0,
     received_date DATE,
     received_by INT,
-    FOREIGN KEY (product_id) REFERENCES products(product_id),
+    FOREIGN KEY (pid) REFERENCES products(pid),
     FOREIGN KEY (sku) REFERENCES products(SKU),
     INDEX idx_po_id (po_id),
     INDEX idx_vendor (vendor),
     INDEX idx_status (status),
-    INDEX idx_purchase_orders_metrics (product_id, date, status, ordered, received),
+    INDEX idx_purchase_orders_metrics (pid, date, status, ordered, received),
-    INDEX idx_po_product_date (product_id, date),
+    INDEX idx_po_product_date (pid, date),
-    INDEX idx_po_product_status (product_id, status),
+    INDEX idx_po_product_status (pid, status),
-    UNIQUE KEY unique_po_product (po_id, product_id)
+    UNIQUE KEY unique_po_product (po_id, pid)
 ) ENGINE=InnoDB;
 
 SET FOREIGN_KEY_CHECKS = 1;

inventory-server/scripts/import-from-prod.js

@@ -118,7 +118,7 @@ async function importCategories(prodConnection, localConnection) {
     for (const type of typeOrder) {
       const [categories] = await prodConnection.query(`
         SELECT 
-          pc.cat_id as id,
+          pc.cat_id,
           pc.name,
           pc.type,
           CASE 
@@ -144,10 +144,10 @@ async function importCategories(prodConnection, localConnection) {
         
         // Check which parents exist
         const [existingParents] = await localConnection.query(
-          'SELECT id FROM categories WHERE id IN (?)',
+          'SELECT cat_id FROM categories WHERE cat_id IN (?)',
           [parentIds]
         );
-        const existingParentIds = new Set(existingParents.map(p => p.id));
+        const existingParentIds = new Set(existingParents.map(p => p.cat_id));
 
         // Filter categories and track skipped ones
         categoriesToInsert = categories.filter(cat => 
@@ -184,7 +184,7 @@ async function importCategories(prodConnection, localConnection) {
       ).join(',');
 
       const values = categoriesToInsert.flatMap(cat => [
-        cat.id,
+        cat.cat_id,
         cat.name,
         cat.type,
         cat.parent_id,
@@ -192,8 +192,9 @@ async function importCategories(prodConnection, localConnection) {
         'active'
       ]);
 
+      // Insert categories and create relationships in one query to avoid race conditions
       await localConnection.query(`
-        INSERT INTO categories (id, name, type, parent_id, description, status, created_at, updated_at)
+        INSERT INTO categories (cat_id, name, type, parent_id, description, status, created_at, updated_at)
         VALUES ${placeholders}
         ON DUPLICATE KEY UPDATE
           name = VALUES(name),
@@ -241,45 +242,216 @@ async function importProducts(prodConnection, localConnection) {
   const startTime = Date.now();
 
   try {
-    // First get all products
+    // Get products from production
     const [rows] = await prodConnection.query(`
       SELECT 
-        p.pid as id,
+        p.pid AS product_id,
-        p.description as title,
+        p.description AS title,
-        p.notes as description,
+        p.notes AS description,
-        p.itemnumber as SKU,
+        p.itemnumber AS SKU,
-        p.date_created as created_at,
+        p.date_created AS created_at,
-        p.datein as first_received,
+        p.datein AS first_received,
-        p.available_local as stock_quantity,
+        p.location AS location,
-        p.price_each as price,
+        (
-        p.sellingprice as regular_price,
+          SELECT
-        p.cost_each as cost_price,
+            i.available_local - COALESCE(
-        p.cost_landed as landing_cost_price,
+              (
-        p.upc as barcode,
+                SELECT
-        p.harmonized_tariff_code,
+                  SUM(oi.qty_ordered - oi.qty_placed)
-        p.stamp as updated_at,
+                FROM
-        CASE WHEN p.show + p.buyable > 0 THEN 1 ELSE 0 END as visible,
+                  order_items oi
-        1 as managing_stock,
+                  JOIN _order o ON oi.order_id = o.order_id
-        CASE WHEN p.reorder IN (127, 0) THEN 1 ELSE 0 END as replenishable,
+                WHERE
-        p.supplier_name as vendor,
+                  oi.prod_pid = i.pid
-        p.supplier_itemnumber as vendor_reference,
+                  AND o.date_placed != '0000-00-00 00:00:00'
-        p.notions_itemnumber as notions_reference,
+                  AND o.date_shipped = '0000-00-00 00:00:00'
-        p.permalink,
+                  AND oi.pick_finished = 0
-        p.image,
+                  AND oi.qty_back = 0
-        p.image_175,
+                  AND o.order_status != 15
-        p.image_full,
+                  AND o.order_status < 90
-        p.brand,
+                  AND oi.qty_ordered >= oi.qty_placed
-        p.line,
+                  AND oi.qty_ordered > 0
-        p.subline,
+              ),
-        p.artist,
+              0
-        p.options,
+            ) AS stock_quantity
-        p.tags,
+          FROM
-        GROUP_CONCAT(DISTINCT pc.cat_id) as categories
+            shop_inventory i
-      FROM products p
+          WHERE
+            i.pid = p.pid
+            AND i.store = 0
+            AND i.show + i.buyable > 0
+          LIMIT 1
+        ) AS stock_quantity,
+        ci.onpreorder AS preorder_count,
+        pnb.inventory AS notions_inv_count,
+        (
+          SELECT
+            price_each
+          FROM
+            product_current_prices
+          WHERE
+            pid = p.pid
+            AND active = 1
+          ORDER BY
+            qty_buy ASC
+          LIMIT 1
+        ) AS price,
+        p.sellingprice AS regular_price,
+        (
+          SELECT
+            ROUND(AVG(costeach), 5)
+          FROM
+            product_inventory
+          WHERE
+            pid = p.pid
+            AND COUNT > 0
+        ) AS cost_price,
+        NULL AS landing_cost_price,
+        p.upc AS barcode,
+        p.harmonized_tariff_code AS harmonized_tariff_code,
+        p.stamp AS updated_at,
+        CASE
+          WHEN si.show + si.buyable > 0 THEN 1
+          ELSE 0
+        END AS visible,
+        CASE
+          WHEN p.reorder >= 0 THEN 1
+          ELSE 0
+        END AS replenishable,
+        s.companyname AS vendor,
+        sid.supplier_itemnumber AS vendor_reference,
+        sid.notions_itemnumber AS notions_reference,
+        CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
+        (
+          SELECT
+            GROUP_CONCAT(pc.name SEPARATOR ', ')
+          FROM
+            product_category_index pci
+            JOIN product_categories pc ON pci.cat_id = pc.cat_id
+          WHERE
+            pci.pid = p.pid
+            AND pc.hidden = 0
+        ) AS categories,
+        (
+          SELECT
+            CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-t-', PI.iid, '.jpg')
+          FROM
+            product_images PI
+          WHERE
+            PI.pid = p.pid
+            AND PI.hidden = 0
+          ORDER BY
+            PI.order DESC,
+            PI.iid
+          LIMIT 1
+        ) AS image,
+        (
+          SELECT
+            CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-175x175-', PI.iid, '.jpg')
+          FROM
+            product_images PI
+          WHERE
+            PI.pid = p.pid
+            AND PI.hidden = 0
+            AND PI.width = 175
+          ORDER BY
+            PI.order DESC,
+            PI.iid
+          LIMIT 1
+        ) AS image_175,
+        (
+          SELECT
+            CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-o-', PI.iid, '.jpg')
+          FROM
+            product_images PI
+          WHERE
+            PI.pid = p.pid
+            AND PI.hidden = 0
+          ORDER BY
+            PI.width DESC,
+            PI.height DESC,
+            PI.iid
+          LIMIT 1
+        ) AS image_full,
+        (
+          SELECT
+            name
+          FROM
+            product_categories
+          WHERE
+            cat_id = p.company
+        ) AS brand,
+        (
+          SELECT
+            name
+          FROM
+            product_categories
+          WHERE
+            cat_id = p.line
+        ) AS line,
+        (
+          SELECT
+            name
+          FROM
+            product_categories
+          WHERE
+            cat_id = p.subline
+        ) AS subline,
+        (
+          SELECT
+            name
+          FROM
+            product_categories
+          WHERE
+            cat_id = p.artist
+        ) AS artist,
+        NULL AS options,
+        NULL AS tags,
+        COALESCE(
+          CASE
+            WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
+            ELSE sid.supplier_qty_per_unit
+          END,
+          sid.notions_qty_per_unit
+        ) AS moq,
+        NULL AS uom,
+        p.rating,
+        p.rating_votes AS reviews,
+        p.weight,
+        p.length,
+        p.width,
+        p.height,
+        (
+          SELECT
+            COUNT(*)
+          FROM
+            mybasket mb
+          WHERE
+            mb.item = p.pid
+            AND mb.qty > 0
+        ) AS baskets,
+        (
+          SELECT
+            COUNT(*)
+          FROM
+            product_notify pn
+          WHERE
+            pn.pid = p.pid
+        ) AS notifies,
+        p.totalsold AS total_sold,
+        p.country_of_origin as country_of_origin,
+        pls.date_sold as date_last_sold
+      FROM
+        products p
+        LEFT JOIN current_inventory ci ON p.pid = ci.pid
+        LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
+        LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
+        LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
+        LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
         LEFT JOIN product_category_index pci ON p.pid = pci.pid
         LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
+        LEFT JOIN product_last_sold pls ON p.pid = pls.pid
       WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
-        AND pc.hidden = 0
       GROUP BY p.pid
     `);
 
@@ -291,61 +463,40 @@ async function importProducts(prodConnection, localConnection) {
     for (let i = 0; i < rows.length; i += BATCH_SIZE) {
       const batch = rows.slice(i, i + BATCH_SIZE);
       
-      // Create placeholders for batch insert
+      console.log(`Inserting ${batch.length} products`);
-      const placeholders = batch.map(() => 
-        '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
-      ).join(',');
 
-      // Flatten values for batch insert
+      const values = batch.flatMap(r => [
-      const values = batch.flatMap(row => [
+        r.product_id, r.title, r.description, r.SKU, r.created_at, r.first_received,
-        row.id,
+        r.stock_quantity, r.preorder_count, r.notions_inv_count, r.price, r.regular_price,
-        row.title,
+        r.cost_price, r.landing_cost_price, r.barcode, r.harmonized_tariff_code,
-        row.description,
+        r.updated_at, r.visible, 1, r.replenishable, r.vendor,
-        row.SKU,
+        r.vendor_reference, r.notions_reference, r.permalink,
-        row.created_at,
+        r.image, r.image_175, r.image_full, r.brand, r.line, r.subline, r.artist,
-        row.first_received,
+        r.options, r.tags, r.moq, r.uom, r.rating, r.reviews, r.weight, r.length,
-        row.stock_quantity || 0,
+        r.width, r.height, r.country_of_origin, r.location, r.total_sold,
-        row.price || 0,
+        r.baskets, r.notifies, r.date_last_sold
-        row.regular_price || 0,
-        row.cost_price,
-        row.landing_cost_price,
-        row.barcode,
-        row.harmonized_tariff_code,
-        row.updated_at,
-        row.visible,
-        row.managing_stock,
-        row.replenishable,
-        row.vendor,
-        row.vendor_reference,
-        row.notions_reference,
-        row.permalink,
-        row.image,
-        row.image_175,
-        row.image_full,
-        row.brand,
-        row.line,
-        row.subline,
-        row.artist,
-        row.options,
-        row.tags
       ]);
 
-      await localConnection.query(`
+      // Create a constant for the SQL query to improve readability
+      const insertProductsSQL = `
         INSERT INTO products (
-          id, title, description, SKU, created_at, first_received,
+          product_id, title, description, SKU, created_at, first_received,
-          stock_quantity, price, regular_price, cost_price, landing_cost_price,
+          stock_quantity, preorder_count, notions_inv_count, price, regular_price,
-          barcode, harmonized_tariff_code, updated_at, visible, managing_stock,
+          cost_price, landing_cost_price, barcode, harmonized_tariff_code,
-          replenishable, vendor, vendor_reference, notions_reference, permalink,
+          updated_at, visible, managing_stock, replenishable, vendor,
-          image, image_175, image_full, brand, line, subline, artist, options, tags
+          vendor_reference, notions_reference, permalink,
+          image, image_175, image_full, brand, line, subline, artist,
+          options, tags, moq, uom, rating, reviews, weight, length,
+          width, height, country_of_origin, location, total_sold,
+          baskets, notifies, date_last_sold
         )
-        VALUES ${placeholders}
+        VALUES ${batch.map(() => '(' + '?,'.repeat(45).slice(0,-1) + ')').join(',')}
         ON DUPLICATE KEY UPDATE
           title = VALUES(title),
           description = VALUES(description),
-          SKU = VALUES(SKU),
-          created_at = VALUES(created_at),
-          first_received = VALUES(first_received),
           stock_quantity = VALUES(stock_quantity),
+          preorder_count = VALUES(preorder_count),
+          notions_inv_count = VALUES(notions_inv_count),
           price = VALUES(price),
           regular_price = VALUES(regular_price),
           cost_price = VALUES(cost_price),
@@ -354,7 +505,6 @@ async function importProducts(prodConnection, localConnection) {
           harmonized_tariff_code = VALUES(harmonized_tariff_code),
           updated_at = VALUES(updated_at),
           visible = VALUES(visible),
-          managing_stock = VALUES(managing_stock),
           replenishable = VALUES(replenishable),
           vendor = VALUES(vendor),
           vendor_reference = VALUES(vendor_reference),
@@ -368,8 +518,23 @@ async function importProducts(prodConnection, localConnection) {
           subline = VALUES(subline),
           artist = VALUES(artist),
           options = VALUES(options),
-          tags = VALUES(tags)
+          tags = VALUES(tags),
-      `, values);
+          moq = VALUES(moq),
+          uom = VALUES(uom),
+          rating = VALUES(rating),
+          reviews = VALUES(reviews),
+          weight = VALUES(weight),
+          length = VALUES(length),
+          width = VALUES(width),
+          height = VALUES(height),
+          country_of_origin = VALUES(country_of_origin),
+          location = VALUES(location),
+          total_sold = VALUES(total_sold),
+          baskets = VALUES(baskets),
+          notifies = VALUES(notifies),
+          date_last_sold = VALUES(date_last_sold)`;
+
+      await localConnection.query(insertProductsSQL, values);
 
       current += batch.length;
       updateProgress(current, total, 'Products import', startTime);
@@ -400,8 +565,8 @@ async function importProductCategories(prodConnection, localConnection) {
     // Get product category relationships from production
     const [rows] = await prodConnection.query(`
       SELECT DISTINCT
-        pci.pid as product_id,
+        pci.pid,
-        pci.cat_id as category_id
+        pci.cat_id
       FROM
         product_category_index pci
         JOIN product_categories pc ON pci.cat_id = pc.cat_id
@@ -421,13 +586,13 @@ async function importProductCategories(prodConnection, localConnection) {
       const placeholders = batch.map(() => '(?, ?)').join(',');
 
       // Flatten values for batch insert
-      const values = batch.flatMap(row => [row.product_id, row.category_id]);
+      const values = batch.flatMap(row => [row.cat_id, row.pid]);
 
       await localConnection.query(`
-        INSERT INTO product_categories (product_id, category_id)
+        INSERT INTO product_categories (cat_id, pid)
         VALUES ${placeholders}
         ON DUPLICATE KEY UPDATE
-          category_id = VALUES(category_id)
+          cat_id = VALUES(cat_id)
       `, values);
 
       current += batch.length;

inventory/src/components/settings/DataManagement.tsx

@@ -52,7 +52,7 @@ export function DataManagement() {
   const [purchaseOrdersProgress, setPurchaseOrdersProgress] = useState<ImportProgress | null>(null);
   const [resetProgress, setResetProgress] = useState<ImportProgress | null>(null);
   const [eventSource, setEventSource] = useState<EventSource | null>(null);
-  const [limits] = useState<ImportLimits>({
+  const [] = useState<ImportLimits>({
     products: 0,
     orders: 0,
     purchaseOrders: 0