matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Product import fixes, more category import fixes

Browse Source
This commit is contained in:
Matt
2025-01-25 15:24:51 -05:00
parent 1694562947
commit c1cf78973d
5 changed files with 328 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
inventory-server/db/config-schema.sql
View File

@@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS stock_thresholds (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS lead_time_thresholds (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
@@ -43,7 +43,7 @@ CREATE TABLE IF NOT EXISTS sales_velocity_config (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
@@ -67,7 +67,7 @@ CREATE TABLE IF NOT EXISTS safety_stock_config (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
@@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS turnover_config (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
@@ -140,7 +140,7 @@ SELECT
FROM
stock_thresholds st
LEFT JOIN
categories c ON st.category_id = c.id
categories c ON st.category_id = c.cat_id
ORDER BY
CASE
WHEN st.category_id IS NULL AND st.vendor IS NULL THEN 1

74
inventory-server/db/metrics-schema.sql
View File

@@ -3,7 +3,7 @@ SET FOREIGN_KEY_CHECKS = 0;
-- Temporary tables for batch metrics processing
CREATE TABLE IF NOT EXISTS temp_sales_metrics (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
daily_sales_avg DECIMAL(10,3),
weekly_sales_avg DECIMAL(10,3),
monthly_sales_avg DECIMAL(10,3),
@@ -11,21 +11,21 @@ CREATE TABLE IF NOT EXISTS temp_sales_metrics (
avg_margin_percent DECIMAL(10,3),
first_sale_date DATE,
last_sale_date DATE,
PRIMARY KEY (product_id)
PRIMARY KEY (pid)
);
CREATE TABLE IF NOT EXISTS temp_purchase_metrics (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
avg_lead_time_days INT,
last_purchase_date DATE,
first_received_date DATE,
last_received_date DATE,
PRIMARY KEY (product_id)
PRIMARY KEY (pid)
);
-- New table for product metrics
CREATE TABLE IF NOT EXISTS product_metrics (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Sales velocity metrics
daily_sales_avg DECIMAL(10,3),
@@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS product_metrics (
last_purchase_date DATE,
first_received_date DATE,
last_received_date DATE,
-- Classification
-- Classification metrics
abc_class CHAR(1),
stock_status VARCHAR(20),
-- Turnover metrics
@@ -67,8 +67,8 @@ CREATE TABLE IF NOT EXISTS product_metrics (
forecast_accuracy DECIMAL(5,2) DEFAULT NULL,
forecast_bias DECIMAL(5,2) DEFAULT NULL,
last_forecast_date DATE DEFAULT NULL,
PRIMARY KEY (product_id),
FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
PRIMARY KEY (pid),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_metrics_revenue (total_revenue),
INDEX idx_metrics_stock_status (stock_status),
INDEX idx_metrics_lead_time (lead_time_status),
@@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS product_metrics (
-- New table for time-based aggregates
CREATE TABLE IF NOT EXISTS product_time_aggregates (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
-- Sales metrics
@@ -97,8 +97,8 @@ CREATE TABLE IF NOT EXISTS product_time_aggregates (
profit_margin DECIMAL(10,3),
inventory_value DECIMAL(10,3),
gmroi DECIMAL(10,3),
PRIMARY KEY (product_id, year, month),
FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
PRIMARY KEY (pid, year, month),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_date (year, month)
);
@@ -159,7 +159,7 @@ CREATE TABLE IF NOT EXISTS category_metrics (
-- Status
status VARCHAR(20) DEFAULT 'active',
PRIMARY KEY (category_id),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_status (status),
INDEX idx_category_growth (growth_rate),
INDEX idx_metrics_last_calculated (last_calculated_at),
@@ -198,7 +198,7 @@ CREATE TABLE IF NOT EXISTS category_time_metrics (
avg_margin DECIMAL(5,2),
turnover_rate DECIMAL(12,3),
PRIMARY KEY (category_id, year, month),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_date (year, month)
);
@@ -214,7 +214,7 @@ CREATE TABLE IF NOT EXISTS category_sales_metrics (
avg_price DECIMAL(10,3) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (category_id, brand, period_start, period_end),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_brand (category_id, brand),
INDEX idx_period (period_start, period_end)
);
@@ -261,14 +261,14 @@ CREATE TABLE IF NOT EXISTS brand_time_metrics (
-- New table for sales forecasts
CREATE TABLE IF NOT EXISTS sales_forecasts (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
forecast_date DATE NOT NULL,
forecast_units DECIMAL(10,2) DEFAULT 0,
forecast_revenue DECIMAL(10,2) DEFAULT 0,
confidence_level DECIMAL(5,2) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (product_id, forecast_date),
FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
PRIMARY KEY (pid, forecast_date),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_forecast_date (forecast_date),
INDEX idx_forecast_last_calculated (last_calculated_at)
);
@@ -282,7 +282,7 @@ CREATE TABLE IF NOT EXISTS category_forecasts (
confidence_level DECIMAL(5,2) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (category_id, forecast_date),
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_forecast_date (forecast_date),
INDEX idx_category_forecast_last_calculated (last_calculated_at)
);
@@ -311,17 +311,17 @@ SET FOREIGN_KEY_CHECKS = 1;
CREATE OR REPLACE VIEW inventory_health AS
WITH product_thresholds AS (
SELECT
p.product_id,
p.pid,
COALESCE(
-- Try category+vendor specific
(SELECT critical_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT critical_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT critical_days FROM stock_thresholds st
@@ -336,13 +336,13 @@ WITH product_thresholds AS (
COALESCE(
-- Try category+vendor specific
(SELECT reorder_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT reorder_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT reorder_days FROM stock_thresholds st
@@ -357,13 +357,13 @@ WITH product_thresholds AS (
COALESCE(
-- Try category+vendor specific
(SELECT overstock_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT overstock_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.category_id
WHERE pc.product_id = p.product_id
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT overstock_days FROM stock_thresholds st
@@ -378,7 +378,7 @@ WITH product_thresholds AS (
FROM products p
)
SELECT
p.product_id,
p.pid,
p.SKU,
p.title,
p.stock_quantity,
@@ -396,16 +396,16 @@ SELECT
FROM
products p
LEFT JOIN
product_metrics pm ON p.product_id = pm.product_id
product_metrics pm ON p.pid = pm.pid
LEFT JOIN
product_thresholds pt ON p.product_id = pt.product_id
product_thresholds pt ON p.pid = pt.pid
WHERE
p.managing_stock = true;
-- Create view for category performance trends
CREATE OR REPLACE VIEW category_performance_trends AS
SELECT
c.id as category_id,
c.cat_id as category_id,
c.name,
c.description,
p.name as parent_name,
@@ -425,6 +425,6 @@ SELECT
FROM
categories c
LEFT JOIN
categories p ON c.parent_id = p.id
categories p ON c.parent_id = p.cat_id
LEFT JOIN
category_metrics cm ON c.id = cm.category_id;
category_metrics cm ON c.cat_id = cm.category_id;

47
inventory-server/db/schema.sql
View File

@@ -4,13 +4,15 @@ SET FOREIGN_KEY_CHECKS = 0;
-- Create tables
CREATE TABLE products (
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
title VARCHAR(255) NOT NULL,
description TEXT,
SKU VARCHAR(50) NOT NULL,
created_at TIMESTAMP NULL,
first_received TIMESTAMP NULL,
stock_quantity INT DEFAULT 0,
preorder_count INT DEFAULT 0,
notions_inv_count INT DEFAULT 0,
price DECIMAL(10, 3) NOT NULL,
regular_price DECIMAL(10, 3) NOT NULL,
cost_price DECIMAL(10, 3),
@@ -49,7 +51,7 @@ CREATE TABLE products (
baskets INT UNSIGNED DEFAULT 0,
notifies INT UNSIGNED DEFAULT 0,
date_last_sold DATE,
PRIMARY KEY (product_id),
PRIMARY KEY (pid),
UNIQUE KEY unique_sku (SKU),
INDEX idx_vendor (vendor),
INDEX idx_brand (brand),
@@ -60,7 +62,7 @@ CREATE TABLE products (
-- Create categories table with hierarchy support
CREATE TABLE categories (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
cat_id BIGINT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(100) NOT NULL,
type SMALLINT NOT NULL COMMENT '10=section, 11=category, 12=subcategory, 13=subsubcategory, 1=company, 2=line, 3=subline, 40=artist',
parent_id BIGINT,
@@ -68,8 +70,7 @@ CREATE TABLE categories (
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
status VARCHAR(20) DEFAULT 'active',
UNIQUE KEY unique_category (id),
FOREIGN KEY (parent_id) REFERENCES categories(id),
FOREIGN KEY (parent_id) REFERENCES categories(cat_id),
INDEX idx_parent (parent_id),
INDEX idx_type (type),
INDEX idx_status (status),
@@ -90,20 +91,20 @@ CREATE TABLE vendor_details (
-- Create product_categories junction table
CREATE TABLE product_categories (
product_id BIGINT NOT NULL,
category_id BIGINT NOT NULL,
PRIMARY KEY (product_id, category_id),
FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE,
FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE,
INDEX idx_category (category_id),
INDEX idx_product (product_id)
cat_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
PRIMARY KEY (pid, cat_id),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
FOREIGN KEY (cat_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category (cat_id),
INDEX idx_product (pid)
) ENGINE=InnoDB;
-- Create orders table with its indexes
CREATE TABLE orders (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
order_number VARCHAR(50) NOT NULL,
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
SKU VARCHAR(50) NOT NULL,
date DATE NOT NULL,
price DECIMAL(10, 3) NOT NULL,
@@ -120,15 +121,15 @@ CREATE TABLE orders (
shipping_address TEXT,
billing_address TEXT,
canceled BOOLEAN DEFAULT false,
FOREIGN KEY (product_id) REFERENCES products(product_id),
FOREIGN KEY (pid) REFERENCES products(pid),
FOREIGN KEY (SKU) REFERENCES products(SKU),
INDEX idx_order_number (order_number),
INDEX idx_customer (customer),
INDEX idx_date (date),
INDEX idx_status (status),
INDEX idx_orders_metrics (product_id, date, canceled, quantity, price),
INDEX idx_orders_product_date (product_id, date),
UNIQUE KEY unique_order_product (order_number, product_id)
INDEX idx_orders_metrics (pid, date, canceled, quantity, price),
INDEX idx_orders_product_date (pid, date),
UNIQUE KEY unique_order_product (order_number, pid)
) ENGINE=InnoDB;
-- Create purchase_orders table with its indexes
@@ -138,7 +139,7 @@ CREATE TABLE purchase_orders (
vendor VARCHAR(100) NOT NULL,
date DATE NOT NULL,
expected_date DATE,
product_id BIGINT NOT NULL,
pid BIGINT NOT NULL,
sku VARCHAR(50) NOT NULL,
cost_price DECIMAL(10, 3) NOT NULL,
status VARCHAR(20) DEFAULT 'pending' COMMENT 'canceled,created,electronically_ready_send,ordered,preordered,electronically_sent,receiving_started,closed',
@@ -147,15 +148,15 @@ CREATE TABLE purchase_orders (
received INT DEFAULT 0,
received_date DATE,
received_by INT,
FOREIGN KEY (product_id) REFERENCES products(product_id),
FOREIGN KEY (pid) REFERENCES products(pid),
FOREIGN KEY (sku) REFERENCES products(SKU),
INDEX idx_po_id (po_id),
INDEX idx_vendor (vendor),
INDEX idx_status (status),
INDEX idx_purchase_orders_metrics (product_id, date, status, ordered, received),
INDEX idx_po_product_date (product_id, date),
INDEX idx_po_product_status (product_id, status),
UNIQUE KEY unique_po_product (po_id, product_id)
INDEX idx_purchase_orders_metrics (pid, date, status, ordered, received),
INDEX idx_po_product_date (pid, date),
INDEX idx_po_product_status (pid, status),
UNIQUE KEY unique_po_product (po_id, pid)
) ENGINE=InnoDB;
SET FOREIGN_KEY_CHECKS = 1;

355
inventory-server/scripts/import-from-prod.js
View File

@@ -118,7 +118,7 @@ async function importCategories(prodConnection, localConnection) {
for (const type of typeOrder) {
const [categories] = await prodConnection.query(`
SELECT
pc.cat_id as id,
pc.cat_id,
pc.name,
pc.type,
CASE
@@ -144,10 +144,10 @@ async function importCategories(prodConnection, localConnection) {
// Check which parents exist
const [existingParents] = await localConnection.query(
'SELECT id FROM categories WHERE id IN (?)',
'SELECT cat_id FROM categories WHERE cat_id IN (?)',
[parentIds]
);
const existingParentIds = new Set(existingParents.map(p => p.id));
const existingParentIds = new Set(existingParents.map(p => p.cat_id));
// Filter categories and track skipped ones
categoriesToInsert = categories.filter(cat =>
@@ -184,7 +184,7 @@ async function importCategories(prodConnection, localConnection) {
).join(',');
const values = categoriesToInsert.flatMap(cat => [
cat.id,
cat.cat_id,
cat.name,
cat.type,
cat.parent_id,
@@ -192,8 +192,9 @@ async function importCategories(prodConnection, localConnection) {
'active'
]);
// Insert categories and create relationships in one query to avoid race conditions
await localConnection.query(`
INSERT INTO categories (id, name, type, parent_id, description, status, created_at, updated_at)
INSERT INTO categories (cat_id, name, type, parent_id, description, status, created_at, updated_at)
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
name = VALUES(name),
@@ -241,45 +242,216 @@ async function importProducts(prodConnection, localConnection) {
const startTime = Date.now();
try {
// First get all products
// Get products from production
const [rows] = await prodConnection.query(`
SELECT
p.pid as id,
p.description as title,
p.notes as description,
p.itemnumber as SKU,
p.date_created as created_at,
p.datein as first_received,
p.available_local as stock_quantity,
p.price_each as price,
p.sellingprice as regular_price,
p.cost_each as cost_price,
p.cost_landed as landing_cost_price,
p.upc as barcode,
p.harmonized_tariff_code,
p.stamp as updated_at,
CASE WHEN p.show + p.buyable > 0 THEN 1 ELSE 0 END as visible,
1 as managing_stock,
CASE WHEN p.reorder IN (127, 0) THEN 1 ELSE 0 END as replenishable,
p.supplier_name as vendor,
p.supplier_itemnumber as vendor_reference,
p.notions_itemnumber as notions_reference,
p.permalink,
p.image,
p.image_175,
p.image_full,
p.brand,
p.line,
p.subline,
p.artist,
p.options,
p.tags,
GROUP_CONCAT(DISTINCT pc.cat_id) as categories
FROM products p
LEFT JOIN product_category_index pci ON p.pid = pci.pid
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
p.pid AS product_id,
p.description AS title,
p.notes AS description,
p.itemnumber AS SKU,
p.date_created AS created_at,
p.datein AS first_received,
p.location AS location,
(
SELECT
i.available_local - COALESCE(
(
SELECT
SUM(oi.qty_ordered - oi.qty_placed)
FROM
order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE
oi.prod_pid = i.pid
AND o.date_placed != '0000-00-00 00:00:00'
AND o.date_shipped = '0000-00-00 00:00:00'
AND oi.pick_finished = 0
AND oi.qty_back = 0
AND o.order_status != 15
AND o.order_status < 90
AND oi.qty_ordered >= oi.qty_placed
AND oi.qty_ordered > 0
),
0
) AS stock_quantity
FROM
shop_inventory i
WHERE
i.pid = p.pid
AND i.store = 0
AND i.show + i.buyable > 0
LIMIT 1
) AS stock_quantity,
ci.onpreorder AS preorder_count,
pnb.inventory AS notions_inv_count,
(
SELECT
price_each
FROM
product_current_prices
WHERE
pid = p.pid
AND active = 1
ORDER BY
qty_buy ASC
LIMIT 1
) AS price,
p.sellingprice AS regular_price,
(
SELECT
ROUND(AVG(costeach), 5)
FROM
product_inventory
WHERE
pid = p.pid
AND COUNT > 0
) AS cost_price,
NULL AS landing_cost_price,
p.upc AS barcode,
p.harmonized_tariff_code AS harmonized_tariff_code,
p.stamp AS updated_at,
CASE
WHEN si.show + si.buyable > 0 THEN 1
ELSE 0
END AS visible,
CASE
WHEN p.reorder >= 0 THEN 1
ELSE 0
END AS replenishable,
s.companyname AS vendor,
sid.supplier_itemnumber AS vendor_reference,
sid.notions_itemnumber AS notions_reference,
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
(
SELECT
GROUP_CONCAT(pc.name SEPARATOR ', ')
FROM
product_category_index pci
JOIN product_categories pc ON pci.cat_id = pc.cat_id
WHERE
pci.pid = p.pid
AND pc.hidden = 0
) AS categories,
(
SELECT
CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-t-', PI.iid, '.jpg')
FROM
product_images PI
WHERE
PI.pid = p.pid
AND PI.hidden = 0
ORDER BY
PI.order DESC,
PI.iid
LIMIT 1
) AS image,
(
SELECT
CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-175x175-', PI.iid, '.jpg')
FROM
product_images PI
WHERE
PI.pid = p.pid
AND PI.hidden = 0
AND PI.width = 175
ORDER BY
PI.order DESC,
PI.iid
LIMIT 1
) AS image_175,
(
SELECT
CONCAT('https://sbing.com/i/products/0000/', SUBSTRING(LPAD(p.pid, 6, '0'), 1, 3), '/', p.pid, '-o-', PI.iid, '.jpg')
FROM
product_images PI
WHERE
PI.pid = p.pid
AND PI.hidden = 0
ORDER BY
PI.width DESC,
PI.height DESC,
PI.iid
LIMIT 1
) AS image_full,
(
SELECT
name
FROM
product_categories
WHERE
cat_id = p.company
) AS brand,
(
SELECT
name
FROM
product_categories
WHERE
cat_id = p.line
) AS line,
(
SELECT
name
FROM
product_categories
WHERE
cat_id = p.subline
) AS subline,
(
SELECT
name
FROM
product_categories
WHERE
cat_id = p.artist
) AS artist,
NULL AS options,
NULL AS tags,
COALESCE(
CASE
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
ELSE sid.supplier_qty_per_unit
END,
sid.notions_qty_per_unit
) AS moq,
NULL AS uom,
p.rating,
p.rating_votes AS reviews,
p.weight,
p.length,
p.width,
p.height,
(
SELECT
COUNT(*)
FROM
mybasket mb
WHERE
mb.item = p.pid
AND mb.qty > 0
) AS baskets,
(
SELECT
COUNT(*)
FROM
product_notify pn
WHERE
pn.pid = p.pid
) AS notifies,
p.totalsold AS total_sold,
p.country_of_origin as country_of_origin,
pls.date_sold as date_last_sold
FROM
products p
LEFT JOIN current_inventory ci ON p.pid = ci.pid
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
LEFT JOIN product_category_index pci ON p.pid = pci.pid
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
WHERE p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 2 YEAR)
AND pc.hidden = 0
GROUP BY p.pid
`);
@@ -291,61 +463,40 @@ async function importProducts(prodConnection, localConnection) {
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
// Create placeholders for batch insert
const placeholders = batch.map(() =>
'(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
).join(',');
console.log(`Inserting ${batch.length} products`);
// Flatten values for batch insert
const values = batch.flatMap(row => [
row.id,
row.title,
row.description,
row.SKU,
row.created_at,
row.first_received,
row.stock_quantity || 0,
row.price || 0,
row.regular_price || 0,
row.cost_price,
row.landing_cost_price,
row.barcode,
row.harmonized_tariff_code,
row.updated_at,
row.visible,
row.managing_stock,
row.replenishable,
row.vendor,
row.vendor_reference,
row.notions_reference,
row.permalink,
row.image,
row.image_175,
row.image_full,
row.brand,
row.line,
row.subline,
row.artist,
row.options,
row.tags
const values = batch.flatMap(r => [
r.product_id, r.title, r.description, r.SKU, r.created_at, r.first_received,
r.stock_quantity, r.preorder_count, r.notions_inv_count, r.price, r.regular_price,
r.cost_price, r.landing_cost_price, r.barcode, r.harmonized_tariff_code,
r.updated_at, r.visible, 1, r.replenishable, r.vendor,
r.vendor_reference, r.notions_reference, r.permalink,
r.image, r.image_175, r.image_full, r.brand, r.line, r.subline, r.artist,
r.options, r.tags, r.moq, r.uom, r.rating, r.reviews, r.weight, r.length,
r.width, r.height, r.country_of_origin, r.location, r.total_sold,
r.baskets, r.notifies, r.date_last_sold
]);
await localConnection.query(`
// Create a constant for the SQL query to improve readability
const insertProductsSQL = `
INSERT INTO products (
id, title, description, SKU, created_at, first_received,
stock_quantity, price, regular_price, cost_price, landing_cost_price,
barcode, harmonized_tariff_code, updated_at, visible, managing_stock,
replenishable, vendor, vendor_reference, notions_reference, permalink,
image, image_175, image_full, brand, line, subline, artist, options, tags
product_id, title, description, SKU, created_at, first_received,
stock_quantity, preorder_count, notions_inv_count, price, regular_price,
cost_price, landing_cost_price, barcode, harmonized_tariff_code,
updated_at, visible, managing_stock, replenishable, vendor,
vendor_reference, notions_reference, permalink,
image, image_175, image_full, brand, line, subline, artist,
options, tags, moq, uom, rating, reviews, weight, length,
width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold
)
VALUES ${placeholders}
VALUES ${batch.map(() => '(' + '?,'.repeat(45).slice(0,-1) + ')').join(',')}
ON DUPLICATE KEY UPDATE
title = VALUES(title),
description = VALUES(description),
SKU = VALUES(SKU),
created_at = VALUES(created_at),
first_received = VALUES(first_received),
stock_quantity = VALUES(stock_quantity),
preorder_count = VALUES(preorder_count),
notions_inv_count = VALUES(notions_inv_count),
price = VALUES(price),
regular_price = VALUES(regular_price),
cost_price = VALUES(cost_price),
@@ -354,7 +505,6 @@ async function importProducts(prodConnection, localConnection) {
harmonized_tariff_code = VALUES(harmonized_tariff_code),
updated_at = VALUES(updated_at),
visible = VALUES(visible),
managing_stock = VALUES(managing_stock),
replenishable = VALUES(replenishable),
vendor = VALUES(vendor),
vendor_reference = VALUES(vendor_reference),
@@ -368,8 +518,23 @@ async function importProducts(prodConnection, localConnection) {
subline = VALUES(subline),
artist = VALUES(artist),
options = VALUES(options),
tags = VALUES(tags)
`, values);
tags = VALUES(tags),
moq = VALUES(moq),
uom = VALUES(uom),
rating = VALUES(rating),
reviews = VALUES(reviews),
weight = VALUES(weight),
length = VALUES(length),
width = VALUES(width),
height = VALUES(height),
country_of_origin = VALUES(country_of_origin),
location = VALUES(location),
total_sold = VALUES(total_sold),
baskets = VALUES(baskets),
notifies = VALUES(notifies),
date_last_sold = VALUES(date_last_sold)`;
await localConnection.query(insertProductsSQL, values);
current += batch.length;
updateProgress(current, total, 'Products import', startTime);
@@ -400,8 +565,8 @@ async function importProductCategories(prodConnection, localConnection) {
// Get product category relationships from production
const [rows] = await prodConnection.query(`
SELECT DISTINCT
pci.pid as product_id,
pci.cat_id as category_id
pci.pid,
pci.cat_id
FROM
product_category_index pci
JOIN product_categories pc ON pci.cat_id = pc.cat_id
@@ -421,13 +586,13 @@ async function importProductCategories(prodConnection, localConnection) {
const placeholders = batch.map(() => '(?, ?)').join(',');
// Flatten values for batch insert
const values = batch.flatMap(row => [row.product_id, row.category_id]);
const values = batch.flatMap(row => [row.cat_id, row.pid]);
await localConnection.query(`
INSERT INTO product_categories (product_id, category_id)
INSERT INTO product_categories (cat_id, pid)
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
category_id = VALUES(category_id)
cat_id = VALUES(cat_id)
`, values);
current += batch.length;