matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Newsletter recommendation tweaks, add campaign history dialog

Browse Source
This commit is contained in:
Matt
2026-02-01 17:37:08 -05:00
parent 450fd96e19
commit 2744e82264
9 changed files with 1232 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
inventory-server/scripts/import/products.js
View File

@@ -75,6 +75,7 @@ async function setupTemporaryTables(connection) {
artist TEXT,
categories TEXT,
created_at TIMESTAMP WITH TIME ZONE,
date_online TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
landing_cost_price NUMERIC(14, 4),
barcode TEXT,
@@ -98,6 +99,7 @@ async function setupTemporaryTables(connection) {
baskets INTEGER,
notifies INTEGER,
date_last_sold TIMESTAMP WITH TIME ZONE,
shop_score NUMERIC(10, 2) DEFAULT 0,
primary_iid INTEGER,
image TEXT,
image_175 TEXT,
@@ -137,13 +139,14 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
p.notes AS description,
p.itemnumber AS sku,
p.date_created,
p.date_ol,
p.datein AS first_received,
p.location,
p.upc AS barcode,
p.harmonized_tariff_code,
p.stamp AS updated_at,
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
CASE
CASE
WHEN p.reorder < 0 THEN 0
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
@@ -160,20 +163,20 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
COALESCE(pnb.inventory, 0) as notions_inv_count,
COALESCE(pcp.price_each, 0) as price,
COALESCE(p.sellingprice, 0) AS regular_price,
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
THEN (
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
WHERE pid = p.pid AND count > 0
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
END AS vendor_reference,
sid.notions_itemnumber AS notions_reference,
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
@@ -181,7 +184,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
pc2.name AS line,
pc3.name AS subline,
pc4.name AS artist,
COALESCE(CASE
COALESCE(CASE
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
ELSE sid.supplier_qty_per_unit
END, sid.notions_qty_per_unit) AS moq,
@@ -194,17 +197,18 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
p.country_of_origin,
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
pls.date_sold as date_last_sold,
COALESCE(p.score, 0) as shop_score,
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
AND pc.type IN (10, 20, 11, 21, 12, 13)
AND pci.cat_id NOT IN (16, 17)
THEN pci.cat_id
THEN pci.cat_id
END) as category_ids
FROM products p
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
@@ -238,8 +242,8 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
const batch = prodData.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 48; // 48 columns
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -264,6 +268,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
row.artist,
row.category_ids,
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -287,6 +292,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
row.primary_iid,
imageUrls.image,
imageUrls.image_175,
@@ -301,11 +307,11 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
INSERT INTO products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
)
VALUES ${placeholders}
ON CONFLICT (pid) DO NOTHING
@@ -343,13 +349,14 @@ async function materializeCalculations(prodConnection, localConnection, incremen
p.notes AS description,
p.itemnumber AS sku,
p.date_created,
p.date_ol,
p.datein AS first_received,
p.location,
p.upc AS barcode,
p.harmonized_tariff_code,
p.stamp AS updated_at,
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
CASE
CASE
WHEN p.reorder < 0 THEN 0
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
@@ -366,20 +373,20 @@ async function materializeCalculations(prodConnection, localConnection, incremen
COALESCE(pnb.inventory, 0) as notions_inv_count,
COALESCE(pcp.price_each, 0) as price,
COALESCE(p.sellingprice, 0) AS regular_price,
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
THEN (
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
WHERE pid = p.pid AND count > 0
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
END AS vendor_reference,
sid.notions_itemnumber AS notions_reference,
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
@@ -387,7 +394,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
pc2.name AS line,
pc3.name AS subline,
pc4.name AS artist,
COALESCE(CASE
COALESCE(CASE
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
ELSE sid.supplier_qty_per_unit
END, sid.notions_qty_per_unit) AS moq,
@@ -400,17 +407,18 @@ async function materializeCalculations(prodConnection, localConnection, incremen
p.country_of_origin,
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
pls.date_sold as date_last_sold,
COALESCE(p.score, 0) as shop_score,
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
AND pc.type IN (10, 20, 11, 21, 12, 13)
AND pci.cat_id NOT IN (16, 17)
THEN pci.cat_id
THEN pci.cat_id
END) as category_ids
FROM products p
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
@@ -449,8 +457,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen
await withRetry(async () => {
const placeholders = batch.map((_, idx) => {
const base = idx * 48; // 48 columns
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -475,6 +483,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
row.artist,
row.category_ids,
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -498,6 +507,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
row.primary_iid,
imageUrls.image,
imageUrls.image_175,
@@ -511,11 +521,11 @@ async function materializeCalculations(prodConnection, localConnection, incremen
INSERT INTO temp_products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
) VALUES ${placeholders}
ON CONFLICT (pid) DO UPDATE SET
title = EXCLUDED.title,
@@ -535,6 +545,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
subline = EXCLUDED.subline,
artist = EXCLUDED.artist,
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
@@ -558,13 +569,14 @@ async function materializeCalculations(prodConnection, localConnection, incremen
baskets = EXCLUDED.baskets,
notifies = EXCLUDED.notifies,
date_last_sold = EXCLUDED.date_last_sold,
shop_score = EXCLUDED.shop_score,
primary_iid = EXCLUDED.primary_iid,
image = EXCLUDED.image,
image_175 = EXCLUDED.image_175,
image_full = EXCLUDED.image_full,
options = EXCLUDED.options,
tags = EXCLUDED.tags
RETURNING
RETURNING
xmax = 0 as inserted
`, values);
}, `Error inserting batch ${i} to ${i + batch.length}`);
@@ -614,8 +626,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen
AND t.barcode IS NOT DISTINCT FROM p.barcode
AND t.updated_at IS NOT DISTINCT FROM p.updated_at
AND t.total_sold IS NOT DISTINCT FROM p.total_sold
-- Check key fields that are likely to change
-- We don't need to check every single field, just the important ones
AND t.date_online IS NOT DISTINCT FROM p.date_online
AND t.shop_score IS NOT DISTINCT FROM p.shop_score
`);
// Get count of products that need updating
@@ -688,6 +700,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
t.artist,
t.categories,
t.created_at,
t.date_online,
t.first_received,
t.landing_cost_price,
t.barcode,
@@ -710,6 +723,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
t.baskets,
t.notifies,
t.date_last_sold,
t.shop_score,
t.primary_iid,
t.image,
t.image_175,
@@ -728,8 +742,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
const batch = products.rows.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 47; // 47 columns
return `(${Array.from({ length: 47 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 49; // 49 columns
return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -754,6 +768,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
row.artist,
row.categories,
validateDate(row.created_at),
validateDate(row.date_online),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -777,6 +792,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
imageUrls.image,
imageUrls.image_175,
imageUrls.image_full,
@@ -790,11 +806,11 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
INSERT INTO products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, image, image_175, image_full, options, tags
)
VALUES ${placeholders}
ON CONFLICT (pid) DO UPDATE SET
@@ -815,6 +831,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
subline = EXCLUDED.subline,
artist = EXCLUDED.artist,
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
@@ -838,15 +855,16 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
baskets = EXCLUDED.baskets,
notifies = EXCLUDED.notifies,
date_last_sold = EXCLUDED.date_last_sold,
shop_score = EXCLUDED.shop_score,
image = EXCLUDED.image,
image_175 = EXCLUDED.image_175,
image_full = EXCLUDED.image_full,
options = EXCLUDED.options,
tags = EXCLUDED.tags
RETURNING
RETURNING
xmax = 0 as inserted
)
SELECT
SELECT
COUNT(*) FILTER (WHERE inserted) as inserted,
COUNT(*) FILTER (WHERE NOT inserted) as updated
FROM upserted