matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Newsletter recommendation tweaks, add campaign history dialog

Browse Source
This commit is contained in:
Matt
2026-02-01 17:37:08 -05:00
parent 450fd96e19
commit 2744e82264
9 changed files with 1232 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
inventory-server/db/migrations/add-date-online-and-shop-score.sql Normal file
View File

@@ -0,0 +1,20 @@
-- Migration: Add date_online and shop_score columns to products table
-- These fields are imported from production to improve newsletter recommendation accuracy:
-- date_online = products.date_ol in production (date product went live on the shop)
-- shop_score = products.score in production (sales-based popularity score)
--
-- After running this migration, do a full (non-incremental) import to backfill:
-- INCREMENTAL_UPDATE=false node scripts/import-from-prod.js
-- Add date_online column (production: products.date_ol)
ALTER TABLE products ADD COLUMN IF NOT EXISTS date_online TIMESTAMP WITH TIME ZONE;
-- Add shop_score column (production: products.score)
-- Using NUMERIC(10,2) to preserve the decimal precision from production
ALTER TABLE products ADD COLUMN IF NOT EXISTS shop_score NUMERIC(10, 2) DEFAULT 0;
-- If shop_score was previously created as INTEGER, convert it
ALTER TABLE products ALTER COLUMN shop_score TYPE NUMERIC(10, 2);
-- Index on date_online for the newsletter "new products" filter
CREATE INDEX IF NOT EXISTS idx_products_date_online ON products(date_online);

2
inventory-server/db/schema.sql
View File

@@ -21,6 +21,7 @@ CREATE TABLE products (
description TEXT,
sku TEXT NOT NULL,
created_at TIMESTAMP WITH TIME ZONE,
date_online TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
stock_quantity INTEGER DEFAULT 0,
preorder_count INTEGER DEFAULT 0,
@@ -63,6 +64,7 @@ CREATE TABLE products (
baskets INTEGER DEFAULT 0,
notifies INTEGER DEFAULT 0,
date_last_sold DATE,
shop_score NUMERIC(10, 2) DEFAULT 0,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (pid)
);

112
inventory-server/scripts/import/products.js
View File

@@ -75,6 +75,7 @@ async function setupTemporaryTables(connection) {
artist TEXT,
categories TEXT,
created_at TIMESTAMP WITH TIME ZONE,
date_online TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
landing_cost_price NUMERIC(14, 4),
barcode TEXT,
@@ -98,6 +99,7 @@ async function setupTemporaryTables(connection) {
baskets INTEGER,
notifies INTEGER,
date_last_sold TIMESTAMP WITH TIME ZONE,
shop_score NUMERIC(10, 2) DEFAULT 0,
primary_iid INTEGER,
image TEXT,
image_175 TEXT,
@@ -137,13 +139,14 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
p.notes AS description,
p.itemnumber AS sku,
p.date_created,
p.date_ol,
p.datein AS first_received,
p.location,
p.upc AS barcode,
p.harmonized_tariff_code,
p.stamp AS updated_at,
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
CASE
CASE
WHEN p.reorder < 0 THEN 0
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
@@ -160,20 +163,20 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
COALESCE(pnb.inventory, 0) as notions_inv_count,
COALESCE(pcp.price_each, 0) as price,
COALESCE(p.sellingprice, 0) AS regular_price,
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
THEN (
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
WHERE pid = p.pid AND count > 0
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
END AS vendor_reference,
sid.notions_itemnumber AS notions_reference,
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
@@ -181,7 +184,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
pc2.name AS line,
pc3.name AS subline,
pc4.name AS artist,
COALESCE(CASE
COALESCE(CASE
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
ELSE sid.supplier_qty_per_unit
END, sid.notions_qty_per_unit) AS moq,
@@ -194,17 +197,18 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
p.country_of_origin,
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
pls.date_sold as date_last_sold,
COALESCE(p.score, 0) as shop_score,
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
AND pc.type IN (10, 20, 11, 21, 12, 13)
AND pci.cat_id NOT IN (16, 17)
THEN pci.cat_id
THEN pci.cat_id
END) as category_ids
FROM products p
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
@@ -238,8 +242,8 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
const batch = prodData.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 48; // 48 columns
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -264,6 +268,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
row.artist,
row.category_ids,
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -287,6 +292,7 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
row.primary_iid,
imageUrls.image,
imageUrls.image_175,
@@ -301,11 +307,11 @@ async function importMissingProducts(prodConnection, localConnection, missingPid
INSERT INTO products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
)
VALUES ${placeholders}
ON CONFLICT (pid) DO NOTHING
@@ -343,13 +349,14 @@ async function materializeCalculations(prodConnection, localConnection, incremen
p.notes AS description,
p.itemnumber AS sku,
p.date_created,
p.date_ol,
p.datein AS first_received,
p.location,
p.upc AS barcode,
p.harmonized_tariff_code,
p.stamp AS updated_at,
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
CASE
CASE
WHEN p.reorder < 0 THEN 0
WHEN p.date_created >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR) THEN 1
WHEN COALESCE(pnb.inventory, 0) > 0 THEN 1
@@ -366,20 +373,20 @@ async function materializeCalculations(prodConnection, localConnection, incremen
COALESCE(pnb.inventory, 0) as notions_inv_count,
COALESCE(pcp.price_each, 0) as price,
COALESCE(p.sellingprice, 0) AS regular_price,
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
CASE
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
THEN (
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
SELECT ROUND(SUM(costeach * count) / SUM(count), 5)
FROM product_inventory
WHERE pid = p.pid AND count > 0
)
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
END AS cost_price,
NULL as landing_cost_price,
s.companyname AS vendor,
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
CASE
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
ELSE sid.supplier_itemnumber
END AS vendor_reference,
sid.notions_itemnumber AS notions_reference,
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
@@ -387,7 +394,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
pc2.name AS line,
pc3.name AS subline,
pc4.name AS artist,
COALESCE(CASE
COALESCE(CASE
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
ELSE sid.supplier_qty_per_unit
END, sid.notions_qty_per_unit) AS moq,
@@ -400,17 +407,18 @@ async function materializeCalculations(prodConnection, localConnection, incremen
p.country_of_origin,
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
(SELECT COALESCE(SUM(oi.qty_ordered), 0)
FROM order_items oi
JOIN _order o ON oi.order_id = o.order_id
WHERE oi.prod_pid = p.pid AND o.order_status >= 20) AS total_sold,
pls.date_sold as date_last_sold,
COALESCE(p.score, 0) as shop_score,
(SELECT iid FROM product_images WHERE pid = p.pid AND \`order\` = 255 LIMIT 1) AS primary_iid,
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
GROUP_CONCAT(DISTINCT CASE
WHEN pc.cat_id IS NOT NULL
AND pc.type IN (10, 20, 11, 21, 12, 13)
AND pci.cat_id NOT IN (16, 17)
THEN pci.cat_id
THEN pci.cat_id
END) as category_ids
FROM products p
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
@@ -449,8 +457,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen
await withRetry(async () => {
const placeholders = batch.map((_, idx) => {
const base = idx * 48; // 48 columns
return `(${Array.from({ length: 48 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 50; // 50 columns
return `(${Array.from({ length: 50 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -475,6 +483,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
row.artist,
row.category_ids,
validateDate(row.date_created),
validateDate(row.date_ol),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -498,6 +507,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
row.primary_iid,
imageUrls.image,
imageUrls.image_175,
@@ -511,11 +521,11 @@ async function materializeCalculations(prodConnection, localConnection, incremen
INSERT INTO temp_products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, primary_iid, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, primary_iid, image, image_175, image_full, options, tags
) VALUES ${placeholders}
ON CONFLICT (pid) DO UPDATE SET
title = EXCLUDED.title,
@@ -535,6 +545,7 @@ async function materializeCalculations(prodConnection, localConnection, incremen
subline = EXCLUDED.subline,
artist = EXCLUDED.artist,
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
@@ -558,13 +569,14 @@ async function materializeCalculations(prodConnection, localConnection, incremen
baskets = EXCLUDED.baskets,
notifies = EXCLUDED.notifies,
date_last_sold = EXCLUDED.date_last_sold,
shop_score = EXCLUDED.shop_score,
primary_iid = EXCLUDED.primary_iid,
image = EXCLUDED.image,
image_175 = EXCLUDED.image_175,
image_full = EXCLUDED.image_full,
options = EXCLUDED.options,
tags = EXCLUDED.tags
RETURNING
RETURNING
xmax = 0 as inserted
`, values);
}, `Error inserting batch ${i} to ${i + batch.length}`);
@@ -614,8 +626,8 @@ async function materializeCalculations(prodConnection, localConnection, incremen
AND t.barcode IS NOT DISTINCT FROM p.barcode
AND t.updated_at IS NOT DISTINCT FROM p.updated_at
AND t.total_sold IS NOT DISTINCT FROM p.total_sold
-- Check key fields that are likely to change
-- We don't need to check every single field, just the important ones
AND t.date_online IS NOT DISTINCT FROM p.date_online
AND t.shop_score IS NOT DISTINCT FROM p.shop_score
`);
// Get count of products that need updating
@@ -688,6 +700,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
t.artist,
t.categories,
t.created_at,
t.date_online,
t.first_received,
t.landing_cost_price,
t.barcode,
@@ -710,6 +723,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
t.baskets,
t.notifies,
t.date_last_sold,
t.shop_score,
t.primary_iid,
t.image,
t.image_175,
@@ -728,8 +742,8 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
const batch = products.rows.slice(i, i + BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 47; // 47 columns
return `(${Array.from({ length: 47 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
const base = idx * 49; // 49 columns
return `(${Array.from({ length: 49 }, (_, i) => `$${base + i + 1}`).join(', ')})`;
}).join(',');
const values = batch.flatMap(row => {
@@ -754,6 +768,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
row.artist,
row.categories,
validateDate(row.created_at),
validateDate(row.date_online),
validateDate(row.first_received),
row.landing_cost_price,
row.barcode,
@@ -777,6 +792,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
row.baskets,
row.notifies,
validateDate(row.date_last_sold),
Number(row.shop_score) || 0,
imageUrls.image,
imageUrls.image_175,
imageUrls.image_full,
@@ -790,11 +806,11 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
INSERT INTO products (
pid, title, description, sku, stock_quantity, preorder_count, notions_inv_count,
price, regular_price, cost_price, vendor, vendor_reference, notions_reference,
brand, line, subline, artist, categories, created_at, first_received,
brand, line, subline, artist, categories, created_at, date_online, first_received,
landing_cost_price, barcode, harmonized_tariff_code, updated_at, visible,
managing_stock, replenishable, permalink, moq, uom, rating, reviews,
weight, length, width, height, country_of_origin, location, total_sold,
baskets, notifies, date_last_sold, image, image_175, image_full, options, tags
baskets, notifies, date_last_sold, shop_score, image, image_175, image_full, options, tags
)
VALUES ${placeholders}
ON CONFLICT (pid) DO UPDATE SET
@@ -815,6 +831,7 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
subline = EXCLUDED.subline,
artist = EXCLUDED.artist,
created_at = EXCLUDED.created_at,
date_online = EXCLUDED.date_online,
first_received = EXCLUDED.first_received,
landing_cost_price = EXCLUDED.landing_cost_price,
barcode = EXCLUDED.barcode,
@@ -838,15 +855,16 @@ async function importProducts(prodConnection, localConnection, incrementalUpdate
baskets = EXCLUDED.baskets,
notifies = EXCLUDED.notifies,
date_last_sold = EXCLUDED.date_last_sold,
shop_score = EXCLUDED.shop_score,
image = EXCLUDED.image,
image_175 = EXCLUDED.image_175,
image_full = EXCLUDED.image_full,
options = EXCLUDED.options,
tags = EXCLUDED.tags
RETURNING
RETURNING
xmax = 0 as inserted
)
SELECT
SELECT
COUNT(*) FILTER (WHERE inserted) as inserted,
COUNT(*) FILTER (WHERE NOT inserted) as updated
FROM upserted

332
inventory-server/src/routes/newsletter.js
View File

@@ -10,13 +10,13 @@ const REF_DATE_CTE = `
// Category definitions matching production website logic:
//
// NEW: created_at within 31 days, NOT preorder (mutually exclusive on prod)
// NEW: date_online within 31 days (matches prod's date_ol), NOT preorder
// PRE-ORDER: preorder_count > 0, NOT new
// CLEARANCE: (regular_price - price) / regular_price >= 0.35, price > 0
// CLEARANCE: (regular_price - price) / regular_price >= 0.35 (matches prod's 35% clearance threshold)
// DAILY DEALS: product_daily_deals table
// BACK IN STOCK: date_last_received > date_first_received, received within 14d,
// first received > 30d ago, excludes new products (prod excludes datein < 30d)
// BESTSELLERS: ranked by recent order volume (prod's "hot" logic)
// BESTSELLERS: shop_score > 20 + in stock + recent sales (matches prod's /shop/hot page)
//
// Mutual exclusivity:
// - New and Pre-order are exclusive: if preorder_count > 0, it's preorder not new
@@ -29,7 +29,7 @@ const CATEGORY_FILTERS = {
clearance: "AND is_clearance = true",
daily_deals: "AND is_daily_deal = true",
back_in_stock: "AND is_back_in_stock = true",
bestsellers: "AND COALESCE(sales_30d, 0) > 0",
bestsellers: "AND shop_score > 20 AND COALESCE(current_stock, 0) > 0 AND COALESCE(sales_30d, 0) > 0",
never_featured: "AND times_featured IS NULL AND line_last_featured_at IS NULL",
};
@@ -38,6 +38,8 @@ function buildScoredCTE({ forCount = false } = {}) {
const selectColumns = forCount ? `
p.pid,
p.created_at,
p.date_online,
p.shop_score,
p.preorder_count,
p.price,
p.regular_price,
@@ -62,6 +64,7 @@ function buildScoredCTE({ forCount = false } = {}) {
p.vendor,
p.price,
p.regular_price,
p.shop_score,
p.image_175 as image,
p.permalink,
p.stock_quantity,
@@ -70,6 +73,7 @@ function buildScoredCTE({ forCount = false } = {}) {
p.categories,
p.line,
p.created_at as product_created_at,
p.date_online,
p.first_received,
p.date_last_sold,
p.total_sold,
@@ -101,7 +105,7 @@ function buildScoredCTE({ forCount = false } = {}) {
EXTRACT(DAY FROM ref.d - lh.line_last_featured_at)::int as line_days_since_featured,
COALESCE(nh.last_featured_at, lh.line_last_featured_at) as effective_last_featured,
EXTRACT(DAY FROM ref.d - COALESCE(nh.last_featured_at, lh.line_last_featured_at))::int as effective_days_since_featured,
EXTRACT(DAY FROM ref.d - p.created_at)::int as age_days
EXTRACT(DAY FROM ref.d - COALESCE(p.date_online, p.created_at))::int as age_days
`;
return `
@@ -144,10 +148,11 @@ function buildScoredCTE({ forCount = false } = {}) {
-- === CATEGORY FLAGS (production-accurate, mutually exclusive where needed) ===
-- NEW: within 31 days of reference date, AND not on preorder
-- NEW: date_online within 31 days of reference date, AND not on preorder
-- Uses date_online (prod's date_ol) instead of created_at for accuracy
CASE
WHEN p.preorder_count > 0 THEN false
WHEN p.created_at > ref.d - INTERVAL '31 days' THEN true
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '31 days' THEN true
ELSE false
END as is_new,
@@ -157,7 +162,7 @@ function buildScoredCTE({ forCount = false } = {}) {
ELSE false
END as is_preorder,
-- CLEARANCE: 35%+ discount off regular price, price must be > 0
-- CLEARANCE: 35%+ discount off regular price (matches prod threshold), price must be > 0
CASE
WHEN p.price > 0 AND p.regular_price > 0 AND p.price < p.regular_price
AND ((p.regular_price - p.price) / p.regular_price * 100) >= 35
@@ -184,7 +189,7 @@ function buildScoredCTE({ forCount = false } = {}) {
-- We use date_last_received/date_first_received as our equivalents
CASE
WHEN p.preorder_count > 0 THEN false
WHEN p.created_at > ref.d - INTERVAL '31 days' THEN false
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '31 days' THEN false
WHEN pm.date_last_received > ref.d - INTERVAL '14 days'
AND pm.date_last_received > pm.date_first_received
AND pm.date_first_received < ref.d - INTERVAL '30 days'
@@ -195,11 +200,11 @@ function buildScoredCTE({ forCount = false } = {}) {
-- === RECOMMENDATION SCORE ===
(
-- New product boost (first 31 days, not preorder)
-- New product boost (first 31 days by date_online, not preorder)
CASE
WHEN p.preorder_count > 0 THEN 0
WHEN p.created_at > ref.d - INTERVAL '14 days' THEN 50
WHEN p.created_at > ref.d - INTERVAL '31 days' THEN 35
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '14 days' THEN 50
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '31 days' THEN 35
ELSE 0
END
-- Pre-order boost
@@ -219,7 +224,7 @@ function buildScoredCTE({ forCount = false } = {}) {
-- Back in stock boost (only for actual restocks, not new arrivals)
+ CASE
WHEN p.preorder_count = 0
AND p.created_at <= ref.d - INTERVAL '31 days'
AND COALESCE(p.date_online, p.created_at) <= ref.d - INTERVAL '31 days'
AND pm.date_last_received > ref.d - INTERVAL '14 days'
AND pm.date_last_received > pm.date_first_received
AND pm.date_first_received < ref.d - INTERVAL '30 days'
@@ -266,8 +271,12 @@ function buildScoredCTE({ forCount = false } = {}) {
+ CASE WHEN pm.abc_class = 'A' THEN 10
WHEN pm.abc_class = 'B' THEN 5
ELSE 0 END
-- In-stock requirement
+ CASE WHEN COALESCE(pm.current_stock, 0) <= 0 AND COALESCE(p.preorder_count, 0) = 0 THEN -100 ELSE 0 END
-- Stock penalty
+ CASE
WHEN COALESCE(pm.current_stock, 0) <= 0 AND COALESCE(p.preorder_count, 0) = 0 THEN -100
WHEN COALESCE(pm.current_stock, 0) <= 2 AND COALESCE(p.preorder_count, 0) = 0 THEN -20
ELSE 0
END
) as score
FROM ref, products p
@@ -368,7 +377,7 @@ router.get('/stats', async (req, res) => {
-- Unfeatured new products
(SELECT COUNT(*) FROM products p, ref
WHERE p.visible = true AND p.preorder_count = 0
AND p.created_at > ref.d - INTERVAL '31 days'
AND COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '31 days'
AND p.pid NOT IN (SELECT pid FROM featured_pids)
) as unfeatured_new,
-- Back in stock, not yet featured since restock
@@ -377,7 +386,7 @@ router.get('/stats', async (req, res) => {
CROSS JOIN ref
WHERE p.visible = true
AND p.preorder_count = 0
AND p.created_at <= ref.d - INTERVAL '31 days'
AND COALESCE(p.date_online, p.created_at) <= ref.d - INTERVAL '31 days'
AND pm.date_last_received > ref.d - INTERVAL '14 days'
AND pm.date_last_received > pm.date_first_received
AND pm.date_first_received < ref.d - INTERVAL '30 days'
@@ -422,4 +431,293 @@ router.get('/stats', async (req, res) => {
}
});
// GET /api/newsletter/score-breakdown/:pid
// Returns the individual scoring factors for a single product (debug endpoint)
router.get('/score-breakdown/:pid', async (req, res) => {
const pool = req.app.locals.pool;
const { pid } = req.params;
try {
const { rows } = await pool.query(`
WITH ${REF_DATE_CTE},
newsletter_history AS (
SELECT pid, COUNT(*) as times_featured, MAX(sent_at) as last_featured_at
FROM klaviyo_campaign_products GROUP BY pid
),
line_history AS (
SELECT p2.line,
COUNT(DISTINCT kcp.pid) FILTER (WHERE kcp.sent_at > (SELECT d FROM ref) - INTERVAL '7 days') as line_products_featured_7d
FROM products p2
JOIN klaviyo_campaign_products kcp ON kcp.pid = p2.pid
WHERE p2.line IS NOT NULL AND p2.line != ''
GROUP BY p2.line
),
line_sizes AS (
SELECT line, COUNT(*) as line_product_count
FROM products WHERE visible = true AND line IS NOT NULL AND line != '' GROUP BY line
)
SELECT
-- New product boost
CASE
WHEN p.preorder_count > 0 THEN 0
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '14 days' THEN 50
WHEN COALESCE(p.date_online, p.created_at) > ref.d - INTERVAL '31 days' THEN 35
ELSE 0
END as new_boost,
-- Pre-order boost
CASE WHEN p.preorder_count > 0 THEN 30 ELSE 0 END as preorder_boost,
-- Clearance boost
CASE
WHEN p.price > 0 AND p.regular_price > 0 AND p.price < p.regular_price
AND ((p.regular_price - p.price) / p.regular_price * 100) >= 35
THEN LEAST(((p.regular_price - p.price) / p.regular_price * 50)::int, 25)
ELSE 0
END as clearance_boost,
-- Sales velocity
CASE WHEN COALESCE(pm.sales_7d, 0) >= 5 THEN 15
WHEN COALESCE(pm.sales_7d, 0) >= 2 THEN 10
WHEN COALESCE(pm.sales_7d, 0) >= 1 THEN 5
ELSE 0 END as velocity_boost,
-- Back in stock
CASE
WHEN p.preorder_count = 0
AND COALESCE(p.date_online, p.created_at) <= ref.d - INTERVAL '31 days'
AND pm.date_last_received > ref.d - INTERVAL '14 days'
AND pm.date_last_received > pm.date_first_received
AND pm.date_first_received < ref.d - INTERVAL '30 days'
AND pm.current_stock > 0
THEN 25 ELSE 0
END as back_in_stock_boost,
-- Interest
LEAST((COALESCE(p.baskets, 0) + COALESCE(p.notifies, 0)) / 2, 15) as interest_boost,
-- Recency
CASE
WHEN COALESCE(nh.last_featured_at, lh.line_last_featured_at) IS NULL THEN 10
WHEN COALESCE(nh.last_featured_at, lh.line_last_featured_at) > ref.d - INTERVAL '2 days' THEN -30
WHEN COALESCE(nh.last_featured_at, lh.line_last_featured_at) > ref.d - INTERVAL '5 days' THEN -15
WHEN COALESCE(nh.last_featured_at, lh.line_last_featured_at) > ref.d - INTERVAL '10 days' THEN -5
ELSE 5
END as recency_adj,
-- Over-featured
CASE
WHEN COALESCE(nh.times_featured, 0) > 15 THEN -10
WHEN COALESCE(nh.times_featured, 0) > 8 THEN -5
ELSE 0
END as over_featured_adj,
-- Line saturation
CASE
WHEN lh2.line_products_featured_7d IS NOT NULL
AND ls.line_product_count IS NOT NULL AND ls.line_product_count > 0
AND (lh2.line_products_featured_7d::float / ls.line_product_count) > 0.7
THEN -10
WHEN lh2.line_products_featured_7d IS NOT NULL AND lh2.line_products_featured_7d >= 4
THEN -5
ELSE 0
END as line_saturation_adj,
-- Price tier
CASE
WHEN COALESCE(p.price, 0) < 3 THEN -15
WHEN COALESCE(p.price, 0) < 8 THEN -5
WHEN COALESCE(p.price, 0) >= 25 THEN 5
ELSE 0
END as price_tier_adj,
-- ABC class
CASE WHEN pm.abc_class = 'A' THEN 10 WHEN pm.abc_class = 'B' THEN 5 ELSE 0 END as abc_boost,
-- Stock penalty
CASE
WHEN COALESCE(pm.current_stock, 0) <= 0 AND COALESCE(p.preorder_count, 0) = 0 THEN -100
WHEN COALESCE(pm.current_stock, 0) <= 2 AND COALESCE(p.preorder_count, 0) = 0 THEN -20
ELSE 0
END as stock_penalty
FROM ref, products p
LEFT JOIN product_metrics pm ON pm.pid = p.pid
LEFT JOIN newsletter_history nh ON nh.pid = p.pid
LEFT JOIN LATERAL (
SELECT MAX(kcp.sent_at) as line_last_featured_at
FROM products p3
JOIN klaviyo_campaign_products kcp ON kcp.pid = p3.pid
WHERE p3.line = p.line AND p.line IS NOT NULL AND p.line != ''
) lh ON true
LEFT JOIN line_history lh2 ON lh2.line = p.line AND p.line IS NOT NULL AND p.line != ''
LEFT JOIN line_sizes ls ON ls.line = p.line AND p.line IS NOT NULL AND p.line != ''
WHERE p.pid = $1
`, [pid]);
if (rows.length === 0) {
return res.status(404).json({ error: 'Product not found' });
}
res.json(rows[0]);
} catch (error) {
console.error('Error fetching score breakdown:', error);
res.status(500).json({ error: 'Failed to fetch score breakdown' });
}
});
// GET /api/newsletter/campaigns
// Returns all campaigns with product counts and links
router.get('/campaigns', async (req, res) => {
const pool = req.app.locals.pool;
try {
const [campaignsResult, linksResult, summaryResult] = await Promise.all([
pool.query(`
SELECT
kcp.campaign_id,
kcp.campaign_name,
kcp.sent_at,
COUNT(*) as product_count,
json_agg(json_build_object(
'pid', kcp.pid,
'title', p.title,
'sku', p.sku,
'brand', p.brand,
'line', p.line,
'image', p.image_175,
'product_url', kcp.product_url
) ORDER BY p.brand, p.line, p.title) as products
FROM klaviyo_campaign_products kcp
LEFT JOIN products p ON p.pid = kcp.pid
GROUP BY kcp.campaign_id, kcp.campaign_name, kcp.sent_at
ORDER BY kcp.sent_at DESC
`),
pool.query(`
SELECT campaign_id, campaign_name, sent_at, link_url, link_type
FROM klaviyo_campaign_links
ORDER BY sent_at DESC
`),
pool.query(`
SELECT
COUNT(DISTINCT campaign_id) as total_campaigns,
COUNT(DISTINCT pid) as total_unique_products,
ROUND(COUNT(*)::numeric / NULLIF(COUNT(DISTINCT campaign_id), 0), 1) as avg_products_per_campaign
FROM klaviyo_campaign_products
`)
]);
// Group links by campaign_id
const linksByCampaign = {};
for (const link of linksResult.rows) {
if (!linksByCampaign[link.campaign_id]) linksByCampaign[link.campaign_id] = [];
linksByCampaign[link.campaign_id].push(link);
}
const campaigns = campaignsResult.rows.map(c => ({
...c,
links: linksByCampaign[c.campaign_id] || []
}));
res.json({
campaigns,
summary: summaryResult.rows[0]
});
} catch (error) {
console.error('Error fetching campaigns:', error);
res.status(500).json({ error: 'Failed to fetch campaigns' });
}
});
// GET /api/newsletter/campaigns/products
// Returns product-level aggregate stats across all campaigns
router.get('/campaigns/products', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { rows } = await pool.query(`
SELECT
kcp.pid,
p.title,
p.sku,
p.brand,
p.image_175 as image,
p.permalink,
COUNT(*) as times_featured,
MIN(kcp.sent_at) as first_featured_at,
MAX(kcp.sent_at) as last_featured_at,
EXTRACT(DAY FROM NOW() - MAX(kcp.sent_at))::int as days_since_featured,
EXTRACT(DAY FROM MAX(kcp.sent_at) - MIN(kcp.sent_at))::int as featured_span_days,
CASE WHEN COUNT(*) > 1
THEN ROUND(EXTRACT(DAY FROM MAX(kcp.sent_at) - MIN(kcp.sent_at))::numeric / (COUNT(*) - 1), 1)
ELSE NULL
END as avg_days_between_features,
json_agg(json_build_object(
'campaign_id', kcp.campaign_id,
'campaign_name', kcp.campaign_name,
'sent_at', kcp.sent_at
) ORDER BY kcp.sent_at DESC) as campaigns
FROM klaviyo_campaign_products kcp
LEFT JOIN products p ON p.pid = kcp.pid
GROUP BY kcp.pid, p.title, p.sku, p.brand, p.image_175, p.permalink
ORDER BY COUNT(*) DESC, MAX(kcp.sent_at) DESC
`);
res.json({ products: rows });
} catch (error) {
console.error('Error fetching campaign products:', error);
res.status(500).json({ error: 'Failed to fetch campaign products' });
}
});
// GET /api/newsletter/campaigns/brands
// Returns brand-level aggregate stats across all campaigns
router.get('/campaigns/brands', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { rows } = await pool.query(`
SELECT
COALESCE(p.brand, 'Unknown') as brand,
COUNT(DISTINCT kcp.pid) as product_count,
COUNT(*) as times_featured,
MIN(kcp.sent_at) as first_featured_at,
MAX(kcp.sent_at) as last_featured_at,
EXTRACT(DAY FROM NOW() - MAX(kcp.sent_at))::int as days_since_featured,
CASE WHEN COUNT(DISTINCT kcp.campaign_id) > 1
THEN ROUND(EXTRACT(DAY FROM MAX(kcp.sent_at) - MIN(kcp.sent_at))::numeric / (COUNT(DISTINCT kcp.campaign_id) - 1), 1)
ELSE NULL
END as avg_days_between_features,
json_agg(DISTINCT jsonb_build_object(
'campaign_id', kcp.campaign_id,
'campaign_name', kcp.campaign_name,
'sent_at', kcp.sent_at
)) as campaigns
FROM klaviyo_campaign_products kcp
LEFT JOIN products p ON p.pid = kcp.pid
GROUP BY COALESCE(p.brand, 'Unknown')
ORDER BY COUNT(*) DESC, MAX(kcp.sent_at) DESC
`);
res.json({ brands: rows });
} catch (error) {
console.error('Error fetching campaign brands:', error);
res.status(500).json({ error: 'Failed to fetch campaign brands' });
}
});
// GET /api/newsletter/campaigns/links
// Returns link-level aggregate stats across all campaigns
router.get('/campaigns/links', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { rows } = await pool.query(`
SELECT
link_url,
link_type,
COUNT(*) as times_used,
MIN(sent_at) as first_used_at,
MAX(sent_at) as last_used_at,
EXTRACT(DAY FROM NOW() - MAX(sent_at))::int as days_since_used,
json_agg(DISTINCT campaign_name ORDER BY campaign_name) as campaign_names
FROM klaviyo_campaign_links
GROUP BY link_url, link_type
ORDER BY COUNT(*) DESC, MAX(sent_at) DESC
`);
res.json({ links: rows });
} catch (error) {
console.error('Error fetching campaign links:', error);
res.status(500).json({ error: 'Failed to fetch campaign links' });
}
});
module.exports = router;