Add AI name/description validation to product editor
This commit is contained in:
@@ -1,30 +0,0 @@
|
||||
-- Stores individual product links found in Klaviyo campaign emails
|
||||
CREATE TABLE IF NOT EXISTS klaviyo_campaign_products (
|
||||
id SERIAL PRIMARY KEY,
|
||||
campaign_id TEXT NOT NULL,
|
||||
campaign_name TEXT,
|
||||
sent_at TIMESTAMPTZ,
|
||||
pid BIGINT NOT NULL,
|
||||
product_url TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(campaign_id, pid)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_campaign_id ON klaviyo_campaign_products(campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_pid ON klaviyo_campaign_products(pid);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_sent_at ON klaviyo_campaign_products(sent_at);
|
||||
|
||||
-- Stores non-product shop links (categories, filters, etc.) found in campaigns
|
||||
CREATE TABLE IF NOT EXISTS klaviyo_campaign_links (
|
||||
id SERIAL PRIMARY KEY,
|
||||
campaign_id TEXT NOT NULL,
|
||||
campaign_name TEXT,
|
||||
sent_at TIMESTAMPTZ,
|
||||
link_url TEXT NOT NULL,
|
||||
link_type TEXT, -- 'category', 'brand', 'filter', 'clearance', 'deals', 'other'
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(campaign_id, link_url)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kcl_campaign_id ON klaviyo_campaign_links(campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcl_sent_at ON klaviyo_campaign_links(sent_at);
|
||||
@@ -1,279 +0,0 @@
|
||||
/**
|
||||
* Extract products featured in Klaviyo campaign emails and store in DB.
|
||||
*
|
||||
* - Fetches recent sent campaigns from Klaviyo API
|
||||
* - Gets template HTML for each campaign message
|
||||
* - Parses out product links (/shop/{id}) and other shop links
|
||||
* - Inserts into klaviyo_campaign_products and klaviyo_campaign_links tables
|
||||
*
|
||||
* Usage: node scripts/poc-campaign-products.js [limit] [offset]
|
||||
* limit: number of sent campaigns to process (default: 10)
|
||||
* offset: number of sent campaigns to skip before processing (default: 0)
|
||||
*
|
||||
* Requires DB_* env vars (from inventory-server .env) and KLAVIYO_API_KEY.
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import pg from 'pg';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Load klaviyo .env for API key
|
||||
dotenv.config({ path: path.resolve(__dirname, '../.env') });
|
||||
// Also load the main inventory-server .env for DB credentials
|
||||
const mainEnvPath = '/var/www/html/inventory/.env';
|
||||
if (fs.existsSync(mainEnvPath)) {
|
||||
dotenv.config({ path: mainEnvPath });
|
||||
}
|
||||
|
||||
const API_KEY = process.env.KLAVIYO_API_KEY;
|
||||
const REVISION = process.env.KLAVIYO_API_REVISION || '2026-01-15';
|
||||
const BASE_URL = 'https://a.klaviyo.com/api';
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('KLAVIYO_API_KEY not set in .env');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Klaviyo API helpers ──────────────────────────────────────────────
|
||||
|
||||
const headers = {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Klaviyo-API-Key ${API_KEY}`,
|
||||
'revision': REVISION,
|
||||
};
|
||||
|
||||
async function klaviyoGet(endpoint, params = {}) {
|
||||
const url = new URL(`${BASE_URL}${endpoint}`);
|
||||
for (const [k, v] of Object.entries(params)) {
|
||||
url.searchParams.append(k, v);
|
||||
}
|
||||
return klaviyoFetch(url.toString());
|
||||
}
|
||||
|
||||
async function klaviyoFetch(url) {
|
||||
const res = await fetch(url, { headers });
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Klaviyo ${res.status} on ${url}: ${body}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function getRecentCampaigns(limit, offset = 0) {
|
||||
const campaigns = [];
|
||||
const messageMap = {};
|
||||
let skipped = 0;
|
||||
|
||||
let data = await klaviyoGet('/campaigns', {
|
||||
'filter': 'equals(messages.channel,"email")',
|
||||
'sort': '-scheduled_at',
|
||||
'include': 'campaign-messages',
|
||||
});
|
||||
|
||||
while (true) {
|
||||
for (const c of (data.data || [])) {
|
||||
if (c.attributes?.status === 'Sent') {
|
||||
if (skipped < offset) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
campaigns.push(c);
|
||||
if (campaigns.length >= limit) break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const inc of (data.included || [])) {
|
||||
if (inc.type === 'campaign-message') {
|
||||
messageMap[inc.id] = inc;
|
||||
}
|
||||
}
|
||||
|
||||
const nextUrl = data.links?.next;
|
||||
if (campaigns.length >= limit || !nextUrl) break;
|
||||
|
||||
const progress = skipped < offset
|
||||
? `Skipped ${skipped}/${offset}...`
|
||||
: `Fetched ${campaigns.length}/${limit} sent campaigns, loading next page...`;
|
||||
console.log(` ${progress}`);
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
data = await klaviyoFetch(nextUrl);
|
||||
}
|
||||
|
||||
return { campaigns: campaigns.slice(0, limit), messageMap };
|
||||
}
|
||||
|
||||
async function getTemplateHtml(messageId) {
|
||||
const data = await klaviyoGet(`/campaign-messages/${messageId}/template`, {
|
||||
'fields[template]': 'html,name',
|
||||
});
|
||||
return {
|
||||
templateId: data.data?.id,
|
||||
templateName: data.data?.attributes?.name,
|
||||
html: data.data?.attributes?.html || '',
|
||||
};
|
||||
}
|
||||
|
||||
// ── HTML parsing ─────────────────────────────────────────────────────
|
||||
|
||||
function parseProductsFromHtml(html) {
|
||||
const seen = new Set();
|
||||
const products = [];
|
||||
|
||||
const linkRegex = /href="([^"]*acherryontop\.com\/shop\/(\d+))[^"]*"/gi;
|
||||
let match;
|
||||
while ((match = linkRegex.exec(html)) !== null) {
|
||||
const productId = match[2];
|
||||
if (!seen.has(productId)) {
|
||||
seen.add(productId);
|
||||
products.push({
|
||||
siteProductId: productId,
|
||||
url: match[1],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const categoryLinks = [];
|
||||
const catRegex = /href="([^"]*acherryontop\.com\/shop\/[^"]+)"/gi;
|
||||
while ((match = catRegex.exec(html)) !== null) {
|
||||
const url = match[1];
|
||||
if (/\/shop\/\d+$/.test(url)) continue;
|
||||
if (!categoryLinks.includes(url)) categoryLinks.push(url);
|
||||
}
|
||||
|
||||
return { products, categoryLinks };
|
||||
}
|
||||
|
||||
function classifyLink(url) {
|
||||
if (/\/shop\/(new|pre-order|backinstock)/.test(url)) return 'filter';
|
||||
if (/\/shop\/company\//.test(url)) return 'brand';
|
||||
if (/\/shop\/clearance/.test(url)) return 'clearance';
|
||||
if (/\/shop\/daily_deals/.test(url)) return 'deals';
|
||||
if (/\/shop\/category\//.test(url)) return 'category';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
// ── Database ─────────────────────────────────────────────────────────
|
||||
|
||||
function createPool() {
|
||||
return new pg.Pool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
port: process.env.DB_PORT || 5432,
|
||||
ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
|
||||
});
|
||||
}
|
||||
|
||||
async function insertProducts(pool, campaignId, campaignName, sentAt, products) {
|
||||
if (products.length === 0) return 0;
|
||||
|
||||
let inserted = 0;
|
||||
for (const p of products) {
|
||||
try {
|
||||
await pool.query(
|
||||
`INSERT INTO klaviyo_campaign_products
|
||||
(campaign_id, campaign_name, sent_at, pid, product_url)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (campaign_id, pid) DO NOTHING`,
|
||||
[campaignId, campaignName, sentAt, parseInt(p.siteProductId), p.url]
|
||||
);
|
||||
inserted++;
|
||||
} catch (err) {
|
||||
console.error(` Error inserting product ${p.siteProductId}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
async function insertLinks(pool, campaignId, campaignName, sentAt, links) {
|
||||
if (links.length === 0) return 0;
|
||||
|
||||
let inserted = 0;
|
||||
for (const url of links) {
|
||||
try {
|
||||
await pool.query(
|
||||
`INSERT INTO klaviyo_campaign_links
|
||||
(campaign_id, campaign_name, sent_at, link_url, link_type)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (campaign_id, link_url) DO NOTHING`,
|
||||
[campaignId, campaignName, sentAt, url, classifyLink(url)]
|
||||
);
|
||||
inserted++;
|
||||
} catch (err) {
|
||||
console.error(` Error inserting link: ${err.message}`);
|
||||
}
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const limit = parseInt(process.argv[2]) || 10;
|
||||
const offset = parseInt(process.argv[3]) || 0;
|
||||
const pool = createPool();
|
||||
|
||||
try {
|
||||
// Fetch campaigns
|
||||
console.log(`Fetching up to ${limit} recent campaigns (offset: ${offset})...\n`);
|
||||
const { campaigns, messageMap } = await getRecentCampaigns(limit, offset);
|
||||
console.log(`Found ${campaigns.length} sent campaigns.\n`);
|
||||
|
||||
let totalProducts = 0;
|
||||
let totalLinks = 0;
|
||||
|
||||
for (const campaign of campaigns) {
|
||||
const name = campaign.attributes?.name || 'Unnamed';
|
||||
const sentAt = campaign.attributes?.send_time;
|
||||
|
||||
console.log(`━━━ ${name} (${sentAt?.slice(0, 10) || 'no date'}) ━━━`);
|
||||
|
||||
const msgIds = (campaign.relationships?.['campaign-messages']?.data || [])
|
||||
.map(r => r.id);
|
||||
|
||||
if (msgIds.length === 0) {
|
||||
console.log(' No messages.\n');
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const msgId of msgIds) {
|
||||
const msg = messageMap[msgId];
|
||||
const subject = msg?.attributes?.definition?.content?.subject;
|
||||
if (subject) console.log(` Subject: ${subject}`);
|
||||
|
||||
try {
|
||||
const template = await getTemplateHtml(msgId);
|
||||
const { products, categoryLinks } = parseProductsFromHtml(template.html);
|
||||
|
||||
const pInserted = await insertProducts(pool, campaign.id, name, sentAt, products);
|
||||
const lInserted = await insertLinks(pool, campaign.id, name, sentAt, categoryLinks);
|
||||
|
||||
console.log(` ${products.length} products (${pInserted} new), ${categoryLinks.length} links (${lInserted} new)`);
|
||||
totalProducts += pInserted;
|
||||
totalLinks += lInserted;
|
||||
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
} catch (err) {
|
||||
console.log(` Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
console.log(`Done. Inserted ${totalProducts} product rows, ${totalLinks} link rows.`);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user