diff --git a/inventory-server/dashboard/klaviyo-server/package-lock.json b/inventory-server/dashboard/klaviyo-server/package-lock.json index 5fd3a9e..04b0e11 100644 --- a/inventory-server/dashboard/klaviyo-server/package-lock.json +++ b/inventory-server/dashboard/klaviyo-server/package-lock.json @@ -16,6 +16,7 @@ "ioredis": "^5.4.1", "luxon": "^3.5.0", "node-fetch": "^3.3.2", + "pg": "^8.18.0", "recharts": "^2.15.0" }, "devDependencies": { @@ -1379,6 +1380,95 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.18.0.tgz", + "integrity": "sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.11.0", + "pg-protocol": "^1.11.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.11.0.tgz", + "integrity": "sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.11.0.tgz", + "integrity": "sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -1392,6 +1482,45 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -1809,6 +1938,15 @@ "node": ">=10" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", @@ -1952,6 +2090,15 @@ "engines": { "node": ">= 8" } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/inventory-server/dashboard/klaviyo-server/package.json b/inventory-server/dashboard/klaviyo-server/package.json index 021f450..e5bee1d 100644 --- a/inventory-server/dashboard/klaviyo-server/package.json +++ b/inventory-server/dashboard/klaviyo-server/package.json @@ -17,6 +17,7 @@ "ioredis": "^5.4.1", "luxon": "^3.5.0", "node-fetch": "^3.3.2", + "pg": "^8.18.0", "recharts": "^2.15.0" }, "devDependencies": { diff --git a/inventory-server/dashboard/klaviyo-server/scripts/create-campaign-products-table.sql b/inventory-server/dashboard/klaviyo-server/scripts/create-campaign-products-table.sql new file mode 100644 index 0000000..0e564d9 --- /dev/null +++ b/inventory-server/dashboard/klaviyo-server/scripts/create-campaign-products-table.sql @@ -0,0 +1,30 @@ +-- Stores individual product links found in Klaviyo campaign emails +CREATE TABLE IF NOT EXISTS klaviyo_campaign_products ( + id SERIAL PRIMARY KEY, + campaign_id TEXT NOT NULL, + campaign_name TEXT, + sent_at TIMESTAMPTZ, + pid BIGINT NOT NULL, + product_url TEXT, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(campaign_id, pid) +); + +CREATE INDEX IF NOT EXISTS idx_kcp_campaign_id ON klaviyo_campaign_products(campaign_id); +CREATE INDEX IF NOT EXISTS idx_kcp_pid ON klaviyo_campaign_products(pid); +CREATE INDEX IF NOT EXISTS idx_kcp_sent_at ON klaviyo_campaign_products(sent_at); + +-- Stores non-product shop links (categories, filters, etc.) found in campaigns +CREATE TABLE IF NOT EXISTS klaviyo_campaign_links ( + id SERIAL PRIMARY KEY, + campaign_id TEXT NOT NULL, + campaign_name TEXT, + sent_at TIMESTAMPTZ, + link_url TEXT NOT NULL, + link_type TEXT, -- 'category', 'brand', 'filter', 'clearance', 'deals', 'other' + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(campaign_id, link_url) +); + +CREATE INDEX IF NOT EXISTS idx_kcl_campaign_id ON klaviyo_campaign_links(campaign_id); +CREATE INDEX IF NOT EXISTS idx_kcl_sent_at ON klaviyo_campaign_links(sent_at); diff --git a/inventory-server/dashboard/klaviyo-server/scripts/import-campaign-products.js b/inventory-server/dashboard/klaviyo-server/scripts/import-campaign-products.js new file mode 100644 index 0000000..eccf643 --- /dev/null +++ b/inventory-server/dashboard/klaviyo-server/scripts/import-campaign-products.js @@ -0,0 +1,251 @@ +/** + * Extract products featured in Klaviyo campaign emails and store in DB. + * + * - Fetches recent sent campaigns from Klaviyo API + * - Gets template HTML for each campaign message + * - Parses out product links (/shop/{id}) and other shop links + * - Inserts into klaviyo_campaign_products and klaviyo_campaign_links tables + * + * Usage: node scripts/poc-campaign-products.js [limit] + * limit: number of recent campaigns to process (default: 10) + * + * Requires DB_* env vars (from inventory-server .env) and KLAVIYO_API_KEY. + */ + +import fetch from 'node-fetch'; +import pg from 'pg'; +import dotenv from 'dotenv'; +import path from 'path'; +import fs from 'fs'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// Load klaviyo .env for API key +dotenv.config({ path: path.resolve(__dirname, '../.env') }); +// Also load the main inventory-server .env for DB credentials +const mainEnvPath = '/var/www/html/inventory/.env'; +if (fs.existsSync(mainEnvPath)) { + dotenv.config({ path: mainEnvPath }); +} + +const API_KEY = process.env.KLAVIYO_API_KEY; +const REVISION = process.env.KLAVIYO_API_REVISION || '2026-01-15'; +const BASE_URL = 'https://a.klaviyo.com/api'; + +if (!API_KEY) { + console.error('KLAVIYO_API_KEY not set in .env'); + process.exit(1); +} + +// ── Klaviyo API helpers ────────────────────────────────────────────── + +const headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Authorization': `Klaviyo-API-Key ${API_KEY}`, + 'revision': REVISION, +}; + +async function klaviyoGet(endpoint, params = {}) { + const url = new URL(`${BASE_URL}${endpoint}`); + for (const [k, v] of Object.entries(params)) { + url.searchParams.append(k, v); + } + const res = await fetch(url.toString(), { headers }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Klaviyo ${res.status} on ${endpoint}: ${body}`); + } + return res.json(); +} + +async function getRecentCampaigns(limit) { + const data = await klaviyoGet('/campaigns', { + 'filter': 'equals(messages.channel,"email")', + 'sort': '-scheduled_at', + 'include': 'campaign-messages', + }); + + const campaigns = (data.data || []) + .filter(c => c.attributes?.status === 'Sent') + .slice(0, limit); + + const messageMap = {}; + for (const inc of (data.included || [])) { + if (inc.type === 'campaign-message') { + messageMap[inc.id] = inc; + } + } + + return { campaigns, messageMap }; +} + +async function getTemplateHtml(messageId) { + const data = await klaviyoGet(`/campaign-messages/${messageId}/template`, { + 'fields[template]': 'html,name', + }); + return { + templateId: data.data?.id, + templateName: data.data?.attributes?.name, + html: data.data?.attributes?.html || '', + }; +} + +// ── HTML parsing ───────────────────────────────────────────────────── + +function parseProductsFromHtml(html) { + const seen = new Set(); + const products = []; + + const linkRegex = /href="([^"]*acherryontop\.com\/shop\/(\d+))[^"]*"/gi; + let match; + while ((match = linkRegex.exec(html)) !== null) { + const productId = match[2]; + if (!seen.has(productId)) { + seen.add(productId); + products.push({ + siteProductId: productId, + url: match[1], + }); + } + } + + const categoryLinks = []; + const catRegex = /href="([^"]*acherryontop\.com\/shop\/[^"]+)"/gi; + while ((match = catRegex.exec(html)) !== null) { + const url = match[1]; + if (/\/shop\/\d+$/.test(url)) continue; + if (!categoryLinks.includes(url)) categoryLinks.push(url); + } + + return { products, categoryLinks }; +} + +function classifyLink(url) { + if (/\/shop\/(new|pre-order|backinstock)/.test(url)) return 'filter'; + if (/\/shop\/company\//.test(url)) return 'brand'; + if (/\/shop\/clearance/.test(url)) return 'clearance'; + if (/\/shop\/daily_deals/.test(url)) return 'deals'; + if (/\/shop\/category\//.test(url)) return 'category'; + return 'other'; +} + +// ── Database ───────────────────────────────────────────────────────── + +function createPool() { + return new pg.Pool({ + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + port: process.env.DB_PORT || 5432, + ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false, + }); +} + +async function insertProducts(pool, campaignId, campaignName, sentAt, products) { + if (products.length === 0) return 0; + + let inserted = 0; + for (const p of products) { + try { + await pool.query( + `INSERT INTO klaviyo_campaign_products + (campaign_id, campaign_name, sent_at, pid, product_url) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (campaign_id, pid) DO NOTHING`, + [campaignId, campaignName, sentAt, parseInt(p.siteProductId), p.url] + ); + inserted++; + } catch (err) { + console.error(` Error inserting product ${p.siteProductId}: ${err.message}`); + } + } + return inserted; +} + +async function insertLinks(pool, campaignId, campaignName, sentAt, links) { + if (links.length === 0) return 0; + + let inserted = 0; + for (const url of links) { + try { + await pool.query( + `INSERT INTO klaviyo_campaign_links + (campaign_id, campaign_name, sent_at, link_url, link_type) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (campaign_id, link_url) DO NOTHING`, + [campaignId, campaignName, sentAt, url, classifyLink(url)] + ); + inserted++; + } catch (err) { + console.error(` Error inserting link: ${err.message}`); + } + } + return inserted; +} + +// ── Main ───────────────────────────────────────────────────────────── + +async function main() { + const limit = parseInt(process.argv[2]) || 10; + const pool = createPool(); + + try { + // Fetch campaigns + console.log(`Fetching up to ${limit} recent campaigns...\n`); + const { campaigns, messageMap } = await getRecentCampaigns(limit); + console.log(`Found ${campaigns.length} sent campaigns.\n`); + + let totalProducts = 0; + let totalLinks = 0; + + for (const campaign of campaigns) { + const name = campaign.attributes?.name || 'Unnamed'; + const sentAt = campaign.attributes?.send_time; + + console.log(`━━━ ${name} (${sentAt?.slice(0, 10) || 'no date'}) ━━━`); + + const msgIds = (campaign.relationships?.['campaign-messages']?.data || []) + .map(r => r.id); + + if (msgIds.length === 0) { + console.log(' No messages.\n'); + continue; + } + + for (const msgId of msgIds) { + const msg = messageMap[msgId]; + const subject = msg?.attributes?.definition?.content?.subject; + if (subject) console.log(` Subject: ${subject}`); + + try { + const template = await getTemplateHtml(msgId); + const { products, categoryLinks } = parseProductsFromHtml(template.html); + + const pInserted = await insertProducts(pool, campaign.id, name, sentAt, products); + const lInserted = await insertLinks(pool, campaign.id, name, sentAt, categoryLinks); + + console.log(` ${products.length} products (${pInserted} new), ${categoryLinks.length} links (${lInserted} new)`); + totalProducts += pInserted; + totalLinks += lInserted; + + await new Promise(r => setTimeout(r, 200)); + } catch (err) { + console.log(` Error: ${err.message}`); + } + } + console.log(''); + } + + console.log(`Done. Inserted ${totalProducts} product rows, ${totalLinks} link rows.`); + } finally { + await pool.end(); + } +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +});