Add newsletter recommendations

inventory-server/dashboard/klaviyo-server/scripts/import-campaign-products.js

@@ -6,8 +6,9 @@
  * - Parses out product links (/shop/{id}) and other shop links
  * - Inserts into klaviyo_campaign_products and klaviyo_campaign_links tables
  *
- * Usage: node scripts/poc-campaign-products.js [limit]
- *   limit: number of recent campaigns to process (default: 10)
+ * Usage: node scripts/poc-campaign-products.js [limit] [offset]
+ *   limit:  number of sent campaigns to process (default: 10)
+ *   offset: number of sent campaigns to skip before processing (default: 0)
  *
  * Requires DB_* env vars (from inventory-server .env) and KLAVIYO_API_KEY.
  */
@@ -52,33 +53,59 @@ async function klaviyoGet(endpoint, params = {}) {
   for (const [k, v] of Object.entries(params)) {
     url.searchParams.append(k, v);
   }
-  const res = await fetch(url.toString(), { headers });
+  return klaviyoFetch(url.toString());
+}
+
+async function klaviyoFetch(url) {
+  const res = await fetch(url, { headers });
   if (!res.ok) {
     const body = await res.text();
-    throw new Error(`Klaviyo ${res.status} on ${endpoint}: ${body}`);
+    throw new Error(`Klaviyo ${res.status} on ${url}: ${body}`);
   }
   return res.json();
 }
 
-async function getRecentCampaigns(limit) {
-  const data = await klaviyoGet('/campaigns', {
+async function getRecentCampaigns(limit, offset = 0) {
+  const campaigns = [];
+  const messageMap = {};
+  let skipped = 0;
+
+  let data = await klaviyoGet('/campaigns', {
     'filter': 'equals(messages.channel,"email")',
     'sort': '-scheduled_at',
     'include': 'campaign-messages',
   });
 
-  const campaigns = (data.data || [])
-    .filter(c => c.attributes?.status === 'Sent')
-    .slice(0, limit);
-
-  const messageMap = {};
-  for (const inc of (data.included || [])) {
-    if (inc.type === 'campaign-message') {
-      messageMap[inc.id] = inc;
+  while (true) {
+    for (const c of (data.data || [])) {
+      if (c.attributes?.status === 'Sent') {
+        if (skipped < offset) {
+          skipped++;
+          continue;
+        }
+        campaigns.push(c);
+        if (campaigns.length >= limit) break;
+      }
     }
+
+    for (const inc of (data.included || [])) {
+      if (inc.type === 'campaign-message') {
+        messageMap[inc.id] = inc;
+      }
+    }
+
+    const nextUrl = data.links?.next;
+    if (campaigns.length >= limit || !nextUrl) break;
+
+    const progress = skipped < offset
+      ? `Skipped ${skipped}/${offset}...`
+      : `Fetched ${campaigns.length}/${limit} sent campaigns, loading next page...`;
+    console.log(`  ${progress}`);
+    await new Promise(r => setTimeout(r, 200));
+    data = await klaviyoFetch(nextUrl);
   }
 
-  return { campaigns, messageMap };
+  return { campaigns: campaigns.slice(0, limit), messageMap };
 }
 
 async function getTemplateHtml(messageId) {
@@ -190,12 +217,13 @@ async function insertLinks(pool, campaignId, campaignName, sentAt, links) {
 
 async function main() {
   const limit = parseInt(process.argv[2]) || 10;
+  const offset = parseInt(process.argv[3]) || 0;
   const pool = createPool();
 
   try {
     // Fetch campaigns
-    console.log(`Fetching up to ${limit} recent campaigns...\n`);
-    const { campaigns, messageMap } = await getRecentCampaigns(limit);
+    console.log(`Fetching up to ${limit} recent campaigns (offset: ${offset})...\n`);
+    const { campaigns, messageMap } = await getRecentCampaigns(limit, offset);
     console.log(`Found ${campaigns.length} sent campaigns.\n`);
 
     let totalProducts = 0;