← back to Handbag Auth Nextjs

scripts/pollHandbagRss.ts

217 lines

// RSS Feed Polling Script for Handbag Marketplace Data
// Polls configured RSS feeds, normalizes items, and stores new listings

import fs from "fs";
import path from "path";
import crypto from "crypto";
import type {
  HandbagFeedItem,
  HandbagRssSource,
} from "../src/types/handbagRss";

// RSS Parser types (will work once installed)
interface RSSItem {
  title?: string;
  link?: string;
  guid?: string;
  isoDate?: string;
  pubDate?: string;
  contentSnippet?: string;
  content?: string;
  enclosure?: { url?: string };
}

interface RSSFeed {
  items: RSSItem[];
}

// Import RSS parser
import RSSParser from "rss-parser";
const parser = new RSSParser();

function loadSources(): HandbagRssSource[] {
  const file = path.resolve("data/handbag_rss_sources.json");
  if (!fs.existsSync(file)) {
    console.warn("No RSS sources file found at:", file);
    return [];
  }
  return JSON.parse(fs.readFileSync(file, "utf8"));
}

function loadExistingItems(): HandbagFeedItem[] {
  const file = path.resolve("data/handbag_rss_items.json");
  if (!fs.existsSync(file)) return [];
  return JSON.parse(fs.readFileSync(file, "utf8"));
}

function saveItems(items: HandbagFeedItem[]) {
  const file = path.resolve("data/handbag_rss_items.json");
  fs.writeFileSync(file, JSON.stringify(items, null, 2));
}

function makeId(sourceId: string, link: string, guid?: string): string {
  const base = guid || link;
  return `${sourceId}-${crypto
    .createHash("md5")
    .update(base)
    .digest("hex")}`;
}

// Enhanced metadata extraction from title and content
function extractMetadataFromTitle(title: string, content?: string) {
  const lower = title.toLowerCase();
  const combined = (title + " " + (content || "")).toLowerCase();

  // Brand detection
  const brands = [
    "hermès", "hermes", "chanel", "louis vuitton", "lv", "gucci", "prada",
    "dior", "fendi", "bottega veneta", "celine", "valentino", "givenchy",
    "balenciaga", "saint laurent", "ysl", "burberry", "mulberry", "goyard"
  ];
  let brand: string | undefined;
  for (const b of brands) {
    if (combined.includes(b)) {
      brand = b;
      break;
    }
  }

  // Condition detection
  let condition: "new" | "used" | "unknown" = "unknown";
  if (lower.includes("new") || lower.includes("nwt")) {
    condition = "new";
  } else if (
    lower.includes("excellent") ||
    lower.includes("very good") ||
    lower.includes("good") ||
    lower.includes("pre-owned") ||
    lower.includes("used") ||
    lower.includes("vintage")
  ) {
    condition = "used";
  }

  // Price extraction (basic)
  const priceMatch = combined.match(/\$[\d,]+(?:\.\d{2})?/);
  let price: number | undefined;
  let currency = "USD";
  if (priceMatch) {
    price = parseFloat(priceMatch[0].replace(/[$,]/g, ""));
  }

  return {
    brand,
    condition,
    price,
    currency: price ? currency : undefined,
  };
}

async function pollSource(
  source: HandbagRssSource,
  existingById: Map<string, HandbagFeedItem>
): Promise<HandbagFeedItem[]> {
  if (!source.active) return [];

  console.log(`Polling ${source.name} (${source.feedUrl})`);

  try {
    const feed: RSSFeed = await parser.parseURL(source.feedUrl);
    const newItems: HandbagFeedItem[] = [];

    for (const entry of feed.items) {
      const link = entry.link || "";
      if (!link) continue;

      const id = makeId(source.id, link, entry.guid);
      if (existingById.has(id)) continue;

      const title = entry.title || link;
      const content = entry.contentSnippet || entry.content;
      const meta = extractMetadataFromTitle(title, content);

      // Extract image from enclosure or content
      let imageUrl: string | undefined;
      if (entry.enclosure?.url) {
        imageUrl = entry.enclosure.url;
      }

      const item: HandbagFeedItem = {
        id,
        sourceId: source.id,
        title,
        link,
        publishedAt: entry.isoDate || entry.pubDate,
        summary: content,
        imageUrl,
        brand: meta.brand,
        condition: meta.condition,
        price: meta.price,
        currency: meta.currency,
      };

      newItems.push(item);
    }

    return newItems;
  } catch (err) {
    console.error(`Error polling ${source.name}:`, err);
    return [];
  }
}

async function main() {
  const sources = loadSources();
  if (sources.length === 0) {
    console.log("No RSS sources configured.");
    return;
  }

  const existing = loadExistingItems();
  const existingById = new Map(existing.map((i) => [i.id, i]));

  let newCount = 0;
  const allNewItems: HandbagFeedItem[] = [];

  for (const source of sources) {
    const items = await pollSource(source, existingById);
    for (const item of items) {
      existingById.set(item.id, item);
      existing.push(item);
      allNewItems.push(item);
      newCount++;
    }
  }

  if (newCount > 0) {
    saveItems(existing);

    // Log summary
    console.log(`\n=== RSS Poll Complete ===`);
    console.log(`New items: ${newCount}`);
    console.log(`Total items: ${existing.length}`);

    // Show sample of new items
    console.log(`\nSample of new items:`);
    allNewItems.slice(0, 5).forEach((item) => {
      console.log(`- ${item.title}`);
      console.log(`  ${item.link}`);
      if (item.price) console.log(`  Price: ${item.currency}${item.price}`);
      console.log();
    });
  } else {
    console.log("No new items found.");
  }

  console.log(`Done. New handbag RSS items: ${newCount}`);
}

const isMainModule = import.meta.url === `file://${process.argv[1]}`;

if (isMainModule) {
  main().catch((err) => {
    console.error(err);
    process.exit(1);
  });
}