`claude-automation/ingestion.ts`

217 lines
#!/usr/bin/env ts-node
/**
 * Claude Code Automation: Handbag Data Ingestion
 * Runs every 0.9 seconds to fetch latest handbag listings
 */

import axios from 'axios'
import { PrismaClient } from '@prisma/client'

const prisma = new PrismaClient()

interface Listing {
  sku: string
  brand: string
  model: string
  price: number
  condition: string
  imageUrl: string
  source: string
  affiliateLink?: string
}

class HandbagIngestion {
  private sources = [
    { name: 'Fashionphile', baseUrl: 'https://www.fashionphile.com', affiliateId: 'luxvault' },
    { name: 'TheRealReal', baseUrl: 'https://www.therealreal.com', affiliateId: 'luxvault' },
    { name: 'Rebag', baseUrl: 'https://shop.rebag.com', affiliateId: 'luxvault' },
    { name: 'Vestiaire', baseUrl: 'https://www.vestiairecollective.com', affiliateId: 'luxvault' },
    { name: 'StockX', baseUrl: 'https://stockx.com/handbags', affiliateId: 'luxvault' }
  ]

  private topBrands = [
    'Hermès', 'Chanel', 'Louis Vuitton', 'Dior', 'Gucci',
    'Prada', 'Fendi', 'Bottega Veneta', 'Celine', 'Goyard',
    'Balenciaga', 'Saint Laurent', 'Valentino', 'Burberry', 'Loewe',
    'Givenchy', 'Chloé', 'Mulberry', 'Proenza Schouler', 'The Row'
  ]

  private topModels = [
    'Birkin', 'Kelly', 'Constance', 'Classic Flap', 'Boy Bag',
    'Neverfull', 'Speedy', 'Alma', 'Saddle', 'Lady Dior',
    'Marmont', 'Jackie', 'Baguette', 'Peekaboo', 'Cassette',
    'Cabat', 'City', 'Puzzle', 'Belt Bag', 'Tote'
  ]

  async scrapeSource(source: any): Promise<Listing[]> {
    try {
      // This is a simplified version - in production, use proper web scraping
      // with Playwright/Puppeteer for JavaScript-heavy sites

      const listings: Listing[] = []

      // Mock data for demonstration
      // Real implementation would scrape actual websites
      for (let i = 0; i < 20; i++) {
        const brand = this.topBrands[Math.floor(Math.random() * this.topBrands.length)]
        const model = this.topModels[Math.floor(Math.random() * this.topModels.length)]

        listings.push({
          sku: `${brand.replace(/\s+/g, '_')}_${model.replace(/\s+/g, '_')}_${Date.now()}_${i}`,
          brand,
          model,
          price: Math.floor(Math.random() * 50000) + 1000,
          condition: ['New', 'Excellent', 'Very Good', 'Good'][Math.floor(Math.random() * 4)],
          imageUrl: `https://placeholder.com/handbag-${i}.jpg`,
          source: source.name,
          affiliateLink: `${source.baseUrl}/product-${i}?aff=${source.affiliateId}`
        })
      }

      return listings
    } catch (error) {
      console.error(`Error scraping ${source.name}:`, error)
      return []
    }
  }

  async normalizeListing(listing: Listing): Promise<Listing> {
    // Normalize brand and model names
    listing.brand = listing.brand.trim().toUpperCase()
    listing.model = listing.model.trim()

    // Normalize condition
    const conditionMap: Record<string, string> = {
      'NEW': 'New',
      'EXCELLENT': 'Excellent',
      'VERY GOOD': 'Very Good',
      'GOOD': 'Good'
    }
    listing.condition = conditionMap[listing.condition.toUpperCase()] || listing.condition

    return listing
  }

  async detectAnomalies(listing: Listing): Promise<boolean> {
    // Check if price is anomalously low
    // Query historical prices for this SKU
    const historicalPrices = await prisma.listing.findMany({
      where: {
        brand: listing.brand,
        model: listing.model
      },
      orderBy: { crawledAt: 'desc' },
      take: 30,
      select: { priceUsd: true }
    })

    if (historicalPrices.length > 0) {
      const avgPrice = historicalPrices.reduce((sum, p) => sum + (p.priceUsd || 0), 0) / historicalPrices.length
      const priceDropPercent = ((avgPrice - listing.price) / avgPrice) * 100

      if (priceDropPercent > 20) {
        console.log(`🚨 Anomaly detected: ${listing.brand} ${listing.model} is ${priceDropPercent.toFixed(1)}% below average`)
        return true
      }
    }

    return false
  }

  async storeListing(listing: Listing): Promise<void> {
    try {
      // Try to find existing listing first
      const existing = await prisma.listing.findFirst({
        where: { productUrl: listing.affiliateLink || '' }
      })

      if (existing) {
        await prisma.listing.update({
          where: { id: existing.id },
          data: {
            priceUsd: listing.price,
            condition: listing.condition,
            imageUrl: listing.imageUrl,
            crawledAt: new Date().toISOString()
          }
        })
      } else {
        await prisma.listing.create({
          data: {
          brand: listing.brand,
          title: `${listing.brand} ${listing.model}`,
          model: listing.model,
          priceJpy: listing.price * 150, // Convert USD to JPY (mock rate)
          priceUsd: listing.price,
          condition: listing.condition,
          imageUrl: listing.imageUrl,
          productUrl: listing.affiliateLink || '',
          listingType: 'buy-now',
          source: listing.source,
          crawledAt: new Date().toISOString()
          }
        })
      }
    } catch (error) {
      console.error('Error storing listing:', error)
    }
  }

  async run(): Promise<void> {
    console.log(`[${new Date().toISOString()}] 🔄 Starting handbag ingestion...`)

    const startTime = Date.now()
    let totalListings = 0
    let anomalies = 0
    let errors = 0

    for (const source of this.sources) {
      try {
        const listings = await this.scrapeSource(source)

        for (const listing of listings) {
          const normalized = await this.normalizeListing(listing)
          const isAnomaly = await this.detectAnomalies(normalized)

          if (isAnomaly) anomalies++

          await this.storeListing(normalized)
          totalListings++
        }
      } catch (error) {
        errors++
        console.error(`Error processing source ${source.name}:`, error)
      }
    }

    const duration = Date.now() - startTime

    console.log(`✅ Ingestion complete: ${totalListings} listings, ${anomalies} anomalies, ${errors} errors (${duration}ms)`)

    // Send Slack notification if configured
    if (process.env.SLACK_WEBHOOK_URL) {
      await this.notifySlack(totalListings, anomalies, errors, duration)
    }
  }

  async notifySlack(total: number, anomalies: number, errors: number, duration: number): Promise<void> {
    try {
      await axios.post(process.env.SLACK_WEBHOOK_URL!, {
        text: `📊 Handbag Ingestion Report\n• Total: ${total} listings\n• Anomalies: ${anomalies}\n• Errors: ${errors}\n• Duration: ${duration}ms`
      })
    } catch (error) {
      console.error('Failed to send Slack notification:', error)
    }
  }
}

// Run continuously every 0.9 seconds
const ingestion = new HandbagIngestion()

setInterval(async () => {
  await ingestion.run()
}, 900) // 0.9 seconds

// Initial run
ingestion.run()