← back to Handbag Authentication

crawler/index.js

185 lines

require('dotenv').config();
const Database = require('../db/schema');
const KomehyoScraper = require('./komehyo');
const YahooAuctionsJPScraper = require('./yahoo-jp');
const MercariJPScraper = require('./mercari');
const EbayJapanScraper = require('./ebay-japan');
const RakutenGlobalScraper = require('./rakuten-global');
const RagtagScraper = require('./ragtag');
const HermesSpecialistScraper = require('./hermes-specialist');
const LouisVuittonSpecialistScraper = require('./lv-specialist');
const FendiSpecialistScraper = require('./fendi-specialist');
const PradaSpecialistScraper = require('./prada-specialist');
const ChanelSpecialistScraper = require('./chanel-specialist');
const GovDealsScraper = require('./govdeals-scraper');
const GoodwillScraper = require('./goodwill-scraper');

const JPY_TO_USD = 150; // Update with real-time rate or API

class CrawlerOrchestrator {
  constructor() {
    this.db = new Database(process.env.DATABASE_PATH);
    this.scrapers = {
      // Brand Specialists (High-Value Targeted Scraping)
      hermes_specialist: new HermesSpecialistScraper(),
      chanel_specialist: new ChanelSpecialistScraper(),
      lv_specialist: new LouisVuittonSpecialistScraper(),
      fendi_specialist: new FendiSpecialistScraper(),
      prada_specialist: new PradaSpecialistScraper(),

      // US Government & Charity Auctions (HUGE ARBITRAGE POTENTIAL!)
      govdeals: new GovDealsScraper(),
      goodwill: new GoodwillScraper(),

      // General Multi-Brand Scrapers
      yahoo_auctions_jp: new YahooAuctionsJPScraper(),
      komehyo: new KomehyoScraper(),
      mercari_jp: new MercariJPScraper(),
      ebay_japan: new EbayJapanScraper(),
      rakuten_global: new RakutenGlobalScraper(),
      ragtag: new RagtagScraper()
    };
  }

  async crawlAll() {
    console.log('=== Starting Full Crawl ===\n');
    const startTime = Date.now();

    for (const [name, scraper] of Object.entries(this.scrapers)) {
      if (this.isScraperEnabled(name)) {
        await this.crawlSource(name, scraper);
      } else {
        console.log(`Skipping ${name} (disabled in config)`);
      }
    }

    const duration = ((Date.now() - startTime) / 1000 / 60).toFixed(2);
    console.log(`\n=== Crawl Complete in ${duration} minutes ===`);
  }

  async crawlSource(sourceName, scraper) {
    console.log(`\n--- Crawling ${sourceName} ---`);

    return new Promise((resolveMain) => {
      this.db.startCrawl(sourceName, async (err, crawl_id) => {
        if (err) {
          console.error(`Error starting crawl for ${sourceName}:`, err);
          resolveMain();
          return;
        }

        const stats = {
          items_found: 0,
          items_added: 0,
          items_updated: 0,
          status: 'completed'
        };

        try {
          // Specialist scrapers use scrapeAllModels, others use scrapeAllBrands
          const isSpecialist = sourceName.includes('_specialist');
          const listings = isSpecialist
            ? await scraper.scrapeAllModels()
            : await scraper.scrapeAllBrands();
          stats.items_found = listings.length;

          console.log(`Processing ${listings.length} listings from ${sourceName}...`);

          for (const listing of listings) {
            // Convert JPY to USD if not already set
            if (!listing.price_usd && listing.price_jpy) {
              listing.price_usd = this.convertToUSD(listing.price_jpy);
            }

            // Check if this is a new listing
            const isNew = await new Promise((resolve) => {
              this.db.isNewListing(listing, (err, isNew) => {
                resolve(err ? false : isNew);
              });
            });

            listing.is_new = isNew ? 1 : 0;

            // Upsert to database
            await new Promise((resolve) => {
              this.db.upsertListing(listing, (err) => {
                if (err) {
                  console.error('Error saving listing:', err);
                } else {
                  stats.items_added++;
                  if (isNew) {
                    stats.new_items = (stats.new_items || 0) + 1;
                  }
                }
                resolve();
              });
            });
          }

        } catch (error) {
          console.error(`Error during ${sourceName} crawl:`, error);
          stats.status = 'failed';
        }

        // Update crawl history
        this.db.endCrawl(crawl_id, stats, (err) => {
          if (err) console.error('Error updating crawl history:', err);
          const newItemsMsg = stats.new_items ? ` (🆕 ${stats.new_items} NEW)` : '';
          console.log(`${sourceName} complete: ${stats.items_added} items saved${newItemsMsg}`);
          resolveMain();
        });
      });
    });
  }

  isScraperEnabled(name) {
    const envMap = {
      // Specialists
      hermes_specialist: 'ENABLE_HERMES_SPECIALIST',
      chanel_specialist: 'ENABLE_CHANEL_SPECIALIST',
      lv_specialist: 'ENABLE_LV_SPECIALIST',
      fendi_specialist: 'ENABLE_FENDI_SPECIALIST',
      prada_specialist: 'ENABLE_PRADA_SPECIALIST',

      // General
      ragtag: 'ENABLE_RAGTAG',
      komehyo: 'ENABLE_KOMEHYO',
      yahoo_auctions_jp: 'ENABLE_YAHOO_AUCTIONS',
      mercari_jp: 'ENABLE_MERCARI_JP',
      ebay_japan: 'ENABLE_EBAY_JAPAN',
      rakuten_global: 'ENABLE_RAKUTEN_GLOBAL'
    };

    return process.env[envMap[name]] !== 'false';
  }

  convertToUSD(jpy) {
    return Math.round((jpy / JPY_TO_USD) * 100) / 100;
  }

  close() {
    this.db.close();
  }
}

// Run if called directly
if (require.main === module) {
  const crawler = new CrawlerOrchestrator();

  crawler.crawlAll()
    .then(() => {
      console.log('Crawler finished successfully');
      setTimeout(() => {
        crawler.close();
        process.exit(0);
      }, 2000);
    })
    .catch((error) => {
      console.error('Crawler failed:', error);
      crawler.close();
      process.exit(1);
    });
}

module.exports = CrawlerOrchestrator;