← back to Watches

scripts/crawl-orchestrator.js

195 lines

#!/usr/bin/env node
/**
 * Crawl Orchestrator
 * Manages the historical price crawl process with progress tracking
 */

const { spawn } = require('child_process');
const fs = require('fs');
const path = require('path');

const DATA_DIR = path.join(__dirname, '..', 'data');
const STATUS_FILE = path.join(DATA_DIR, 'crawl-status.json');
const PRICES_FILE = path.join(DATA_DIR, 'historical-prices.json');
const LOG_FILE = path.join(__dirname, '..', 'logs', 'crawl.log');

// Ensure logs directory exists
const logsDir = path.join(__dirname, '..', 'logs');
if (!fs.existsSync(logsDir)) {
  fs.mkdirSync(logsDir, { recursive: true });
}

function log(message) {
  const timestamp = new Date().toISOString();
  const line = `[${timestamp}] ${message}\n`;
  console.log(message);
  fs.appendFileSync(LOG_FILE, line);
}

function loadStatus() {
  if (fs.existsSync(STATUS_FILE)) {
    return JSON.parse(fs.readFileSync(STATUS_FILE, 'utf8'));
  }
  return {
    watchesCompleted: [],
    currentWatch: null,
    totalRequests: 0,
    errors: []
  };
}

function loadPrices() {
  if (fs.existsSync(PRICES_FILE)) {
    return JSON.parse(fs.readFileSync(PRICES_FILE, 'utf8'));
  }
  return { prices: {}, metadata: { totalRecords: 0 } };
}

function runCrawler(args = []) {
  return new Promise((resolve, reject) => {
    const python = spawn('python3', [
      path.join(__dirname, 'bulk-wayback-crawler.py'),
      ...args
    ]);

    let output = '';
    let errorOutput = '';

    python.stdout.on('data', (data) => {
      output += data.toString();
      process.stdout.write(data);
    });

    python.stderr.on('data', (data) => {
      errorOutput += data.toString();
      process.stderr.write(data);
    });

    python.on('close', (code) => {
      if (code === 0) {
        resolve(output);
      } else {
        reject(new Error(`Crawler exited with code ${code}: ${errorOutput}`));
      }
    });

    python.on('error', reject);
  });
}

async function showStatus() {
  const status = loadStatus();
  const prices = loadPrices();

  console.log('\n╔════════════════════════════════════════════════════════════╗');
  console.log('║  HISTORICAL PRICE CRAWL STATUS                             ║');
  console.log('╚════════════════════════════════════════════════════════════╝\n');

  console.log(`Total Records:     ${prices.metadata?.totalRecords || 0}`);
  console.log(`Watches Completed: ${status.watchesCompleted?.length || 0}/15`);
  console.log(`Current Watch:     ${status.currentWatch || 'None'}`);
  console.log(`Total Requests:    ${status.totalRequests || 0}`);
  console.log(`Errors:            ${status.errors?.length || 0}`);
  console.log(`Last Update:       ${prices.metadata?.lastUpdated || 'Never'}`);

  if (prices.prices) {
    console.log('\nRecords per Watch:');
    for (const [watchId, records] of Object.entries(prices.prices)) {
      const count = Array.isArray(records) ? records.length : 0;
      if (count > 0) {
        console.log(`  ${watchId}: ${count} records`);
      }
    }
  }

  if (status.errors?.length > 0) {
    console.log('\nRecent Errors:');
    status.errors.slice(-5).forEach(e => {
      console.log(`  - ${e.watch}: ${e.error}`);
    });
  }
}

async function runFullCrawl(fromYear, toYear) {
  log('Starting full historical price crawl...');
  log(`Date range: ${fromYear} - ${toYear}`);

  try {
    await runCrawler([
      '--from-year', fromYear.toString(),
      '--to-year', toYear.toString()
    ]);
    log('Crawl completed successfully!');
  } catch (error) {
    log(`Crawl failed: ${error.message}`);
    throw error;
  }
}

async function runSingleWatch(watchId, fromYear, toYear) {
  log(`Starting crawl for ${watchId}...`);

  try {
    await runCrawler([
      '--watch', watchId,
      '--from-year', fromYear.toString(),
      '--to-year', toYear.toString()
    ]);
    log(`Crawl for ${watchId} completed!`);
  } catch (error) {
    log(`Crawl for ${watchId} failed: ${error.message}`);
    throw error;
  }
}

async function main() {
  const args = process.argv.slice(2);
  const command = args[0] || 'status';

  switch (command) {
    case 'status':
      await showStatus();
      break;

    case 'start':
      const fromYear = parseInt(args[1]) || 2010;
      const toYear = parseInt(args[2]) || 2025;
      await runFullCrawl(fromYear, toYear);
      break;

    case 'watch':
      const watchId = args[1];
      if (!watchId) {
        console.error('Usage: crawl-orchestrator.js watch <watchId> [fromYear] [toYear]');
        process.exit(1);
      }
      const wFromYear = parseInt(args[2]) || 2010;
      const wToYear = parseInt(args[3]) || 2025;
      await runSingleWatch(watchId, wFromYear, wToYear);
      break;

    case 'resume':
      log('Resuming previous crawl...');
      await runCrawler([]);
      break;

    default:
      console.log(`
Usage: crawl-orchestrator.js <command> [options]

Commands:
  status              Show crawl progress and statistics
  start [from] [to]   Start full crawl (default: 2010-2025)
  watch <id> [from] [to]   Crawl single watch
  resume              Resume previous crawl

Examples:
  node crawl-orchestrator.js status
  node crawl-orchestrator.js start 2015 2025
  node crawl-orchestrator.js watch speedmaster-moonwatch 2010 2025
      `);
  }
}

main().catch(console.error);