← back to Watches

scripts/aggregate-prices.js

218 lines

#!/usr/bin/env node
/**
 * Price Aggregation Pipeline
 * Processes raw historical price data into monthly/yearly aggregates
 */

const fs = require('fs');
const path = require('path');

const DATA_DIR = path.join(__dirname, '..', 'data');
const INPUT_FILE = path.join(DATA_DIR, 'historical-prices.json');
const OUTPUT_FILE = path.join(DATA_DIR, 'aggregated-prices.json');

function loadRawData() {
  if (!fs.existsSync(INPUT_FILE)) {
    console.error('No historical prices data found. Run the crawler first.');
    process.exit(1);
  }
  return JSON.parse(fs.readFileSync(INPUT_FILE, 'utf8'));
}

function detectOutliers(prices) {
  if (prices.length < 4) return { valid: prices, outliers: [] };

  // Sort prices
  const sorted = [...prices].sort((a, b) => a - b);

  // Calculate Q1, Q3, IQR
  const q1Index = Math.floor(sorted.length * 0.25);
  const q3Index = Math.floor(sorted.length * 0.75);
  const q1 = sorted[q1Index];
  const q3 = sorted[q3Index];
  const iqr = q3 - q1;

  // Outlier bounds (1.5 * IQR)
  const lowerBound = q1 - 1.5 * iqr;
  const upperBound = q3 + 1.5 * iqr;

  const valid = prices.filter(p => p >= lowerBound && p <= upperBound);
  const outliers = prices.filter(p => p < lowerBound || p > upperBound);

  return { valid, outliers };
}

function aggregateByMonth(records) {
  const byMonth = {};

  for (const record of records) {
    const month = record.date.substring(0, 7); // YYYY-MM
    if (!byMonth[month]) {
      byMonth[month] = [];
    }
    byMonth[month].push(record.price);
  }

  const aggregated = [];
  for (const [month, prices] of Object.entries(byMonth)) {
    const { valid, outliers } = detectOutliers(prices);

    if (valid.length > 0) {
      aggregated.push({
        month,
        min: Math.min(...valid),
        max: Math.max(...valid),
        avg: Math.round(valid.reduce((a, b) => a + b, 0) / valid.length),
        median: valid.sort((a, b) => a - b)[Math.floor(valid.length / 2)],
        count: valid.length,
        outlierCount: outliers.length
      });
    }
  }

  return aggregated.sort((a, b) => a.month.localeCompare(b.month));
}

function aggregateByYear(records) {
  const byYear = {};

  for (const record of records) {
    const year = record.date.substring(0, 4);
    if (!byYear[year]) {
      byYear[year] = [];
    }
    byYear[year].push(record.price);
  }

  const aggregated = [];
  for (const [year, prices] of Object.entries(byYear)) {
    const { valid, outliers } = detectOutliers(prices);

    if (valid.length > 0) {
      aggregated.push({
        year: parseInt(year),
        min: Math.min(...valid),
        max: Math.max(...valid),
        avg: Math.round(valid.reduce((a, b) => a + b, 0) / valid.length),
        median: valid.sort((a, b) => a - b)[Math.floor(valid.length / 2)],
        count: valid.length,
        outlierCount: outliers.length
      });
    }
  }

  return aggregated.sort((a, b) => a.year - b.year);
}

function calculateTrends(yearlyData) {
  if (yearlyData.length < 2) return null;

  const trends = {
    overall: {},
    yearOverYear: []
  };

  // Overall trend (first to last)
  const first = yearlyData[0];
  const last = yearlyData[yearlyData.length - 1];
  trends.overall = {
    startYear: first.year,
    endYear: last.year,
    startPrice: first.avg,
    endPrice: last.avg,
    absoluteChange: last.avg - first.avg,
    percentChange: Math.round(((last.avg - first.avg) / first.avg) * 1000) / 10,
    annualizedReturn: Math.round(
      (Math.pow(last.avg / first.avg, 1 / (last.year - first.year)) - 1) * 1000
    ) / 10
  };

  // Year over year changes
  for (let i = 1; i < yearlyData.length; i++) {
    const prev = yearlyData[i - 1];
    const curr = yearlyData[i];
    trends.yearOverYear.push({
      year: curr.year,
      change: curr.avg - prev.avg,
      percentChange: Math.round(((curr.avg - prev.avg) / prev.avg) * 1000) / 10
    });
  }

  return trends;
}

function processWatch(watchId, records) {
  console.log(`Processing ${watchId}: ${records.length} records`);

  const monthly = aggregateByMonth(records);
  const yearly = aggregateByYear(records);
  const trends = calculateTrends(yearly);

  // Statistics
  const allPrices = records.map(r => r.price);
  const { valid } = detectOutliers(allPrices);

  return {
    watchId,
    rawRecords: records.length,
    dateRange: {
      from: records[0]?.date,
      to: records[records.length - 1]?.date
    },
    statistics: {
      min: Math.min(...valid),
      max: Math.max(...valid),
      avg: Math.round(valid.reduce((a, b) => a + b, 0) / valid.length),
      median: valid.sort((a, b) => a - b)[Math.floor(valid.length / 2)],
      totalSamples: valid.length,
      outliers: allPrices.length - valid.length
    },
    monthly,
    yearly,
    trends
  };
}

function aggregate() {
  console.log('Loading raw historical price data...');
  const rawData = loadRawData();

  const result = {
    metadata: {
      processedAt: new Date().toISOString(),
      sourceFile: INPUT_FILE,
      rawRecords: rawData.metadata?.totalRecords || 0
    },
    watches: {}
  };

  let totalProcessed = 0;

  for (const [watchId, records] of Object.entries(rawData.prices || {})) {
    if (!Array.isArray(records) || records.length === 0) {
      console.log(`Skipping ${watchId}: no records`);
      continue;
    }

    result.watches[watchId] = processWatch(watchId, records);
    totalProcessed += records.length;
  }

  result.metadata.processedRecords = totalProcessed;
  result.metadata.watchesProcessed = Object.keys(result.watches).length;

  // Save aggregated data
  fs.writeFileSync(OUTPUT_FILE, JSON.stringify(result, null, 2));
  console.log(`\nSaved aggregated data to ${OUTPUT_FILE}`);
  console.log(`Processed ${totalProcessed} records across ${Object.keys(result.watches).length} watches`);

  return result;
}

// Run if called directly
if (require.main === module) {
  aggregate();
}

module.exports = { aggregate, aggregateByMonth, aggregateByYear, calculateTrends };