`services/github-dataset.js`

342 lines
/**
 * GitHub-hosted Price Dataset Integration
 * Fetches luxury watch price data from public GitHub repositories
 */

import { setTimeout } from 'timers/promises';

// GitHub API configuration
const GITHUB_API = 'https://api.github.com';
const GITHUB_RAW = 'https://raw.githubusercontent.com';

// Rate limiting: GitHub allows 60 requests/hour unauthenticated
const RATE_LIMIT_MS = 60000; // 1 request per minute to be safe
let lastRequestTime = 0;
let remainingRequests = 60;

// Cache configuration
const cache = new Map();
const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours

// Known public datasets
const KNOWN_DATASETS = [
  {
    owner: 'philmorefkoung',
    repo: 'Webscrapped-Watch-Dataset',
    path: 'Omega.csv',
    format: 'csv',
    brand: 'omega'
  },
  {
    owner: 'bobbypine',
    repo: 'Rolex',
    path: 'Prices/Weekly_Median_Prices.csv',
    format: 'csv',
    brand: 'rolex'
  }
];

/**
 * Rate-limited fetch for GitHub API
 */
async function rateLimitedFetch(url, options = {}) {
  const now = Date.now();
  const timeSinceLastRequest = now - lastRequestTime;

  // Check remaining requests
  if (remainingRequests <= 5 && timeSinceLastRequest < RATE_LIMIT_MS) {
    const waitTime = RATE_LIMIT_MS - timeSinceLastRequest;
    console.log(`GitHub rate limit approaching, waiting ${waitTime}ms`);
    await setTimeout(waitTime);
  }

  lastRequestTime = Date.now();

  const response = await fetch(url, {
    ...options,
    headers: {
      'Accept': 'application/vnd.github.v3+json',
      'User-Agent': 'OmegaWatchPlatform',
      ...options.headers
    }
  });

  // Update rate limit info from headers
  const remaining = response.headers.get('X-RateLimit-Remaining');
  if (remaining) {
    remainingRequests = parseInt(remaining);
  }

  return response;
}

/**
 * Get cached result or null
 */
function getCached(key) {
  const cached = cache.get(key);
  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
    return cached.data;
  }
  cache.delete(key);
  return null;
}

/**
 * Set cache
 */
function setCache(key, data) {
  cache.set(key, { data, timestamp: Date.now() });
}

/**
 * Parse CSV data
 * @param {string} csvText - Raw CSV content
 * @returns {array} Array of objects
 */
function parseCSV(csvText) {
  const lines = csvText.trim().split('\n');
  if (lines.length < 2) return [];

  const headers = lines[0].split(',').map(h => h.trim().toLowerCase().replace(/['"]/g, ''));

  return lines.slice(1).map(line => {
    const values = [];
    let current = '';
    let inQuotes = false;

    for (const char of line) {
      if (char === '"') {
        inQuotes = !inQuotes;
      } else if (char === ',' && !inQuotes) {
        values.push(current.trim());
        current = '';
      } else {
        current += char;
      }
    }
    values.push(current.trim());

    const obj = {};
    headers.forEach((header, i) => {
      obj[header] = values[i]?.replace(/['"]/g, '') || '';
    });
    return obj;
  });
}

/**
 * Fetch raw file content from GitHub
 * @param {string} owner - Repository owner
 * @param {string} repo - Repository name
 * @param {string} path - File path
 * @returns {string} File content
 */
export async function fetchRawFile(owner, repo, path) {
  const cacheKey = `raw:${owner}/${repo}/${path}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const url = `${GITHUB_RAW}/${owner}/${repo}/main/${path}`;

  try {
    const response = await fetch(url);
    if (!response.ok) {
      // Try master branch
      const masterUrl = `${GITHUB_RAW}/${owner}/${repo}/master/${path}`;
      const masterResponse = await fetch(masterUrl);
      if (!masterResponse.ok) {
        throw new Error(`Failed to fetch ${path}: ${response.status}`);
      }
      const content = await masterResponse.text();
      setCache(cacheKey, content);
      return content;
    }

    const content = await response.text();
    setCache(cacheKey, content);
    return content;

  } catch (error) {
    console.error(`GitHub fetch error: ${error.message}`);
    return null;
  }
}

/**
 * Fetch and parse dataset from GitHub
 * @param {object} dataset - Dataset configuration
 * @returns {array} Parsed data records
 */
export async function fetchDataset(dataset) {
  const content = await fetchRawFile(dataset.owner, dataset.repo, dataset.path);
  if (!content) return [];

  if (dataset.format === 'csv') {
    return parseCSV(content);
  } else if (dataset.format === 'json') {
    try {
      return JSON.parse(content);
    } catch {
      return [];
    }
  }

  return [];
}

/**
 * Fetch Omega watch prices from known datasets
 * @returns {array} Array of price records
 */
export async function fetchOmegaPrices() {
  const cacheKey = 'omega-prices';
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const results = [];

  for (const dataset of KNOWN_DATASETS) {
    if (dataset.brand !== 'omega') continue;

    try {
      const data = await fetchDataset(dataset);

      for (const record of data) {
        // Extract relevant fields (adapt based on actual CSV structure)
        const price = parseFloat(
          (record.price || record.listing_price || record.value || '0')
            .replace(/[^0-9.]/g, '')
        );

        if (price > 0) {
          results.push({
            reference: record.reference || record.ref || record.model_number || '',
            model: record.model || record.name || record.title || '',
            price,
            condition: record.condition || 'unknown',
            source: `github:${dataset.owner}/${dataset.repo}`,
            sourceUrl: `https://github.com/${dataset.owner}/${dataset.repo}`,
            date: record.date || record.scraped_date || new Date().toISOString().slice(0, 10),
            confidence: 4 // High confidence for structured datasets
          });
        }
      }
    } catch (error) {
      console.error(`Error fetching ${dataset.repo}:`, error.message);
    }
  }

  setCache(cacheKey, results);
  return results;
}

/**
 * Search for watch data repositories on GitHub
 * @param {string} query - Search query
 * @returns {array} Array of repository info
 */
export async function searchRepositories(query = 'watch prices dataset') {
  const cacheKey = `search:${query}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const url = `${GITHUB_API}/search/repositories?q=${encodeURIComponent(query)}&sort=stars&per_page=10`;

  try {
    const response = await rateLimitedFetch(url);
    if (!response.ok) {
      throw new Error(`GitHub search error: ${response.status}`);
    }

    const data = await response.json();
    const repos = data.items.map(repo => ({
      owner: repo.owner.login,
      name: repo.name,
      description: repo.description,
      stars: repo.stargazers_count,
      url: repo.html_url,
      lastUpdated: repo.updated_at
    }));

    setCache(cacheKey, repos);
    return repos;

  } catch (error) {
    console.error('GitHub search error:', error.message);
    return [];
  }
}

/**
 * Get price statistics from dataset
 * @param {array} priceData - Array of price records
 * @param {string} reference - Watch reference to filter by
 * @returns {object} Statistics
 */
export function getPriceStatistics(priceData, reference = null) {
  let data = priceData;

  if (reference) {
    data = priceData.filter(p =>
      p.reference?.toLowerCase().includes(reference.toLowerCase())
    );
  }

  if (data.length === 0) {
    return { count: 0, min: 0, max: 0, avg: 0, median: 0 };
  }

  const prices = data.map(p => p.price).sort((a, b) => a - b);

  return {
    count: prices.length,
    min: prices[0],
    max: prices[prices.length - 1],
    avg: Math.round(prices.reduce((a, b) => a + b, 0) / prices.length),
    median: prices[Math.floor(prices.length / 2)],
    stdDev: Math.round(
      Math.sqrt(
        prices.reduce((sum, p) => {
          const avg = prices.reduce((a, b) => a + b, 0) / prices.length;
          return sum + Math.pow(p - avg, 2);
        }, 0) / prices.length
      )
    )
  };
}

/**
 * Clear expired cache entries
 */
export function clearExpiredCache() {
  const now = Date.now();
  for (const [key, value] of cache) {
    if (now - value.timestamp > CACHE_TTL) {
      cache.delete(key);
    }
  }
}

/**
 * Get cache statistics
 */
export function getCacheStats() {
  return {
    entries: cache.size,
    remainingRequests,
    lastRequest: new Date(lastRequestTime).toISOString()
  };
}

export default {
  fetchRawFile,
  fetchDataset,
  fetchOmegaPrices,
  searchRepositories,
  getPriceStatistics,
  clearExpiredCache,
  getCacheStats,
  KNOWN_DATASETS
};