`services/ucDavisAVAScraper.js`

345 lines
const axios = require('axios');

/**
 * UC Davis American Viticultural Areas (AVA) Dataset Scraper
 *
 * Open-licensed, community-contributed spatial dataset of American Viticultural Areas boundaries
 * Maintained by: UC Davis DataLab and Library, UCSB Library, Virginia Tech
 *
 * Dataset Details:
 * - Boundaries for all US American Viticultural Areas
 * - GeoJSON and Shapefile formats
 * - Current and historical AVA boundaries
 * - State-by-state AVA data
 *
 * GitHub: https://github.com/UCDavisLibrary/ava
 * Website: https://ucdavislibrary.github.io/ava/
 * License: Creative Commons CC0 (Public Domain)
 */

class UCDavisAVAScraper {
  constructor() {
    this.githubBaseUrl = 'https://raw.githubusercontent.com/UCDavisLibrary/ava/master';
    this.projectUrl = 'https://ucdavislibrary.github.io/ava/';
    this.githubRepo = 'https://github.com/UCDavisLibrary/ava';

    // Cache for AVA data
    this.avaCache = null;
    this.avaByStateCache = {};
    this.cacheTimestamp = null;
    this.cacheDuration = 7 * 24 * 60 * 60 * 1000; // 7 days (data changes infrequently)

    this.datasetInfo = {
      name: 'UC Davis AVA Project',
      maintainers: 'UC Davis DataLab, UCSB Library, Virginia Tech',
      coverage: 'All US American Viticultural Areas',
      formats: ['GeoJSON', 'Shapefile'],
      license: 'Creative Commons CC0 (Public Domain)',
      updated: 'Regularly maintained'
    };
  }

  /**
   * Search AVAs by name or region
   */
  async search(query, options = {}) {
    try {
      const {
        state = null,
        limit = 50
      } = options;

      // Load AVA data if not cached
      if (!this.isCacheValid()) {
        await this.loadAVAData();
      }

      const queryLower = query.toLowerCase();

      // Filter AVAs by query
      let avas = this.avaCache.features.filter(feature => {
        const name = (feature.properties.name || '').toLowerCase();
        const stateAbbr = (feature.properties.state || '').toLowerCase();
        const county = (feature.properties.county || '').toLowerCase();

        const matchesQuery = name.includes(queryLower) ||
                            county.includes(queryLower);
        const matchesState = !state || stateAbbr === state.toLowerCase();

        return matchesQuery && matchesState;
      });

      // Limit results
      avas = avas.slice(0, limit);

      // Transform to wine-compatible format
      const wines = avas.map(ava => ({
        name: `Wines from ${ava.properties.name || 'Unknown'} AVA`,
        vintage: '',
        price: null,
        rating: null,
        country: 'United States',
        region: `${ava.properties.name}, ${ava.properties.state}`,
        avaName: ava.properties.name,
        avaState: ava.properties.state,
        avaCounty: ava.properties.county,
        avaApproved: ava.properties.approved,
        avaEffective: ava.properties.effective_date,
        url: ava.properties.url || this.projectUrl,
        availability: 'Geographic Region Data',
        source: 'UC Davis AVA',
        geoData: {
          type: ava.geometry.type,
          coordinates: ava.geometry.coordinates
        }
      }));

      return {
        success: true,
        wines,
        count: wines.length,
        source: 'UC Davis AVA Project',
        message: 'AVA geographic data - showing wine regions',
        timestamp: new Date().toISOString()
      };

    } catch (error) {
      console.error('UC Davis AVA scraper error:', error.message);
      return {
        success: false,
        wines: [],
        error: error.message,
        source: 'UC Davis AVA'
      };
    }
  }

  /**
   * Load all AVA data from GitHub
   */
  async loadAVAData() {
    try {
      const url = `${this.githubBaseUrl}/avas_aggregated_files/avas.geojson`;
      const response = await axios.get(url, {
        timeout: 30000,
        headers: {
          'User-Agent': 'Wine-Tracker-App/1.0'
        }
      });

      this.avaCache = response.data;
      this.cacheTimestamp = Date.now();

      console.log(`Loaded ${this.avaCache.features.length} AVAs from UC Davis`);

      return {
        success: true,
        count: this.avaCache.features.length
      };

    } catch (error) {
      console.error('Error loading AVA data:', error.message);
      throw error;
    }
  }

  /**
   * Load AVAs for a specific state
   */
  async loadStateAVAs(stateCode) {
    try {
      const url = `${this.githubBaseUrl}/avas_by_state/${stateCode}_avas.geojson`;
      const response = await axios.get(url, {
        timeout: 30000,
        headers: {
          'User-Agent': 'Wine-Tracker-App/1.0'
        }
      });

      this.avaByStateCache[stateCode] = response.data;

      return {
        success: true,
        state: stateCode,
        count: response.data.features.length,
        data: response.data
      };

    } catch (error) {
      console.error(`Error loading AVAs for ${stateCode}:`, error.message);
      return {
        success: false,
        error: error.message
      };
    }
  }

  /**
   * Get AVA details by name
   */
  async getAVADetails(avaName) {
    if (!this.isCacheValid()) {
      await this.loadAVAData();
    }

    const ava = this.avaCache.features.find(f =>
      (f.properties.name || '').toLowerCase() === avaName.toLowerCase()
    );

    if (!ava) {
      return {
        success: false,
        error: `AVA not found: ${avaName}`
      };
    }

    return {
      success: true,
      ava: {
        name: ava.properties.name,
        state: ava.properties.state,
        county: ava.properties.county,
        approved: ava.properties.approved,
        effectiveDate: ava.properties.effective_date,
        cfrCitation: ava.properties.cfr_citation,
        url: ava.properties.url,
        geometry: ava.geometry,
        source: 'UC Davis AVA Project'
      }
    };
  }

  /**
   * Get list of all AVA names
   */
  async getAllAVANames() {
    if (!this.isCacheValid()) {
      await this.loadAVAData();
    }

    const names = this.avaCache.features.map(f => ({
      name: f.properties.name,
      state: f.properties.state,
      county: f.properties.county
    }));

    return {
      success: true,
      avas: names,
      count: names.length
    };
  }

  /**
   * Get AVAs by state
   */
  async getAVAsByState(stateCode) {
    // Try loading from state-specific file
    if (!this.avaByStateCache[stateCode]) {
      await this.loadStateAVAs(stateCode);
    }

    if (this.avaByStateCache[stateCode]) {
      return {
        success: true,
        state: stateCode,
        avas: this.avaByStateCache[stateCode].features,
        count: this.avaByStateCache[stateCode].features.length
      };
    }

    // Fallback to filtering all AVAs
    if (!this.isCacheValid()) {
      await this.loadAVAData();
    }

    const stateAVAs = this.avaCache.features.filter(f =>
      (f.properties.state || '').toUpperCase() === stateCode.toUpperCase()
    );

    return {
      success: true,
      state: stateCode,
      avas: stateAVAs,
      count: stateAVAs.length
    };
  }

  /**
   * Get major wine states and their AVA counts
   */
  async getWineStates() {
    if (!this.isCacheValid()) {
      await this.loadAVAData();
    }

    const stateCounts = {};

    this.avaCache.features.forEach(f => {
      const state = f.properties.state || 'Unknown';
      stateCounts[state] = (stateCounts[state] || 0) + 1;
    });

    const sorted = Object.entries(stateCounts)
      .map(([state, count]) => ({ state, count }))
      .sort((a, b) => b.count - a.count);

    return {
      success: true,
      states: sorted,
      topStates: sorted.slice(0, 10)
    };
  }

  /**
   * Get dataset statistics
   */
  getStats() {
    return {
      datasetInfo: this.datasetInfo,
      projectUrl: this.projectUrl,
      githubUrl: this.githubRepo,
      downloadUrl: `${this.githubBaseUrl}/avas.geojson`,
      features: [
        'All US American Viticultural Areas',
        'Current and historical boundaries',
        'GeoJSON and Shapefile formats',
        'State-by-state AVA data',
        'Regularly maintained and updated',
        'Open-licensed (CC0 Public Domain)',
        'Community-contributed',
        'Includes approval dates and CFR citations'
      ],
      files: [
        'avas.geojson - All current AVA boundaries',
        'avas_historic.geojson - Historical boundaries',
        'avas_allboundaries.geojson - Current + historical',
        'avas_by_state/*.geojson - State-specific files'
      ],
      useCases: [
        'Wine region identification',
        'Geographic wine search',
        'Terroir analysis',
        'Wine origin verification',
        'Regional wine characteristics',
        'Wine tourism applications',
        'Research on appellation systems',
        'GIS and mapping applications'
      ],
      citation: 'UC Davis Library. (2024). American Viticultural Areas Project. https://github.com/UCDavisLibrary/ava'
    };
  }

  /**
   * Check if cache is valid
   */
  isCacheValid() {
    if (!this.avaCache || !this.cacheTimestamp) {
      return false;
    }
    return (Date.now() - this.cacheTimestamp) < this.cacheDuration;
  }
}

module.exports = new UCDavisAVAScraper();