`backend/src/scraper/googleSearch.js`

118 lines
const axios = require('axios');

class GoogleCyPresSearch {
  constructor() {
    this.delay = 1000; // Be polite to Google
  }

  /**
   * Search Google for cy pres pages for a given organization
   * Uses site-specific search: site:example.org "cy pres" OR "cypres"
   */
  async searchForCyPresPage(domain) {
    try {
      // Extract domain from URL if full URL provided
      const domainOnly = domain.replace(/^https?:\/\//, '').replace(/\/$/, '');

      // Build Google search query
      const query = `site:${domainOnly} ("cy pres" OR "cypres" OR "cy-pres")`;
      const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;

      console.log(`  🔍 Google searching: ${domainOnly}`);

      await this.sleep(this.delay);

      const response = await axios.get(searchUrl, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        },
        timeout: 10000
      });

      const html = response.data;

      // Extract URLs from search results
      // Look for href="/url?q=" pattern in Google results
      const urlPattern = /href="\/url\?q=(https?:\/\/[^&"]+)/g;
      const matches = [];
      let match;

      while ((match = urlPattern.exec(html)) !== null) {
        const url = decodeURIComponent(match[1]);

        // Only include URLs from the target domain
        if (url.includes(domainOnly) &&
            (url.includes('cy-pres') || url.includes('cypres') || url.includes('cy_pres'))) {
          matches.push(url);
        }
      }

      if (matches.length > 0) {
        console.log(`  ✓ Found ${matches.length} potential cy pres URL(s) via Google`);
        return matches[0]; // Return first match
      }

      return null;

    } catch (error) {
      // Google might block automated searches - that's okay, we have fallbacks
      if (error.response && error.response.status === 429) {
        console.log(`  ⚠ Google rate limit hit - will use direct search`);
      }
      return null;
    }
  }

  /**
   * Alternative: Use DuckDuckGo (doesn't block as aggressively)
   */
  async searchDuckDuckGo(domain) {
    try {
      const domainOnly = domain.replace(/^https?:\/\//, '').replace(/\/$/, '');
      const query = `site:${domainOnly} cy pres`;
      const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;

      await this.sleep(this.delay);

      const response = await axios.get(searchUrl, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        },
        timeout: 10000
      });

      const html = response.data;

      // Extract URLs from DuckDuckGo results
      const urlPattern = /href="\/\/duckduckgo\.com\/l\/\?uddg=([^&"]+)/g;
      const matches = [];
      let match;

      while ((match = urlPattern.exec(html)) !== null) {
        const url = decodeURIComponent(match[1]);

        if (url.includes(domainOnly) &&
            (url.includes('cy-pres') || url.includes('cypres'))) {
          matches.push(url);
        }
      }

      if (matches.length > 0) {
        console.log(`  ✓ Found via DuckDuckGo: ${matches[0]}`);
        return matches[0];
      }

      return null;

    } catch (error) {
      return null;
    }
  }

  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

module.exports = GoogleCyPresSearch;