`services/handbagRedditScraper.js`

253 lines
const axios = require('axios');

/**
 * Reddit Handbag Communities Scraper
 *
 * Scrapes data from Reddit communities focused on designer handbags,
 * authentication, market prices, and community discussions.
 *
 * Key Subreddits:
 * 1. r/RepLadies - Replica handbag discussions, reviews, authentication
 * 2. r/DesignerReps - Designer replica community
 * 3. r/handbags - General handbag discussion
 * 4. r/Fashionreps - Fashion replica community
 * 5. Authentication communities on PurseForum crossposted to Reddit
 *
 * Data Points:
 * - Price discussions and comparisons
 * - Authentication guides and tips
 * - Brand reviews and quality assessments
 * - Market trends and "It bags"
 * - Unboxing reviews and comparisons
 */

class HandbagRedditScraper {
  constructor() {
    this.baseUrl = 'https://www.reddit.com';

    this.subreddits = {
      repladies: {
        name: 'r/RepLadies',
        focus: 'Replica handbags, reviews, authentication',
        members: 'Large active community',
        content: 'Unboxings, reviews, side-by-side comparisons, shopping guides'
      },
      designerReps: {
        name: 'r/DesignerReps',
        focus: 'Designer replica discussion',
        content: 'Quality control, seller reviews, authentication'
      },
      handbags: {
        name: 'r/handbags',
        focus: 'General handbag community',
        content: 'Authentic bags, advice, collection sharing'
      },
      fashionReps: {
        name: 'r/FashionReps',
        focus: 'Fashion replicas including bags',
        content: 'Shopping guides, QC checks, reviews'
      },
      repSneakers: {
        name: 'r/RepSneakers',
        focus: 'Primarily sneakers but overlaps with fashion community'
      }
    };

    this.authenticationSources = {
      purseForum: {
        name: 'PurseForum (discussed on Reddit)',
        feature: '"Authenticate This..." section',
        process: 'Community-pooled authentication opinions',
        requirements: 'Interior, exterior, hardware, heat stamps, date codes, labels photos'
      },
      facebook: {
        name: 'Facebook Authentication Groups',
        content: 'Formal and informal counterfeit prevention',
        features: 'Product authentication, moral discussions, gatekeeping'
      }
    };
  }

  /**
   * Search Reddit for handbag discussions
   * Uses Reddit's JSON API (no auth required for public posts)
   */
  async searchReddit(query, options = {}) {
    try {
      const {
        subreddit = 'all', // 'repladies', 'handbags', 'designerreps', or 'all'
        sort = 'relevance', // 'relevance', 'hot', 'top', 'new'
        timeFilter = 'all', // 'hour', 'day', 'week', 'month', 'year', 'all'
        limit = 25
      } = options;

      const subredditPath = subreddit === 'all' ? 'search' : `r/${subreddit}/search`;
      const searchUrl = `${this.baseUrl}/${subredditPath}.json`;

      const response = await axios.get(searchUrl, {
        params: {
          q: query,
          sort: sort,
          t: timeFilter,
          limit: limit,
          restrict_sr: subreddit !== 'all' ? 'on' : 'off'
        },
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        },
        timeout: 10000
      });

      const posts = [];

      if (response.data && response.data.data && response.data.data.children) {
        for (const child of response.data.data.children) {
          const post = child.data;

          posts.push({
            title: post.title,
            author: post.author,
            subreddit: post.subreddit,
            score: post.score,
            numComments: post.num_comments,
            created: new Date(post.created_utc * 1000).toISOString(),
            url: `https://www.reddit.com${post.permalink}`,
            text: post.selftext || '',
            flair: post.link_flair_text,
            isAuthentication: this.isAuthenticationPost(post),
            isReview: this.isReviewPost(post),
            isPriceDiscussion: this.isPriceDiscussion(post)
          });
        }
      }

      return {
        success: true,
        posts,
        count: posts.length,
        query,
        subreddit,
        source: 'Reddit',
        timestamp: new Date().toISOString()
      };

    } catch (error) {
      console.error('Reddit scraper error:', error.message);
      return {
        success: false,
        posts: [],
        error: error.message,
        source: 'Reddit'
      };
    }
  }

  /**
   * Get hot posts from handbag subreddits
   */
  async getHotPosts(subreddit = 'handbags', limit = 25) {
    try {
      const url = `${this.baseUrl}/r/${subreddit}/hot.json`;

      const response = await axios.get(url, {
        params: { limit },
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        },
        timeout: 10000
      });

      const posts = [];

      if (response.data && response.data.data && response.data.data.children) {
        for (const child of response.data.data.children) {
          const post = child.data;

          posts.push({
            title: post.title,
            author: post.author,
            score: post.score,
            numComments: post.num_comments,
            url: `https://www.reddit.com${post.permalink}`,
            thumbnail: post.thumbnail,
            isAuthentication: this.isAuthenticationPost(post),
            isReview: this.isReviewPost(post)
          });
        }
      }

      return {
        success: true,
        posts,
        count: posts.length,
        subreddit,
        source: 'Reddit'
      };

    } catch (error) {
      console.error('Reddit hot posts error:', error.message);
      return {
        success: false,
        posts: [],
        error: error.message
      };
    }
  }

  /**
   * Helper: Detect if post is about authentication
   */
  isAuthenticationPost(post) {
    const keywords = ['authenticate', 'legit check', 'real or fake', 'qc', 'quality check', 'LC'];
    const text = `${post.title} ${post.selftext}`.toLowerCase();
    return keywords.some(keyword => text.includes(keyword.toLowerCase()));
  }

  /**
   * Helper: Detect if post is a review
   */
  isReviewPost(post) {
    const keywords = ['review', 'unboxing', 'haul', 'just bought', 'comparison'];
    const text = `${post.title}`.toLowerCase();
    return keywords.some(keyword => text.includes(keyword.toLowerCase()));
  }

  /**
   * Helper: Detect if post discusses prices
   */
  isPriceDiscussion(post) {
    const keywords = ['price', '$', 'cost', 'worth it', 'budget', 'expensive', 'cheap'];
    const text = `${post.title} ${post.selftext}`.toLowerCase();
    return keywords.some(keyword => text.includes(keyword.toLowerCase()));
  }

  /**
   * Get subreddit information
   */
  getSubredditInfo() {
    return {
      communities: this.subreddits,
      authenticationSources: this.authenticationSources,
      dataTypes: [
        'Authentication discussions',
        'Price comparisons',
        'Quality reviews',
        'Brand reputation',
        'Market trends ("It bags")',
        'Shopping guides',
        'Seller reviews (for replicas)',
        'Unboxing and comparison photos'
      ],
      useCases: [
        'Real-time market sentiment',
        'Authentication education',
        'Price benchmarking',
        'Brand popularity tracking',
        'Trend identification'
      ]
    };
  }
}

module.exports = new HandbagRedditScraper();