← back to Handbag Auth Nextjs

src/app/api/crawler-status/route.ts

156 lines

import { NextRequest, NextResponse } from 'next/server'
import { spawn } from 'child_process'
import { promisify } from 'util'
import fs from 'fs'
import path from 'path'
import { requireAuth, UserRole } from '@/lib/auth'
import { addSecurityHeaders } from '@/lib/securityHeaders'
import { rateLimit, getClientIp } from '@/lib/rateLimit'

// Use spawn instead of exec to prevent command injection
function checkProcessRunning(processName: string): Promise<boolean> {
  return new Promise((resolve) => {
    const ps = spawn('ps', ['aux'])
    let output = ''

    ps.stdout.on('data', (data) => {
      output += data.toString()
    })

    ps.on('close', () => {
      // Safely check if process is in output
      const isRunning = output.includes(processName) && !output.includes('grep')
      resolve(isRunning)
    })

    ps.on('error', () => {
      resolve(false)
    })
  })
}

export async function GET(request: NextRequest) {
  // Apply rate limiting
  const clientIp = getClientIp(request)
  const rateLimitResult = rateLimit(clientIp, {
    windowMs: 60 * 1000,
    maxRequests: 30,
  })

  if (!rateLimitResult.allowed) {
    const response = NextResponse.json(
      { success: false, error: 'Rate limit exceeded' },
      { status: 429 }
    )
    addSecurityHeaders(response)
    return response
  }

  try {
    // Check if crawler is running (safe method)
    const isRunning = await checkProcessRunning('crawlAffiliateSites')

    // Read logs
    let logs: string[] = []
    const logFile = '/tmp/crawl-output.log'
    if (fs.existsSync(logFile)) {
      const content = fs.readFileSync(logFile, 'utf8')
      logs = content.split('\n').filter(l => l.trim()).slice(-100) // Last 100 lines
    }

    // Load crawled data stats
    const dataFile = path.resolve('data/crawled_affiliate_listings.json')
    let stats = {
      totalListings: 0,
      brands: 0,
      merchants: 0,
      newToday: 0,
      brandProgress: {} as Record<string, number>
    }

    if (fs.existsSync(dataFile)) {
      const listings = JSON.parse(fs.readFileSync(dataFile, 'utf8'))

      const brands = new Set(listings.map((l: any) => l.brand))
      const merchants = new Set(listings.map((l: any) => l.merchant))

      const today = new Date().toISOString().split('T')[0]
      const todayListings = listings.filter((l: any) =>
        l.crawledAt?.startsWith(today)
      )

      const brandCounts = listings.reduce((acc: any, item: any) => {
        acc[item.brand] = (acc[item.brand] || 0) + 1
        return acc
      }, {})

      stats = {
        totalListings: listings.length,
        brands: brands.size,
        merchants: merchants.size,
        newToday: todayListings.length,
        brandProgress: brandCounts
      }
    }

    const response = NextResponse.json({
      success: true,
      isRunning,
      logs,
      stats
    })

    addSecurityHeaders(response)
    return response

  } catch (error: any) {
    const response = NextResponse.json({
      success: false,
      error: 'Failed to get crawler status'
    }, { status: 500 })
    addSecurityHeaders(response)
    return response
  }
}

export async function POST(request: NextRequest) {
  // Require admin authentication to start crawler
  const authResult = await requireAuth(request, UserRole.ADMIN)
  if (authResult instanceof NextResponse) {
    return authResult
  }

  try {
    // Start the crawler safely using spawn
    const crawler = spawn('npm', ['run', 'crawl-affiliates'], {
      cwd: '/root/Projects/handbag-auth-nextjs',
      detached: true,
      stdio: ['ignore', 'pipe', 'pipe']
    })

    // Write output to log file
    const logStream = fs.createWriteStream('/tmp/crawl-output.log', { flags: 'a' })
    crawler.stdout?.pipe(logStream)
    crawler.stderr?.pipe(logStream)

    crawler.unref() // Allow process to run independently

    const response = NextResponse.json({
      success: true,
      message: 'Crawler started',
      pid: crawler.pid
    })

    addSecurityHeaders(response)
    return response

  } catch (error: any) {
    const response = NextResponse.json({
      success: false,
      error: 'Failed to start crawler'
    }, { status: 500 })
    addSecurityHeaders(response)
    return response
  }
}