`lib/moderation/content-filter.ts`

221 lines
import bannedTerms from './banned-terms.json'

interface FilterResult {
  clean: boolean
  violations: string[]
  sanitizedText?: string
  severity: 'critical' | 'high' | 'medium' | 'low' | 'none'
}

export class ContentFilter {
  private bannedPatterns: Map<string, RegExp> = new Map()
  
  constructor() {
    this.initializePatterns()
  }
  
  private initializePatterns() {
    // Build regex patterns for all banned terms
    Object.values(bannedTerms.categories).forEach((category: any) => {
      category.terms.forEach((item: any) => {
        const term = item.term || item.phrase
        const variants = item.variants || []
        const allTerms = [term, ...variants]
        
        allTerms.forEach(t => {
          // Create case-insensitive pattern with word boundaries
          const escaped = t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
          const pattern = new RegExp(`\\b${escaped}\\b`, 'gi')
          this.bannedPatterns.set(t.toLowerCase(), pattern)
        })
      })
    })
  }
  
  /**
   * Check if text contains banned terms
   */
  public checkContent(text: string, allowBubbeExceptions: boolean = false): FilterResult {
    const violations: string[] = []
    let severity: FilterResult['severity'] = 'none'
    
    // Check each category
    for (const [categoryName, category] of Object.entries(bannedTerms.categories)) {
      const categoryData = category as any
      
      for (const item of categoryData.terms) {
        const term = item.term || item.phrase
        const variants = item.variants || []
        const allTerms = [term, ...variants]
        
        for (const t of allTerms) {
          const pattern = this.bannedPatterns.get(t.toLowerCase())
          if (pattern && pattern.test(text)) {
            // Check if this term has a Bubbe exception and we're allowing those
            if (allowBubbeExceptions && item.bubbe_exception) {
              continue // Skip this violation for Bubbe character responses
            }
            
            violations.push(`${categoryName}: ${t}`)
            
            // Update severity
            if (categoryData.severity === 'critical') {
              severity = 'critical'
            } else if (categoryData.severity === 'high' && severity !== 'critical') {
              severity = 'high'
            }
          }
        }
      }
    }
    
    return {
      clean: violations.length === 0,
      violations,
      severity,
      sanitizedText: violations.length > 0 ? this.sanitizeText(text) : text
    }
  }
  
  /**
   * Remove or replace banned terms in text
   */
  private sanitizeText(text: string): string {
    let sanitized = text
    
    // Replace banned terms with safe alternatives
    for (const [categoryName, category] of Object.entries(bannedTerms.categories)) {
      const categoryData = category as any
      
      for (const item of categoryData.terms) {
        const term = item.term || item.phrase
        const variants = item.variants || []
        const replacement = item.replacement || '[removed]'
        const allTerms = [term, ...variants]
        
        for (const t of allTerms) {
          const pattern = this.bannedPatterns.get(t.toLowerCase())
          if (pattern) {
            sanitized = sanitized.replace(pattern, replacement)
          }
        }
      }
    }
    
    return sanitized
  }
  
  /**
   * Check if a prompt is trying to elicit banned content
   */
  public checkPrompt(prompt: string): boolean {
    const lowerPrompt = prompt.toLowerCase()
    
    // Check for attempts to get Bubbe to use slurs
    const suspiciousPatterns = [
      /what.*call.*non-jew/i,
      /say.*about.*\brace\b/i,
      /opinion.*different.*people/i,
      /joke.*about.*\bjew/i,
      /tell.*about.*hitler/i,
      /holocaust.*joke/i,
      /use.*word.*goy/i,
      /call.*them.*shiksa/i
    ]
    
    return suspiciousPatterns.some(pattern => pattern.test(lowerPrompt))
  }
  
  /**
   * Get safe response for problematic prompts
   */
  public getSafeResponse(prompt: string): string | null {
    if (this.checkPrompt(prompt)) {
      const responses = [
        "Oy vey, bubbeleh! I don't talk like that. Let's keep it nice, yes?",
        "What kind of meshugana question is that? I raised you better!",
        "Listen here, I don't use such words. Find something else to talk about!",
        "Feh! My mother would wash my mouth with soap for such talk. Next question!",
        "I'm a classy lady, I don't speak that way. Now, what else can I help you with?"
      ]
      return responses[Math.floor(Math.random() * responses.length)]
    }
    return null
  }
  
  /**
   * Check if text contains Holocaust references that aren't educational
   */
  public checkHolocaustContext(text: string, isEducational: boolean = false): boolean {
    const holocaustTerms = bannedTerms.categories.holocaust_references.terms
    
    for (const item of holocaustTerms) {
      const term = item.term
      const variants = item.variants || []
      const allTerms = [term, ...variants]
      
      for (const t of allTerms) {
        if (new RegExp(`\\b${t}\\b`, 'gi').test(text)) {
          // If it's educational context, might be acceptable
          if (isEducational) {
            // Check if it's respectful discussion
            const respectfulMarkers = /\b(history|remembrance|education|memorial|survivor|testimony)\b/gi
            return respectfulMarkers.test(text)
          }
          return false // Not acceptable in casual context
        }
      }
    }
    
    return true // No Holocaust references found
  }
  
  /**
   * Validate Bubbe's response before sending
   */
  public validateResponse(response: string): { valid: boolean; reason?: string } {
    const filterResult = this.checkContent(response, true) // Allow Bubbe exceptions
    
    if (!filterResult.clean) {
      return {
        valid: false,
        reason: `Response contains banned terms: ${filterResult.violations.join(', ')}`
      }
    }
    
    // Additional checks for tone
    if (!this.checkHolocaustContext(response, false)) {
      return {
        valid: false,
        reason: 'Response contains inappropriate Holocaust reference'
      }
    }
    
    return { valid: true }
  }
}

// Export singleton instance
export const contentFilter = new ContentFilter()

// Export function for easy use in API routes
export function filterBubbeResponse(text: string): { 
  safe: boolean
  filtered?: string
  warning?: string 
} {
  const validation = contentFilter.validateResponse(text)
  
  if (!validation.valid) {
    console.error('[CONTENT FILTER] Blocked response:', validation.reason)
    
    // Return a safe fallback response
    return {
      safe: false,
      filtered: "Oy, something got mixed up in my old brain! Let me think again... What did you want to know, bubbeleh?",
      warning: validation.reason
    }
  }
  
  return { safe: true, filtered: text }
}