`lib/config.js`

60 lines
'use strict';
// Central tunables for the recrawl dispatcher. Everything overridable via env.
const path = require('path');
const os = require('os');

const HOME = os.homedir();

module.exports = {
  // --- selection / wave ---
  // Number of REAL scrape attempts (resolved pipelines) per nightly run.
  // DTD verdict: 10/night, tunable to 12 after a stable week.
  BATCH: parseInt(process.env.RECRAWL_BATCH || '10', 10),

  // Hard wall-clock cap for the whole nightly run. DTD: 6-8h.
  MAX_WALL_MS: parseInt(process.env.RECRAWL_MAX_HOURS || '7', 10) * 3600 * 1000,

  // Per-vendor scrape hard timeout (kills a stuck pipeline so it can't eat the cap).
  PER_VENDOR_TIMEOUT_MS: parseInt(process.env.RECRAWL_VENDOR_TIMEOUT_MIN || '40', 10) * 60 * 1000,

  // Politeness jitter between vendors (seconds). Random in [MIN, MAX].
  JITTER_MIN_S: parseInt(process.env.RECRAWL_JITTER_MIN_S || '20', 10),
  JITTER_MAX_S: parseInt(process.env.RECRAWL_JITTER_MAX_S || '90', 10),

  // Re-crawl cadence per priority (days) — how far out next_scheduled_crawl is
  // pushed AFTER a successful refresh. Stalest-first ordering means a full sweep
  // of resolvable vendors keeps rolling continuously.
  CADENCE_DAYS: { HIGH: 7, MEDIUM: 21, LOW: 28, DEFAULT: 21 },

  // Vendors the dispatcher must NOT touch because a dedicated job already owns them.
  // WallQuest has its own weekly launchd job (com.steve.wallquest-refresh) — skip to
  // avoid a double-crawl race on the same portal session. (Documented for Steve.)
  EXCLUDE_VENDOR_CODES: (process.env.RECRAWL_EXCLUDE || 'wallquest').split(',').map(s => s.trim()).filter(Boolean),

  // --- pipeline resolution ---
  // Max SKUs the generic product_url refresher re-fetches per vendor per night.
  GENERIC_LIMIT: parseInt(process.env.RECRAWL_GENERIC_LIMIT || '25', 10),
  DW_ROOT: path.join(HOME, 'Projects', 'Designer-Wallcoverings'),
  REFRESH_SCRIPTS_DIR: path.join(HOME, 'Projects', 'Designer-Wallcoverings', 'scripts'),

  // --- enrichment router (LOCAL MODELS ONLY) ---
  EXO_BASE: process.env.EXO_BASE || 'http://localhost:52415',
  // exo model id for the serve-check + enrichment. exo's daemon can be UP while unable
  // to actually run inference (e.g. mlx not yet built — needs full Xcode), so the router
  // does a REAL completion probe, not just /v1/models. Falls through to Ollama until exo serves.
  EXO_MODEL: process.env.EXO_MODEL || 'mlx-community/Qwen3-VL-4B-Instruct-4bit',
  OLLAMA_BASE: process.env.OLLAMA_BASE || 'http://192.168.1.133:11434',
  OLLAMA_MODEL: process.env.OLLAMA_MODEL || 'qwen3:14b',
  HEALTH_TIMEOUT_MS: 3500,
  ENRICH_TIMEOUT_MS: 45000,

  // --- integrations ---
  CNCP_URL: process.env.CNCP_URL || 'http://127.0.0.1:3333/api/wins',
  DB_ENV_FILE: process.env.RECRAWL_DB_ENV ||
    path.join(HOME, 'Projects', 'Designer-Wallcoverings', 'DW-Programming', 'ImportNewSkufromURL', '.env.local'),

  // --- paths ---
  SKIP_LOG: path.join(__dirname, '..', 'logs', 'unresolved.jsonl'),
  RUN_LOG: path.join(__dirname, '..', 'logs', 'runs.jsonl'),
};