`lib/index.js`

213 lines
'use strict';

const fs   = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const { embed } = require('./embed');
const { redact } = require('./redact');

const VECTORS_PATH = path.join(__dirname, '../data/vectors.jsonl');
const CHUNK_SIZE   = 512;
const OVERLAP      = 50;
const HOME         = process.env.HOME || '/Users/macstudio3';

// ── helpers ──────────────────────────────────────────────────────────────────

function chunkText(text, source_type, source_path) {
  const chunks = [];
  let i = 0;
  while (i < text.length) {
    const slice = text.slice(i, i + CHUNK_SIZE);
    if (slice.trim().length > 20) {
      chunks.push({ source_type, source_path, chunk_text: slice });
    }
    i += CHUNK_SIZE - OVERLAP;
  }
  return chunks;
}

function safeRead(filePath, maxLines) {
  try {
    if (!fs.existsSync(filePath)) return null;
    const raw = fs.readFileSync(filePath, 'utf8');
    if (!maxLines) return raw;
    return raw.split('\n').slice(0, maxLines).join('\n');
  } catch { return null; }
}

// ── source collectors ─────────────────────────────────────────────────────────

function collectPm2() {
  const docs = [];
  try {
    const raw = execSync('pm2 jlist 2>/dev/null', { encoding: 'utf8', timeout: 10000 });
    const procs = JSON.parse(raw);
    for (const p of procs) {
      const name   = p.name || 'unknown';
      const cwd    = p.pm2_env?.pm_cwd || p.pm2_env?.cwd || '';
      const exec   = p.pm2_env?.pm_exec_path || '';
      const status = p.pm2_env?.status || 'unknown';
      const port   = p.pm2_env?.env?.PORT || p.pm2_env?.PORT || '';
      const text = [
        `pm2 service: ${name}`,
        `status: ${status}`,
        `cwd: ${cwd}`,
        `exec: ${exec}`,
        port ? `port: ${port}` : '',
      ].filter(Boolean).join('\n');
      docs.push({ source_type: 'pm2', source_path: `pm2://${name}`, chunk_text: text });

      // Also grab README, package.json description, first 50 lines of server.js
      if (cwd) {
        const readmePath = path.join(cwd, 'README.md');
        const readme = safeRead(readmePath);
        if (readme) {
          chunkText(readme, 'readme', readmePath).forEach(c => docs.push(c));
        }

        const pkgPath = path.join(cwd, 'package.json');
        const pkgRaw = safeRead(pkgPath);
        if (pkgRaw) {
          try {
            const pkg = JSON.parse(pkgRaw);
            const desc = pkg.description || '';
            const scripts = JSON.stringify(pkg.scripts || {});
            if (desc || scripts) {
              docs.push({
                source_type: 'package_json',
                source_path: pkgPath,
                chunk_text: `project: ${name}\ndescription: ${desc}\nscripts: ${scripts}`,
              });
            }
          } catch {}
        }

        // Prefer server.js, then index.js, then app.js
        for (const entry of ['server.js', 'index.js', 'app.js', 'src/server.js', 'src/index.js']) {
          const ep = path.join(cwd, entry);
          const lines = safeRead(ep, 50);
          if (lines) {
            docs.push({
              source_type: 'server_head',
              source_path: ep,
              chunk_text: `service: ${name} (${entry} first 50 lines)\n${lines}`,
            });
            break;
          }
        }
      }
    }
  } catch (e) {
    console.warn('[indexer] pm2 jlist failed:', e.message);
  }
  return docs;
}

function collectMemory() {
  const docs = [];
  // Claude keys its per-project memory dir on the working dir with slashes → dashes.
  // Derive it from the live HOME so this tracks the current machine instead of a
  // hardcoded pre-migration home (was -Users-stevestudio2, which post mac2→mac3
  // migration points at a STALE memory copy — the live memory is under the derived path).
  const dashedHome = HOME.replace(/\//g, '-');
  const memDir = path.join(HOME, '.claude/projects', `${dashedHome}/memory`);
  if (!fs.existsSync(memDir)) return docs;
  const files = fs.readdirSync(memDir).filter(f => f.endsWith('.md'));
  for (const f of files) {
    const fp = path.join(memDir, f);
    const text = safeRead(fp);
    if (text) chunkText(text, 'memory', fp).forEach(c => docs.push(c));
  }
  return docs;
}

function collectCncp() {
  const docs = [];
  const cfgPath = path.join(HOME, 'cncp-starter/cncp-config.json');
  const raw = safeRead(cfgPath);
  if (!raw) return docs;
  try {
    const cfg = JSON.parse(raw);
    const domains = cfg.domains || [];
    // Group into chunks of 10 domains each
    for (let i = 0; i < domains.length; i += 10) {
      const slice = domains.slice(i, i + 10);
      const text = slice.map(d =>
        typeof d === 'string' ? d : `${d.name || ''} | ${d.status || ''} | ${d.reg || ''} | ${d.suggestion || ''}`
      ).join('\n');
      docs.push({ source_type: 'cncp_domains', source_path: cfgPath, chunk_text: `domains:\n${text}` });
    }
  } catch (e) {
    console.warn('[indexer] cncp-config.json parse failed:', e.message);
  }
  return docs;
}

// ── main export ───────────────────────────────────────────────────────────────

async function buildIndex(onProgress) {
  const log = onProgress || console.log;

  log('[indexer] collecting pm2 + project files...');
  const pm2Docs    = collectPm2();
  log(`[indexer] pm2 yielded ${pm2Docs.length} chunks`);

  log('[indexer] collecting memory files...');
  const memDocs    = collectMemory();
  log(`[indexer] memory yielded ${memDocs.length} chunks`);

  log('[indexer] collecting cncp domains...');
  const cncpDocs   = collectCncp();
  log(`[indexer] cncp yielded ${cncpDocs.length} chunks`);

  const all = [...pm2Docs, ...memDocs, ...cncpDocs];
  log(`[indexer] total chunks to embed: ${all.length}`);

  const out = fs.createWriteStream(VECTORS_PATH);
  // Resolve only once the file is fully flushed to disk. server.js calls
  // refreshCache() (a synchronous read) immediately after buildIndex() returns,
  // so returning before 'finish' can load a truncated index. Attach the error
  // listener at creation so a mid-write error can't become an uncaught 'error'.
  const streamDone = new Promise((resolve, reject) => {
    out.on('finish', resolve);
    out.on('error', reject);
  });
  let ok = 0, fail = 0;

  for (let i = 0; i < all.length; i++) {
    const doc = all[i];
    doc.chunk_text = redact(doc.chunk_text);   // strip known secret shapes before embed/persist
    try {
      const embedding = await embed(doc.chunk_text);
      const record = { id: i, ...doc, embedding };
      out.write(JSON.stringify(record) + '\n');
      ok++;
      if (ok % 20 === 0) log(`[indexer] embedded ${ok}/${all.length}...`);
    } catch (e) {
      fail++;
      console.warn(`[indexer] embed fail for chunk ${i}:`, e.message);
    }
  }

  out.end();
  await streamDone;
  log(`[indexer] done. ok=${ok} fail=${fail} path=${VECTORS_PATH}`);
  return { ok, fail, path: VECTORS_PATH };
}

function loadVectors() {
  if (!fs.existsSync(VECTORS_PATH)) return [];
  const lines = fs.readFileSync(VECTORS_PATH, 'utf8').split('\n').filter(Boolean);
  // Skip (don't throw on) an unparseable line so one truncated/corrupt record can't
  // crash server startup — refreshCache() runs at module load and would take the
  // whole process down otherwise.
  const records = [];
  for (const l of lines) {
    try { records.push(JSON.parse(l)); }
    catch (e) { console.warn('[fleet-rag] skipping unparseable vectors.jsonl line:', e.message); }
  }
  return records;
}

module.exports = { buildIndex, loadVectors, VECTORS_PATH };