`build-corpus.js`

223 lines
#!/usr/bin/env node
/*
 * build-corpus.js — aggregate a year of work into one reviewable list (data/items.json).
 * Sources:
 *   A) CNCP wins             → shipped features/apps           (source: "win")
 *   B) DW-Programming/*       → standalone DW apps/tools        (source: "dw-app")
 *   C) wallco/WPB curators    → AI-storefront review tools      (source: "wpb-tool")
 * Each item: { id, source, title, project, date, summary, value, commit, path, port, hasUI }
 */
const fs = require('fs');
const path = require('path');
const os = require('os');

const HOME = os.homedir();
const OUT = path.join(__dirname, 'data', 'items.json');
const items = [];

// ---- A) CNCP wins -------------------------------------------------
function addWins() {
  const snap = path.join(__dirname, 'data', 'wins-snapshot.json');
  let wins = [];
  try { wins = JSON.parse(fs.readFileSync(snap, 'utf8')); } catch {}
  for (const w of wins) {
    items.push({
      id: w.id,
      source: 'win',
      title: w.title,
      project: w.project || '',
      date: w.date || (w.createdAt ? new Date(w.createdAt).toISOString().slice(0, 10) : ''),
      ts: w.createdAt || 0,
      summary: w.summary || '',
      value: w.valueToday || '',
      commit: w.commit || '',
      path: '',
      port: '',
      hasUI: false,
    });
  }
  return wins.length;
}

// ---- helpers for filesystem apps ----------------------------------
function firstReadmeLine(dir) {
  for (const name of ['README.md', 'readme.md', 'VERSION_NOTES.md', 'VERSION.md']) {
    const f = path.join(dir, name);
    if (fs.existsSync(f)) {
      const lines = fs.readFileSync(f, 'utf8').split('\n').map((l) => l.trim());
      // first non-empty, non-heading-only line, else first heading text
      const para = lines.find((l) => l && !l.startsWith('#') && !l.startsWith('---'));
      const head = lines.find((l) => l.startsWith('#'));
      const txt = (para || (head ? head.replace(/^#+\s*/, '') : '') || '').slice(0, 400);
      if (txt) return txt;
    }
  }
  return '';
}
function pkgDesc(dir) {
  const f = path.join(dir, 'package.json');
  if (fs.existsSync(f)) {
    try { return (JSON.parse(fs.readFileSync(f, 'utf8')).description || '').slice(0, 400); } catch {}
  }
  return '';
}
function detectPort(dir) {
  for (const name of ['ecosystem.config.js', '.env', 'server.js', 'index.js']) {
    const f = path.join(dir, name);
    if (fs.existsSync(f)) {
      try {
        const m = fs.readFileSync(f, 'utf8').match(/(?:PORT|port)\s*[:=]\s*['"]?(\d{3,5})/);
        if (m) return m[1];
      } catch {}
    }
  }
  return '';
}
function hasUI(dir) {
  if (fs.existsSync(path.join(dir, 'public', 'index.html'))) return true;
  try { return fs.readdirSync(dir).some((f) => f.endsWith('.html')); } catch { return false; }
}
function dirDate(dir) {
  try {
    const t = fs.statSync(dir).mtime;
    return { date: t.toISOString().slice(0, 10), ts: t.getTime() };
  } catch { return { date: '', ts: 0 }; }
}

// ---- B) DW-Programming app folders --------------------------------
const SKIP_DIRS = new Set([
  'config', 'data', 'lib', 'ls', 'scripts', 'state', 'templates',
  'shared-enhancements', 'node_modules', '.git',
]);
function addDirApps(parent, source, projectLabel) {
  if (!fs.existsSync(parent)) return 0;
  let n = 0;
  for (const name of fs.readdirSync(parent)) {
    if (SKIP_DIRS.has(name) || name.startsWith('.')) continue;
    const dir = path.join(parent, name);
    let st; try { st = fs.statSync(dir); } catch { continue; }
    if (!st.isDirectory()) continue;
    const { date, ts } = dirDate(dir);
    const summary = firstReadmeLine(dir) || pkgDesc(dir) || '';
    items.push({
      id: `${source}:${name}`,
      source,
      title: name,
      project: projectLabel,
      date, ts,
      summary,
      value: '',
      commit: '',
      path: dir,
      port: detectPort(dir),
      hasUI: hasUI(dir),
    });
    n++;
  }
  return n;
}

// ---- C) Standalone curator/viewer HTML tools (wallco + WPB) -------
function addHtmlTools() {
  const roots = [
    { dir: path.join(HOME, 'Projects/wallco-ai/public'), source: 'wpb-tool', label: 'wallco-ai' },
    { dir: path.join(HOME, 'Projects/WallpapersBack/wallpapersback-fork/public'), source: 'wpb-tool', label: 'wallpapersback' },
  ];
  let n = 0;
  for (const { dir, source, label } of roots) {
    if (!fs.existsSync(dir)) continue;
    for (const f of fs.readdirSync(dir)) {
      if (!f.endsWith('.html')) continue;
      if (['index.html', 'privacy.html', 'about.html'].includes(f)) continue;
      const full = path.join(dir, f);
      const { date, ts } = dirDate(full);
      items.push({
        id: `${source}:${label}:${f}`,
        source,
        title: f.replace(/\.html$/, '').replace(/[-_]/g, ' '),
        project: label,
        date, ts,
        summary: `Standalone review/curator tool (${f}) in the ${label} storefront.`,
        value: '',
        commit: '',
        path: full,
        port: '',
        hasUI: true,
      });
      n++;
    }
  }
  return n;
}

// ---- D) skills + E) agents (the "forgotten tool" review) ----------
function frontmatter(file) {
  try {
    const t = fs.readFileSync(file, 'utf8');
    const m = t.match(/^---\n([\s\S]*?)\n---/);
    if (!m) return {};
    const fm = {};
    const nm = m[1].match(/^name:\s*(.+)$/m); if (nm) fm.name = nm[1].trim();
    const ds = m[1].match(/^description:\s*([\s\S]+?)(?:\n[a-z_]+:|\n*$)/m);
    if (ds) fm.description = ds[1].replace(/\s+/g, ' ').trim();
    return fm;
  } catch { return {}; }
}
function addSkills() {
  const root = path.join(HOME, '.claude/skills');
  if (!fs.existsSync(root)) return 0;
  let n = 0;
  for (const name of fs.readdirSync(root)) {
    const sk = path.join(root, name, 'SKILL.md');
    if (!fs.existsSync(sk)) continue;
    let st; try { st = fs.statSync(sk); } catch { continue; }
    const fm = frontmatter(sk);
    items.push({
      id: `skill:${name}`, source: 'skill', title: fm.name || name, project: 'skills',
      date: st.mtime.toISOString().slice(0, 10), ts: st.mtime.getTime(),
      summary: (fm.description || '').slice(0, 500), value: '', commit: '',
      path: sk, port: '', hasUI: false,
    });
    n++;
  }
  return n;
}
function addAgents() {
  const root = path.join(HOME, '.claude/agents');
  if (!fs.existsSync(root)) return 0;
  let n = 0;
  for (const f of fs.readdirSync(root)) {
    if (!f.endsWith('.md')) continue;
    const fp = path.join(root, f);
    let st; try { st = fs.statSync(fp); if (!st.isFile()) continue; } catch { continue; }
    const fm = frontmatter(fp);
    items.push({
      id: `agent:${f.replace(/\.md$/, '')}`, source: 'agent',
      title: fm.name || f.replace(/\.md$/, ''), project: 'agents',
      date: st.mtime.toISOString().slice(0, 10), ts: st.mtime.getTime(),
      summary: (fm.description || '').slice(0, 500), value: '', commit: '',
      path: fp, port: '', hasUI: false,
    });
    n++;
  }
  return n;
}

// ---- run ----------------------------------------------------------
const nWins = addWins();
const nDW = addDirApps(path.join(HOME, 'Projects/Designer-Wallcoverings/DW-Programming'), 'dw-app', 'DW-Programming');
const nTools = addHtmlTools();
const nSkills = addSkills();
const nAgents = addAgents();

// stable sort: newest first
items.sort((a, b) => (b.ts || 0) - (a.ts || 0));
fs.writeFileSync(OUT, JSON.stringify(items, null, 2));
console.log(`corpus built → ${OUT}`);
console.log(`  wins:     ${nWins}`);
console.log(`  dw-apps:  ${nDW}`);
console.log(`  wpb-tools:${nTools}`);
console.log(`  skills:   ${nSkills}`);
console.log(`  agents:   ${nAgents}`);
console.log(`  TOTAL:    ${items.length}`);