← back to build-pages
auto-discover every ~/Projects git repo (541) with a background warmer + TTL cache: git scans run off the request path so home stays ~80ms; curated projects stay featured-first
a16bd035d879f27de73c10fe95acee2d1eab27b7 · 2026-07-28 07:54:53 -0700 · steve@designerwallcoverings.com
Files touched
Diff
commit a16bd035d879f27de73c10fe95acee2d1eab27b7
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Tue Jul 28 07:54:53 2026 -0700
auto-discover every ~/Projects git repo (541) with a background warmer + TTL cache: git scans run off the request path so home stays ~80ms; curated projects stay featured-first
---
server.js | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 71 insertions(+), 10 deletions(-)
diff --git a/server.js b/server.js
index 10da124..cd2cf62 100644
--- a/server.js
+++ b/server.js
@@ -29,9 +29,11 @@ const GIT_SHA = (() => {
})();
const STARTED_AT_ISO = new Date().toISOString();
-// Registry: kebab-slug → {name, repo, blurb}. Add a project by appending here.
-// Slug is the URL path /p/<slug>; repo is an absolute path to the working tree.
-const PROJECTS = {
+// CURATED registry: kebab-slug → {name, repo, blurb}. These get a hand-written
+// name + blurb and sort first (featured). Everything else is AUTO-DISCOVERED from
+// ~/Projects (any top-level dir with a .git). Curated entries override the
+// auto-discovered one for the same repo, so the nice copy wins.
+const CURATED = {
butlr: {
name: 'Butlr',
repo: path.join(HOME, 'Projects/holdforme'),
@@ -58,6 +60,32 @@ const PROJECTS = {
},
};
+// "small-business-builder" → "Small Business Builder" (keeps ALLCAPS acronyms).
+function prettyName(slug) {
+ return String(slug).replace(/[-_]+/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
+}
+
+// Build the live registry: every top-level ~/Projects dir with a .git, plus the
+// curated overrides. Curated repos are keyed by path so an entry whose slug ≠
+// dirname (e.g. butlr → holdforme) isn't also listed under its raw dirname.
+function discoverProjects() {
+ const base = path.join(HOME, 'Projects');
+ const out = {};
+ const curatedRepos = new Set(Object.values(CURATED).map((p) => p.repo));
+ let dirents = [];
+ try { dirents = fs.readdirSync(base, { withFileTypes: true }); } catch (_) { /* fall through to curated-only */ }
+ for (const d of dirents) {
+ if (!d.isDirectory()) continue;
+ const repo = path.join(base, d.name);
+ if (curatedRepos.has(repo)) continue; // a curated entry already covers this repo
+ if (!fs.existsSync(path.join(repo, '.git'))) continue; // only real git repos
+ out[d.name] = { name: prettyName(d.name), repo, blurb: '', aliases: [], discovered: true };
+ }
+ for (const [slug, p] of Object.entries(CURATED)) out[slug] = { ...p, featured: true };
+ return out;
+}
+const PROJECTS = discoverProjects();
+
// ─────────────────────────────────────────────────────────────────────────────
// git helpers — narrow, single-purpose, all run with cwd=repo. Each returns a
// JSON-friendly shape so the routes are just glue.
@@ -72,10 +100,19 @@ async function gitRun(repo, args) {
// drops from ~80ms to ~10ms once warm. The HEAD probe still runs every hit
// so a `git commit` in the source repo shows up on the next refresh.
const _projectCache = new Map();
-async function getProjectBundle(slug, p) {
- const head = (await gitRun(p.repo, ['rev-parse', 'HEAD'])).trim();
+// With ~540 auto-discovered repos, probing HEAD on every hit would spawn 540 git
+// processes per request. So the REQUEST path serves the warm cache untouched for
+// BUNDLE_TTL; a background warmer (force=true) is the only thing that re-checks
+// HEAD and rebuilds changed repos. Keep BUNDLE_TTL > the warm interval so a
+// request never falls through to git.
+const BUNDLE_TTL = 100_000;
+async function getProjectBundle(slug, p, force = false) {
const cached = _projectCache.get(slug);
- if (cached && cached.head === head) return cached;
+ if (!force && cached && Date.now() - cached.at < BUNDLE_TTL) return cached;
+ let head;
+ try { head = (await gitRun(p.repo, ['rev-parse', 'HEAD'])).trim(); }
+ catch (_) { return cached || null; } // empty / broken repo — keep any prior bundle
+ if (cached && cached.head === head) { cached.at = Date.now(); return cached; }
const commits = await listCommits(p.repo);
const facets = extractFacets(commits);
const files = await listFiles(p.repo);
@@ -84,6 +121,22 @@ async function getProjectBundle(slug, p) {
return bundle;
}
+// Bounded-concurrency fan-out so a fleet-wide sweep never launches 540 git
+// processes at once.
+async function mapLimit(items, limit, fn) {
+ const out = new Array(items.length);
+ let i = 0;
+ const run = async () => { while (i < items.length) { const idx = i++; out[idx] = await fn(items[idx], idx); } };
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
+ return out;
+}
+
+// Background warmer — keeps every bundle warm OFF the request path. Runs at boot
+// and on an interval (< BUNDLE_TTL) so user requests only ever read cache.
+async function warmAll() {
+ await mapLimit(Object.entries(PROJECTS), 12, ([slug, p]) => getProjectBundle(slug, p, true).catch(() => null));
+}
+
async function listCommits(repo) {
const fmt = '%H%x01%h%x01%ai%x01%an%x01%s%x01%b%x02';
const raw = await gitRun(repo, ['log', '--no-merges', `--pretty=format:${fmt}`]);
@@ -357,7 +410,7 @@ app.get('/', async (req, res, next) => {
const tzPT = 'America/Los_Angeles';
const todayStartMs = startOfDayInTz(todayInTz(tzPT), tzPT);
const todayEndMs = todayStartMs + 86400000;
- const entries = await Promise.all(Object.entries(PROJECTS).map(async ([slug, p]) => {
+ const entries = await mapLimit(Object.entries(PROJECTS), 16, async ([slug, p]) => {
const bundle = await getProjectBundle(slug, p).catch(() => null);
const count = bundle ? bundle.commits.length : 0;
const latest = bundle && bundle.commits[0] ? bundle.commits[0].dateISO.slice(0, 10) : '';
@@ -372,7 +425,10 @@ app.get('/', async (req, res, next) => {
}
}
return { slug, p, count, latest, latestHash, ideas, today };
- }));
+ });
+ // Featured (curated) first, then alphabetical by name — 540 repos in raw
+ // readdir order is unusable.
+ entries.sort((a, b) => (b.p.featured ? 1 : 0) - (a.p.featured ? 1 : 0) || a.p.name.localeCompare(b.p.name));
const totalCommits = entries.reduce((a, e) => a + e.count, 0);
const totalIdeas = entries.reduce((a, e) => a + e.ideas, 0);
// Velocity counters + 7-day sparkline across the whole fleet. Concats
@@ -484,7 +540,7 @@ app.get('/', async (req, res, next) => {
<section class="bp-section">
<h2>Top skills + agents across the fleet</h2>
- <p class="bp-meta">Auto-extracted from commit messages — how often each skill/agent has been invoked across all 4 build journals.</p>
+ <p class="bp-meta">Auto-extracted from commit messages — how often each skill/agent has been invoked across all ${entries.length} build journals.</p>
<div class="bp-grid">
<div>
<h3 style="font-size:15px;margin:0 0 10px;">Skills</h3>
@@ -1474,5 +1530,10 @@ app.use((err, _req, res, _next) => {
const LISTEN_HOST = process.env.HOST || '127.0.0.1';
app.listen(PORT, LISTEN_HOST, () => {
- console.log(`build-pages listening on http://${LISTEN_HOST}:${PORT}`);
+ console.log(`build-pages listening on http://${LISTEN_HOST}:${PORT} — ${Object.keys(PROJECTS).length} projects`);
+ // Warm the whole fleet in the background so the first request reads cache, then
+ // re-warm every 90s (< BUNDLE_TTL) to pick up new commits without a request ever
+ // touching git across 540 repos.
+ warmAll().then(() => console.log('build-pages: fleet cache warmed')).catch(() => {});
+ setInterval(() => warmAll().catch(() => {}), 90_000);
});
← 0a57042 deploy.conf: tailnet-proxy model; live at build-pages.agenta
·
back to build-pages
·
(newest)