Add overnight SEO and AI discovery audit

5df533c9d269ee5d8308503acacd88afa09a2789 · 2026-08-31 02:08:01 -0700 · Steve Abrams
Files touched

M README.md
A audit-search-discovery.mjs
A data/search-properties.json
M package.json
A run-search-audit-loop.sh
Diff

commit 5df533c9d269ee5d8308503acacd88afa09a2789
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Aug 31 02:08:01 2026 -0700

    Add overnight SEO and AI discovery audit
---
 README.md                   |  7 +++++++
 audit-search-discovery.mjs  | 42 ++++++++++++++++++++++++++++++++++++++++++
 data/search-properties.json | 34 ++++++++++++++++++++++++++++++++++
 package.json                |  5 +++++
 run-search-audit-loop.sh    | 22 ++++++++++++++++++++++
 5 files changed, 110 insertions(+)

diff --git a/README.md b/README.md
index 7048e51..3d88905 100644
--- a/README.md
+++ b/README.md
@@ -20,8 +20,15 @@ clusters** (the real consolidation candidates).
 
 ```bash
 node build-registry.mjs        # re-scan both fleets, rewrite dw-fleet-registry.json
+npm run audit:search           # verify robots, sitemap, and llms.txt on registered public sites
+npm run audit:search:overnight # recheck every 15 minutes for 12 hours
 ```
 
+`data/search-properties.json` also records adjacent public properties and internal
+producers that matter to discovery. `gated-publisher` entries are inventory only:
+the audit never starts them or calls a publishing path. Results are written to
+`search-discovery-report.json` for SEO/AEO monitoring.
+
 Read-only scan; re-runnable. Same inputs → same registry (plus a fresh timestamp).
 
 ## Output: `dw-fleet-registry.json`
diff --git a/audit-search-discovery.mjs b/audit-search-discovery.mjs
new file mode 100644
index 0000000..f2857b7
--- /dev/null
+++ b/audit-search-discovery.mjs
@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+
+const configUrl = new URL('./data/search-properties.json', import.meta.url);
+const outputUrl = new URL('./search-discovery-report.json', import.meta.url);
+const config = JSON.parse(await fs.readFile(configUrl, 'utf8'));
+const results = [];
+
+for (const property of config.properties) {
+  if (property.kind !== 'public-site') {
+    results.push({ id: property.id, kind: property.kind, status: 'not-probed', reason: property.startPolicy || 'not a public search surface' });
+    continue;
+  }
+  const checks = [];
+  for (const endpoint of property.discovery) {
+    const url = new URL(endpoint, property.localUrl).href;
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
+      const contentType = response.headers.get('content-type') || '';
+      const text = await response.text();
+      checks.push({ endpoint, status: response.status, ok: response.ok, contentType, bytes: Buffer.byteLength(text) });
+    } catch (error) {
+      checks.push({ endpoint, status: null, ok: false, error: error.message });
+    }
+  }
+  results.push({ id: property.id, kind: property.kind, status: checks.every(c => c.ok) ? 'ready' : 'incomplete', checks });
+}
+
+const report = {
+  $schema: 'search-discovery-report/v1',
+  generatedAt: new Date().toISOString(),
+  summary: {
+    ready: results.filter(r => r.status === 'ready').length,
+    incomplete: results.filter(r => r.status === 'incomplete').length,
+    gatedOrInternal: results.filter(r => r.status === 'not-probed').length
+  },
+  results
+};
+await fs.writeFile(outputUrl, JSON.stringify(report, null, 2) + '\n');
+console.log(JSON.stringify(report.summary));
+for (const result of results) console.log(`${result.id}: ${result.status}`);
+if (report.summary.incomplete) process.exitCode = 1;
diff --git a/data/search-properties.json b/data/search-properties.json
new file mode 100644
index 0000000..c4f5d5a
--- /dev/null
+++ b/data/search-properties.json
@@ -0,0 +1,34 @@
+{
+  "$schema": "search-properties/v1",
+  "properties": [
+    {
+      "id": "quadrille-house-site",
+      "kind": "public-site",
+      "repoPath": "/Users/macstudio3/Projects/quadrille-house-site",
+      "localUrl": "http://127.0.0.1:9943",
+      "canonicalUrl": "https://quadrille.designerwallcoverings.com",
+      "discovery": ["/", "/robots.txt", "/sitemap.xml", "/llms.txt"]
+    },
+    {
+      "id": "CelebritySignatures",
+      "kind": "public-site",
+      "repoPath": "/Users/macstudio3/Projects/CelebritySignatures",
+      "localUrl": "http://127.0.0.1:9956",
+      "canonicalUrl": "https://celebsignatures.com",
+      "discovery": ["/", "/robots.txt", "/sitemap.xml", "/llms.txt"]
+    },
+    {
+      "id": "animate-museum-posts",
+      "kind": "gated-publisher",
+      "repoPath": "/Users/macstudio3/Projects/animate-museum-posts",
+      "startPolicy": "explicit-approval-required",
+      "discovery": []
+    },
+    {
+      "id": "dw-fleet-registry",
+      "kind": "control-plane",
+      "repoPath": "/Users/macstudio3/Projects/dw-fleet-registry",
+      "discovery": []
+    }
+  ]
+}
diff --git a/package.json b/package.json
index 749b4cb..a45ac8c 100644
--- a/package.json
+++ b/package.json
@@ -2,6 +2,11 @@
   "name": "dw-fleet-registry-viewer",
   "private": true,
   "description": "DW fleet registry viewer — zero hard deps; sharp is optional for /img proxy-side image resizing.",
+  "scripts": {
+    "build": "node build-registry.mjs",
+    "audit:search": "node audit-search-discovery.mjs",
+    "audit:search:overnight": "zsh run-search-audit-loop.sh"
+  },
   "optionalDependencies": {
     "sharp": "^0.33.5"
   }
diff --git a/run-search-audit-loop.sh b/run-search-audit-loop.sh
new file mode 100755
index 0000000..9df45c4
--- /dev/null
+++ b/run-search-audit-loop.sh
@@ -0,0 +1,22 @@
+#!/bin/zsh
+set -u
+
+ROOT="${0:A:h}"
+INTERVAL_SECONDS="${SEARCH_AUDIT_INTERVAL_SECONDS:-900}"
+DURATION_SECONDS="${SEARCH_AUDIT_DURATION_SECONDS:-43200}"
+DEADLINE=$(( EPOCHSECONDS + DURATION_SECONDS ))
+LOG="$ROOT/search-discovery-loop.log"
+
+cd "$ROOT" || exit 1
+print -r -- "$(date -u +%FT%TZ) loop-start interval=${INTERVAL_SECONDS}s duration=${DURATION_SECONDS}s" >> "$LOG"
+
+while (( EPOCHSECONDS < DEADLINE )); do
+  if node audit-search-discovery.mjs >> "$LOG" 2>&1; then
+    print -r -- "$(date -u +%FT%TZ) audit-pass" >> "$LOG"
+  else
+    print -r -- "$(date -u +%FT%TZ) audit-fail" >> "$LOG"
+  fi
+  sleep "$INTERVAL_SECONDS"
+done
+
+print -r -- "$(date -u +%FT%TZ) loop-complete" >> "$LOG"
← ba7d3db chore: macstudio3 migration — reconcile from mac2 + repoint · back to Dw Fleet Registry · Keep discovery audit output ephemeral babde29 →