← back to Cli Printing Press
feat(pipeline): add dogfood automation, anti-AI text filter, and README augmentation
bc5c5db85332ca9a13a5eda5f8c3eaa358afb93c · 2026-03-24 20:28:09 -0700 · Matt Van Horn
- Create dogfood.go with 3-tier CLI testing (no-auth, read-only, sandbox-write)
with evidence capture to dogfood-evidence/ directory and scoring (0-50)
- Create textfilter.go with regex-based anti-AI slop detection
(warns on "comprehensive", "robust", "leverage", etc.)
- Create readme_augment.go that injects real command output from dogfood
evidence into generated README files (marker-based or append mode)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Files touched
A internal/generator/readme_augment.goA internal/generator/textfilter.goA internal/pipeline/dogfood.go
Diff
commit bc5c5db85332ca9a13a5eda5f8c3eaa358afb93c
Author: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Date: Tue Mar 24 20:28:09 2026 -0700
feat(pipeline): add dogfood automation, anti-AI text filter, and README augmentation
- Create dogfood.go with 3-tier CLI testing (no-auth, read-only, sandbox-write)
with evidence capture to dogfood-evidence/ directory and scoring (0-50)
- Create textfilter.go with regex-based anti-AI slop detection
(warns on "comprehensive", "robust", "leverage", etc.)
- Create readme_augment.go that injects real command output from dogfood
evidence into generated README files (marker-based or append mode)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
internal/generator/readme_augment.go | 121 ++++++++++++++++++
internal/generator/textfilter.go | 72 +++++++++++
internal/pipeline/dogfood.go | 231 +++++++++++++++++++++++++++++++++++
3 files changed, 424 insertions(+)
diff --git a/internal/generator/readme_augment.go b/internal/generator/readme_augment.go
new file mode 100644
index 00000000..0515d004
--- /dev/null
+++ b/internal/generator/readme_augment.go
@@ -0,0 +1,121 @@
+package generator
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// AugmentREADME reads evidence files from a dogfood run and injects
+// real command output into the generated README. It looks for marker
+// comments like <!-- HELP_OUTPUT --> and replaces them with captured output.
+//
+// If no markers exist, it appends a "Real Usage Examples" section.
+func AugmentREADME(readmePath, evidenceDir string) error {
+ content, err := os.ReadFile(readmePath)
+ if err != nil {
+ return fmt.Errorf("reading README: %w", err)
+ }
+
+ readme := string(content)
+ modified := false
+
+ // Try marker-based replacement first
+ markers := map[string]string{
+ "<!-- HELP_OUTPUT -->": filepath.Join(evidenceDir, "tier1-help.txt"),
+ "<!-- VERSION_OUTPUT -->": filepath.Join(evidenceDir, "tier1-version.txt"),
+ "<!-- DOCTOR_OUTPUT -->": filepath.Join(evidenceDir, "tier1-doctor.txt"),
+ }
+
+ for marker, evidenceFile := range markers {
+ if strings.Contains(readme, marker) {
+ output, err := os.ReadFile(evidenceFile)
+ if err != nil {
+ continue
+ }
+ replacement := fmt.Sprintf("```\n%s```", string(output))
+ readme = strings.Replace(readme, marker, replacement, 1)
+ modified = true
+ }
+ }
+
+ // If no markers found, append a section with real output
+ if !modified {
+ var section strings.Builder
+ section.WriteString("\n## Real Usage Examples\n\n")
+ section.WriteString("*Captured from automated dogfood testing.*\n\n")
+
+ helpOutput, err := os.ReadFile(filepath.Join(evidenceDir, "tier1-help.txt"))
+ if err == nil && len(helpOutput) > 0 {
+ section.WriteString("### Help\n\n```\n")
+ section.Write(helpOutput)
+ section.WriteString("```\n\n")
+ modified = true
+ }
+
+ doctorOutput, err := os.ReadFile(filepath.Join(evidenceDir, "tier1-doctor.txt"))
+ if err == nil && len(doctorOutput) > 0 {
+ section.WriteString("### Health Check\n\n```\n")
+ section.Write(doctorOutput)
+ section.WriteString("```\n\n")
+ modified = true
+ }
+
+ // Add per-resource help if available
+ resDir := filepath.Join(evidenceDir, "tier1-resources")
+ entries, _ := os.ReadDir(resDir)
+ for _, e := range entries {
+ if e.IsDir() || !strings.HasSuffix(e.Name(), "-help.txt") {
+ continue
+ }
+ resName := strings.TrimSuffix(e.Name(), "-help.txt")
+ output, err := os.ReadFile(filepath.Join(resDir, e.Name()))
+ if err == nil && len(output) > 0 {
+ section.WriteString(fmt.Sprintf("### %s\n\n```\n", resName))
+ section.Write(output)
+ section.WriteString("```\n\n")
+ }
+ }
+
+ // Add tier 2 real API output if available
+ tier2Dir := filepath.Join(evidenceDir, "tier2-reads")
+ t2entries, _ := os.ReadDir(tier2Dir)
+ if len(t2entries) > 0 {
+ section.WriteString("### Real API Output\n\n")
+ for _, e := range t2entries {
+ if e.IsDir() {
+ continue
+ }
+ output, err := os.ReadFile(filepath.Join(tier2Dir, e.Name()))
+ if err == nil && len(output) > 0 {
+ cmdName := strings.TrimSuffix(e.Name(), ".txt")
+ section.WriteString(fmt.Sprintf("#### %s\n\n```json\n", cmdName))
+ // Truncate very long output
+ s := string(output)
+ if len(s) > 2000 {
+ s = s[:2000] + "\n... (truncated)\n"
+ }
+ section.WriteString(s)
+ section.WriteString("```\n\n")
+ }
+ }
+ }
+
+ if modified {
+ readme += section.String()
+ }
+ }
+
+ if !modified {
+ return nil // no evidence to inject
+ }
+
+ // Run anti-AI text filter on the README
+ warnings := CheckText(readme)
+ if len(warnings) > 0 {
+ fmt.Fprint(os.Stderr, FormatWarnings(warnings))
+ }
+
+ return os.WriteFile(readmePath, []byte(readme), 0o644)
+}
diff --git a/internal/generator/textfilter.go b/internal/generator/textfilter.go
new file mode 100644
index 00000000..212c75ff
--- /dev/null
+++ b/internal/generator/textfilter.go
@@ -0,0 +1,72 @@
+package generator
+
+import (
+ "fmt"
+ "regexp"
+ "strings"
+)
+
+// AITextWarning records a single match of AI-characteristic text.
+type AITextWarning struct {
+ Pattern string
+ Match string
+ Line int
+ Context string // surrounding text
+}
+
+// aiSlopPatterns are regex patterns that detect common AI-generated text patterns.
+var aiSlopPatterns = []*regexp.Regexp{
+ regexp.MustCompile(`(?i)\b(comprehensive|robust|seamless|leverage|utilize|facilitate)\b`),
+ regexp.MustCompile(`(?i)here's a .* that`),
+ regexp.MustCompile(`(?i)not just .*, it's`),
+ regexp.MustCompile(`(?i)whether you're .* or .*,`),
+ regexp.MustCompile(`(?i)\b(streamline|empower|cutting-edge|game-changer)\b`),
+ regexp.MustCompile(`(?i)\b(revolutionize|transform|elevate|harness)\b`),
+ regexp.MustCompile(`(?i)in today's .* landscape`),
+ regexp.MustCompile(`(?i)take .* to the next level`),
+}
+
+// CheckText scans text for AI-characteristic patterns and returns warnings.
+// It never modifies text - only reports matches.
+func CheckText(text string) []AITextWarning {
+ var warnings []AITextWarning
+ lines := strings.Split(text, "\n")
+
+ for lineNum, line := range lines {
+ for _, pat := range aiSlopPatterns {
+ matches := pat.FindAllString(line, -1)
+ for _, m := range matches {
+ warnings = append(warnings, AITextWarning{
+ Pattern: pat.String(),
+ Match: m,
+ Line: lineNum + 1,
+ Context: truncateContext(line, 80),
+ })
+ }
+ }
+ }
+
+ return warnings
+}
+
+// FormatWarnings returns a human-readable summary of text filter warnings.
+func FormatWarnings(warnings []AITextWarning) string {
+ if len(warnings) == 0 {
+ return ""
+ }
+
+ var b strings.Builder
+ fmt.Fprintf(&b, "anti-AI text filter: %d warning(s)\n", len(warnings))
+ for _, w := range warnings {
+ fmt.Fprintf(&b, " line %d: matched %q in: %s\n", w.Line, w.Match, w.Context)
+ }
+ return b.String()
+}
+
+func truncateContext(s string, maxLen int) string {
+ s = strings.TrimSpace(s)
+ if len(s) <= maxLen {
+ return s
+ }
+ return s[:maxLen-3] + "..."
+}
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
new file mode 100644
index 00000000..8f91c3ca
--- /dev/null
+++ b/internal/pipeline/dogfood.go
@@ -0,0 +1,231 @@
+package pipeline
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// DogfoodResults holds the output of the dogfood phase.
+type DogfoodResults struct {
+ Tier int `json:"tier"`
+ TotalCommands int `json:"total_commands"`
+ PassedCommands int `json:"passed_commands"`
+ FailedCommands int `json:"failed_commands"`
+ Commands []CommandResult `json:"commands"`
+ Score int `json:"score"` // 0-50
+}
+
+// CommandResult records a single dogfood command execution.
+type CommandResult struct {
+ Tier int `json:"tier"`
+ Command string `json:"command"`
+ ExitCode int `json:"exit_code"`
+ StdoutFile string `json:"stdout_file"`
+ Stderr string `json:"stderr"`
+ DurationMs int `json:"duration_ms"`
+ Pass bool `json:"pass"`
+}
+
+// DogfoodConfig controls what tiers to run.
+type DogfoodConfig struct {
+ BinaryPath string
+ PipelineDir string
+ MaxTier int // 1, 2, or 3
+ SandboxSafe bool // from KnownSpecs
+ CmdTimeout time.Duration // per-command timeout
+ Resources []string // resource names from the generated CLI
+ AuthEnvVars []string // env vars that indicate credentials are available
+}
+
+// RunDogfood executes tiered dogfood testing against a generated CLI binary.
+func RunDogfood(cfg DogfoodConfig) (*DogfoodResults, error) {
+ if cfg.CmdTimeout == 0 {
+ cfg.CmdTimeout = 15 * time.Second
+ }
+ if cfg.MaxTier == 0 {
+ cfg.MaxTier = 1
+ }
+
+ evidenceDir := filepath.Join(cfg.PipelineDir, "dogfood-evidence")
+ if err := os.MkdirAll(evidenceDir, 0o755); err != nil {
+ return nil, fmt.Errorf("creating evidence dir: %w", err)
+ }
+
+ results := &DogfoodResults{Tier: 1}
+
+ // Tier 1: No auth required - always runs
+ tier1Commands := buildTier1Commands(cfg.BinaryPath, cfg.Resources)
+ for _, cmd := range tier1Commands {
+ cr := runCommand(cfg.BinaryPath, cmd, 1, cfg.CmdTimeout, evidenceDir)
+ results.Commands = append(results.Commands, cr)
+ }
+
+ // Tier 2: Read-only with auth - if credentials available and tier >= 2
+ if cfg.MaxTier >= 2 && hasCredentials(cfg.AuthEnvVars) {
+ results.Tier = 2
+ tier2Dir := filepath.Join(evidenceDir, "tier2-reads")
+ os.MkdirAll(tier2Dir, 0o755)
+ tier2Commands := buildTier2Commands(cfg.BinaryPath, cfg.Resources)
+ for _, cmd := range tier2Commands {
+ cr := runCommand(cfg.BinaryPath, cmd, 2, cfg.CmdTimeout, tier2Dir)
+ results.Commands = append(results.Commands, cr)
+ }
+ }
+
+ // Tier 3: Sandbox write ops - only if sandbox safe and tier >= 3
+ if cfg.MaxTier >= 3 && cfg.SandboxSafe {
+ results.Tier = 3
+ tier3Dir := filepath.Join(evidenceDir, "tier3-writes")
+ os.MkdirAll(tier3Dir, 0o755)
+ // Tier 3 is API-specific and would need per-API test plans.
+ // For now, just record that tier 3 was available.
+ fmt.Fprintf(os.Stderr, "info: Tier 3 sandbox testing available but no test plan defined\n")
+ }
+
+ // Compute results
+ for _, cr := range results.Commands {
+ results.TotalCommands++
+ if cr.Pass {
+ results.PassedCommands++
+ } else {
+ results.FailedCommands++
+ }
+ }
+
+ results.Score = computeDogfoodScore(results)
+
+ // Write results
+ if err := writeDogfoodResults(results, cfg.PipelineDir); err != nil {
+ return results, fmt.Errorf("writing results: %w", err)
+ }
+
+ return results, nil
+}
+
+// LoadDogfoodResults reads dogfood-results.json from a pipeline directory.
+func LoadDogfoodResults(pipelineDir string) (*DogfoodResults, error) {
+ data, err := os.ReadFile(filepath.Join(pipelineDir, "dogfood-results.json"))
+ if err != nil {
+ return nil, err
+ }
+ var r DogfoodResults
+ if err := json.Unmarshal(data, &r); err != nil {
+ return nil, err
+ }
+ return &r, nil
+}
+
+func buildTier1Commands(binary string, resources []string) []dogfoodCmd {
+ cmds := []dogfoodCmd{
+ {args: []string{"--help"}, evidenceFile: "tier1-help.txt"},
+ {args: []string{"version"}, evidenceFile: "tier1-version.txt"},
+ {args: []string{"doctor"}, evidenceFile: "tier1-doctor.txt"},
+ }
+ // Add per-resource help commands
+ resDir := "tier1-resources"
+ for _, r := range resources {
+ cmds = append(cmds, dogfoodCmd{
+ args: []string{r, "--help"},
+ evidenceFile: filepath.Join(resDir, r+"-help.txt"),
+ })
+ }
+ return cmds
+}
+
+func buildTier2Commands(binary string, resources []string) []dogfoodCmd {
+ var cmds []dogfoodCmd
+ for _, r := range resources {
+ cmds = append(cmds, dogfoodCmd{
+ args: []string{r, "list", "--json"},
+ evidenceFile: r + "-list.txt",
+ })
+ }
+ return cmds
+}
+
+type dogfoodCmd struct {
+ args []string
+ evidenceFile string
+}
+
+func runCommand(binaryPath string, dc dogfoodCmd, tier int, timeout time.Duration, evidenceDir string) CommandResult {
+ cr := CommandResult{
+ Tier: tier,
+ Command: strings.Join(dc.args, " "),
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ start := time.Now()
+ cmd := exec.CommandContext(ctx, binaryPath, dc.args...)
+ stdout, err := cmd.CombinedOutput()
+ cr.DurationMs = int(time.Since(start).Milliseconds())
+
+ if err != nil {
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ cr.ExitCode = exitErr.ExitCode()
+ } else {
+ cr.ExitCode = -1
+ }
+ cr.Stderr = err.Error()
+ cr.Pass = false
+ } else {
+ cr.ExitCode = 0
+ cr.Pass = true
+ }
+
+ // Write evidence file
+ evidencePath := filepath.Join(evidenceDir, dc.evidenceFile)
+ os.MkdirAll(filepath.Dir(evidencePath), 0o755)
+ os.WriteFile(evidencePath, stdout, 0o644)
+ cr.StdoutFile = evidencePath
+
+ return cr
+}
+
+func hasCredentials(envVars []string) bool {
+ for _, v := range envVars {
+ if os.Getenv(v) != "" {
+ return true
+ }
+ }
+ return false
+}
+
+func computeDogfoodScore(results *DogfoodResults) int {
+ if results.TotalCommands == 0 {
+ return 0
+ }
+
+ // Base score: percentage of passed commands, scaled to 0-40
+ passRate := float64(results.PassedCommands) / float64(results.TotalCommands)
+ score := int(passRate * 40)
+
+ // Bonus for higher tiers
+ switch results.Tier {
+ case 2:
+ score += 5
+ case 3:
+ score += 10
+ }
+
+ if score > 50 {
+ score = 50
+ }
+ return score
+}
+
+func writeDogfoodResults(results *DogfoodResults, pipelineDir string) error {
+ data, err := json.MarshalIndent(results, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(pipelineDir, "dogfood-results.json"), data, 0o644)
+}
← 466715fb feat(pipeline): add Research and Comparative phases with cat
·
back to Cli Printing Press
·
feat(pipeline): add comparative analysis scoring and GoRelea 1f5871bd →