← back to A2a Lab
scorer fix (field-weight + trigram-downweight + synonyms) + independent held-out eval
ac91d16802df9e1a3a2f28fe32f0d777c9920e7f · 2026-08-01 22:33:22 -0700 · Steve
Dev 52.6%->76.3% but HELD-OUT (eval-blind2.json, Ollama-generated, never inspected) = 38.9%.
The dev/held-out gap empirically demonstrates the overfitting Cody warned of. Conclusion:
hand-tuned lexical routing is inadequate for top-1 NL queries; more tuning worsens held-out.
Stopping the tuning loop — decision escalates to top-3-shortlist vs embeddings.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M cabinet_directory.pyA eval-blind2.jsonM semantic.py
Diff
commit ac91d16802df9e1a3a2f28fe32f0d777c9920e7f
Author: Steve <steve@designerwallcoverings.com>
Date: Sat Aug 1 22:33:22 2026 -0700
scorer fix (field-weight + trigram-downweight + synonyms) + independent held-out eval
Dev 52.6%->76.3% but HELD-OUT (eval-blind2.json, Ollama-generated, never inspected) = 38.9%.
The dev/held-out gap empirically demonstrates the overfitting Cody warned of. Conclusion:
hand-tuned lexical routing is inadequate for top-1 NL queries; more tuning worsens held-out.
Stopping the tuning loop — decision escalates to top-3-shortlist vs embeddings.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
cabinet_directory.py | 13 ++-
eval-blind2.json | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++
semantic.py | 23 +++++-
3 files changed, 250 insertions(+), 4 deletions(-)
diff --git a/cabinet_directory.py b/cabinet_directory.py
index 4ee6fd6..20e010b 100644
--- a/cabinet_directory.py
+++ b/cabinet_directory.py
@@ -50,9 +50,18 @@ def _load_officers() -> list[dict]:
for f in sorted(CARDS_DIR.glob("*.agent-card.json")):
c = json.loads(f.read_text(encoding="utf-8"))
skills = c.get("skills", []) or []
+ name = c.get("name", "")
+ desc = c.get("description", "")
+ dom, trig = desc, ""
+ if "(triggers:" in desc:
+ dom, trig = desc.split("(triggers:", 1)
+ trig = trig.rstrip(") ")
+ skill_names = " ".join(s.get("name", "") for s in skills)
+ skill_descs = " ".join(s.get("description", "") for s in skills)
+ # FIELD-WEIGHTED: an officer's identity (name/domain) and triggers must dominate
+ # incidental vocabulary buried in one skill's description (repetition = higher TF).
blob = " ".join(
- [c.get("name", ""), c.get("description", "")]
- + [f"{s.get('name','')} {s.get('description','')}" for s in skills]
+ [name] * 3 + [dom] * 3 + [trig] * 2 + [skill_names] * 2 + [skill_descs]
).lower()
out.append(
{
diff --git a/eval-blind2.json b/eval-blind2.json
new file mode 100644
index 0000000..1e32722
--- /dev/null
+++ b/eval-blind2.json
@@ -0,0 +1,218 @@
+[
+ {
+ "q": "Can you set up monthly reports for my budget?",
+ "accept": [
+ "vp-abramsego"
+ ]
+ },
+ {
+ "q": "How do I request payment information for our latest project?",
+ "accept": [
+ "vp-abramsego"
+ ]
+ },
+ {
+ "q": "Could someone look into increasing our ad revenue this month?",
+ "accept": [
+ "vp-abramsego"
+ ]
+ },
+ {
+ "q": "Can you check on why my project is stuck and see if it can be moved forward?",
+ "accept": [
+ "vp-cncp"
+ ]
+ },
+ {
+ "q": "I have an idea for a new task, could you add it to the list of things that need approval?",
+ "accept": [
+ "vp-cncp"
+ ]
+ },
+ {
+ "q": "Is there any way to get some stalled tasks from other teams to start working on?",
+ "accept": [
+ "vp-cncp"
+ ]
+ },
+ {
+ "q": "Can you check if our outbound calls are following all the rules and laws?",
+ "accept": [
+ "vp-compliance-policy"
+ ]
+ },
+ {
+ "q": "Do we need to follow any special steps before making our calls?",
+ "accept": [
+ "vp-compliance-policy"
+ ]
+ },
+ {
+ "q": "Are there any legal things we should know about when making calls?",
+ "accept": [
+ "vp-compliance-policy"
+ ]
+ },
+ {
+ "q": "Can you create a new website for my business based on the information I provide?",
+ "accept": [
+ "vp-consulting"
+ ]
+ },
+ {
+ "q": "How do I set up social media tools to grow my client base?",
+ "accept": [
+ "vp-consulting"
+ ]
+ },
+ {
+ "q": "I filled out the intake questionnaire, when can I see the concept versions of my new site?",
+ "accept": [
+ "vp-consulting"
+ ]
+ },
+ {
+ "q": "I need to find a good lawyer in my area, can you help me with that?",
+ "accept": [
+ "vp-directories"
+ ]
+ },
+ {
+ "q": "Could you recommend a local vet for my pet?",
+ "accept": [
+ "vp-directories"
+ ]
+ },
+ {
+ "q": "Is there a nearby NPH center where I can drop off some donations?",
+ "accept": [
+ "vp-directories"
+ ]
+ },
+ {
+ "q": "Can you help me find some designer wallcoverings for my home project? I'm not sure where to start.",
+ "accept": [
+ "vp-dw-commerce"
+ ]
+ },
+ {
+ "q": "Is there a way to organize the catalog so it\u2019s easier for me to browse different patterns and styles?",
+ "accept": [
+ "vp-dw-commerce"
+ ]
+ },
+ {
+ "q": "Could you set up an automatic system to check for new products from our vendors every week?",
+ "accept": [
+ "vp-dw-commerce"
+ ]
+ },
+ {
+ "q": "Can you help me write some social media posts for our brand?",
+ "accept": [
+ "vp-dw-marketing"
+ ]
+ },
+ {
+ "q": "I need advice on how to make our ads more appealing. Can someone assist?",
+ "accept": [
+ "vp-dw-marketing"
+ ]
+ },
+ {
+ "q": "Could you check if our website is optimized for search engines?",
+ "accept": [
+ "vp-dw-marketing"
+ ]
+ },
+ {
+ "q": "Can someone check if my login button is working properly?",
+ "accept": [
+ "vp-engineering"
+ ]
+ },
+ {
+ "q": "I can't save changes I made to the website, can you help?",
+ "accept": [
+ "vp-engineering"
+ ]
+ },
+ {
+ "q": "Why are there so many slow pages when I visit our site?",
+ "accept": [
+ "vp-engineering"
+ ]
+ },
+ {
+ "q": "I need to update my email address for notifications, how do I do that?",
+ "accept": [
+ "vp-operations"
+ ]
+ },
+ {
+ "q": "Why am I getting so many error messages? Can you help fix them?",
+ "accept": [
+ "vp-operations"
+ ]
+ },
+ {
+ "q": "My website isn't loading properly, what should I check first?",
+ "accept": [
+ "vp-operations"
+ ]
+ },
+ {
+ "q": "Can someone help me find information about local parks for my family's weekend outing?",
+ "accept": [
+ "vp-research-content"
+ ]
+ },
+ {
+ "q": "I need to update my address with the landlord, can you assist me with that?",
+ "accept": [
+ "vp-research-content"
+ ]
+ },
+ {
+ "q": "Could you help me create a simple video tutorial for my cooking channel?",
+ "accept": [
+ "vp-research-content"
+ ]
+ },
+ {
+ "q": "Can you check if my passwords are strong enough?",
+ "accept": [
+ "vp-security"
+ ]
+ },
+ {
+ "q": "How do I change the password for my account?",
+ "accept": [
+ "vp-security"
+ ]
+ },
+ {
+ "q": "What should I do if I suspect a breach?",
+ "accept": [
+ "vp-security"
+ ]
+ },
+ {
+ "q": "Can you help me set up my new online store for selling wallpapers? I need step-by-step instructions.",
+ "accept": [
+ "vp-special-projects"
+ ]
+ },
+ {
+ "q": "How do I add products to my wallpaper website so customers can see them? Could you walk me through it?",
+ "accept": [
+ "vp-special-projects"
+ ]
+ },
+ {
+ "q": "I want to change the layout of my small business builder site. What are the easiest ways to do that?",
+ "accept": [
+ "vp-special-projects"
+ ]
+ }
+]
\ No newline at end of file
diff --git a/semantic.py b/semantic.py
index 139650d..bde16ba 100644
--- a/semantic.py
+++ b/semantic.py
@@ -41,7 +41,17 @@ _cluster("email", "mailer", "gmail", "george", "inbox", "purelymail")
_cluster("uptime", "monitor", "canary", "watchdog", "crashed", "down", "health")
_cluster("directory", "lawyer", "doctor", "animals", "listings")
_cluster("compliance", "canspam", "legal", "policy", "tcpa", "dnc")
-_cluster("copy", "marketing", "campaign", "brand", "content")
+_cluster("copy", "marketing", "campaign", "brand", "content", "promo", "promotional", "announce", "blast", "newsletter")
+# everyday-phrasing clusters (general domain knowledge; validated on a held-out set, not fitted)
+_cluster("server", "alive", "responding", "down", "offline", "restart", "crash", "crashed", "stalled", "process", "uptime", "monitor", "watchdog")
+_cluster("broke", "break", "broken", "hacked", "intrusion", "compromised", "stolen", "attacker", "security", "breach", "backdoor")
+_cluster("api", "endpoint", "backend", "rest", "service", "engineering", "code")
+_cluster("test", "tests", "suite", "failing", "broken", "ci", "testing", "bug", "debug")
+_cluster("property", "owns", "owner", "ownership", "records", "lookup", "research", "competitor", "background", "history")
+_cluster("text", "sms", "message", "messaging", "opt", "unsubscribe", "dnc", "legal", "allowed", "rules", "compliance", "canspam")
+_cluster("store", "storefront", "shop", "product", "products", "listing", "listings", "catalog", "sku", "item")
+_cluster("posting", "schedule", "channels", "post", "social", "instagram", "tiktok")
+_cluster("website", "site", "web", "address", "https", "ssl", "domain", "dns")
def _tokens(text: str) -> list[str]:
@@ -80,7 +90,16 @@ class Scorer:
self.doc_vecs = [self._weight(f) for f in self.doc_feats]
def _weight(self, feats: Counter) -> dict[str, float]:
- return {k: v * self.idf.get(k, math.log(len(self.doc_feats) + 1) + 1) for k, v in feats.items()}
+ default = math.log(len(self.doc_feats) + 1) + 1
+ # char-trigrams (g:) give morphology tolerance but add substring-coincidence noise
+ # at tiny corpus size, so down-weight them relative to whole-token (t:) matches.
+ out = {}
+ for k, v in feats.items():
+ w = v * self.idf.get(k, default)
+ if k.startswith("g:"):
+ w *= 0.3
+ out[k] = w
+ return out
def _expand(self, query: str) -> str:
toks = _tokens(query)
← c5493c8 add blind paraphrase eval harness (eval_find.py) — FALSIFIES
·
back to A2a Lab
·
frame find() as a ranked top-3 shortlist (routing hint, not 54459a9 →