← back to Local Model Leaderboard Watch
Refuse model loads when local weight shards are incomplete
6fe8096e4798869029f7e2914dc490dd238eafa8 · 2026-09-10 16:49:45 -0700 · Steve Abrams
Files touched
M review.shA test-model-preflight.pyA verification/e2e-proof.jsonA verify-model-weights.py
Diff
commit 6fe8096e4798869029f7e2914dc490dd238eafa8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 16:49:45 2026 -0700
Refuse model loads when local weight shards are incomplete
---
review.sh | 8 ++---
test-model-preflight.py | 72 +++++++++++++++++++++++++++++++++++++++++++++
verification/e2e-proof.json | 48 ++++++++++++++++++++++++++++++
verify-model-weights.py | 72 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 196 insertions(+), 4 deletions(-)
diff --git a/review.sh b/review.sh
index 65ade10..e2bd382 100755
--- a/review.sh
+++ b/review.sh
@@ -4,7 +4,7 @@
# The claude review then runs with NO shell/Bash — only WebSearch/WebFetch/Read/Write
# (research + report + gated recommendation). READ-ONLY: never downloads, never changes the cluster.
set -uo pipefail
-DIR="/Users/macstudio3/Projects/local-model-leaderboard-watch"
+DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
TS="$(date '+%FT%T%z')"
echo "[$TS] === review start ===" >> "$DIR/data/run.log"
@@ -30,12 +30,12 @@ timeout 600 claude -p "$(cat "$DIR/review-prompt.md")" \
rc=$?
# 3) DETERMINISTIC load of today's chosen model — ONLY if it's already downloaded.
-# Never downloads: if the model dir is absent, we leave the current model and rely
+# Never downloads: if model weights are missing or incomplete, we leave the current model and rely
# on the gated pending-approval memo. This is the wrapper's job, not the agent's.
CHOSEN="$(head -1 "$DIR/data/chosen-model.txt" 2>/dev/null | tr -d '[:space:]')"
if [ -n "$CHOSEN" ]; then
DNAME="$(printf '%s' "$CHOSEN" | sed 's#/#--#g')"
- if [ -d "$HOME/.exo/models/$DNAME" ]; then
+ if python3 "$DIR/verify-model-weights.py" "$HOME/.exo/models/$DNAME" >> "$DIR/data/run.log" 2>&1; then
LOADED="$(curl -s -m8 http://127.0.0.1:52415/state 2>/dev/null | python3 -c "import json,sys
try:
d=json.load(sys.stdin); i=d.get('instances',{})
@@ -50,7 +50,7 @@ except: print('')" 2>/dev/null)"
echo "[$TS] chosen $CHOSEN already loaded — no change" >> "$DIR/data/run.log"
fi
else
- echo "[$TS] chosen $CHOSEN NOT on disk — keeping current model; download is gated (see pending-approval)" >> "$DIR/data/run.log"
+ echo "[$TS] chosen $CHOSEN weights missing/incomplete — keeping current model; download is gated (see pending-approval)" >> "$DIR/data/run.log"
fi
fi
echo "[$TS] === review done (exit $rc) ===" >> "$DIR/data/run.log"
diff --git a/test-model-preflight.py b/test-model-preflight.py
new file mode 100644
index 0000000..40ccb48
--- /dev/null
+++ b/test-model-preflight.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Run the actual wrapper in a temporary filesystem with all external calls stubbed."""
+import json
+import os
+from pathlib import Path
+import shutil
+import struct
+import subprocess
+import tempfile
+
+SOURCE = Path(__file__).resolve().parent
+
+def shard(path):
+ header = json.dumps({'weight': {'dtype': 'F32', 'shape': [1], 'data_offsets': [0, 4]}}).encode()
+ path.write_bytes(struct.pack('<Q', len(header)) + header + b'\0'*4)
+
+
+def case(name, setup, expected_posts, chosen='org/model'):
+ with tempfile.TemporaryDirectory(prefix='tk11401-model-') as directory:
+ root = Path(directory)
+ app, home, bins = root/'app', root/'home', root/'bin'
+ app.mkdir(); home.mkdir(); bins.mkdir(); (app/'data').mkdir()
+ for source in ['review.sh', 'verify-model-weights.py']:
+ shutil.copy2(SOURCE/source, app/source)
+ (app/'review-prompt.md').write_text('offline fixture only')
+ (app/'data/chosen-model.txt').write_text(chosen+'\n')
+ model = home/'.exo/models/org--model'; model.mkdir(parents=True)
+ (model/'config.json').write_text('{"model_type":"fixture"}')
+ setup(model)
+ log = root/'requests.jsonl'
+ curl = bins/'curl'
+ curl.write_text('#!/usr/bin/env python3\nimport json,os,sys\nwith open(os.environ["REQUEST_LOG"],"a") as f: f.write(json.dumps(sys.argv[1:])+"\\n")\nprint("{\\"instances\\": {}}")\n')
+ curl.chmod(0o755)
+ timeout = bins/'timeout'; timeout.write_text('#!/bin/sh\nexit 0\n'); timeout.chmod(0o755)
+ env = dict(os.environ, HOME=str(home), PATH=str(bins)+':'+os.environ['PATH'], REQUEST_LOG=str(log))
+ result = subprocess.run(['bash', str(app/'review.sh')], env=env, capture_output=True, text=True, timeout=10)
+ calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
+ posts = [args for args in calls if any('/v1/chat/completions' in arg for arg in args)]
+ assert result.returncode == 0, (name, result.stderr)
+ assert len(posts) == expected_posts, (name, calls, (app/'data/run.log').read_text())
+ if not expected_posts and chosen:
+ assert 'REFUSED' in (app/'data/run.log').read_text(), name
+ return {'name': name, 'verdict': 'PASS', 'model_load_requests': len(posts), 'external_calls': 'all mocked'}
+
+
+def missing(root):
+ (root/'model.safetensors.index.json').write_text('{"weight_map":{"weight":"model-00001-of-00001.safetensors"}}')
+
+def complete(root):
+ missing(root); shard(root/'model-00001-of-00001.safetensors')
+
+def truncated(root):
+ complete(root); p=root/'model-00001-of-00001.safetensors'; p.write_bytes(p.read_bytes()[:-1])
+
+def wrong_tensor(root):
+ complete(root); (root/'model.safetensors.index.json').write_text('{"weight_map":{"absent":"model-00001-of-00001.safetensors"}}')
+
+def traversal(root):
+ (root/'model.safetensors.index.json').write_text('{"weight_map":{"weight":"../outside.safetensors"}}')
+
+results = [
+ case('config-only stub refuses model POST', lambda p: None, 0),
+ case('indexed missing shard refuses model POST', missing, 0),
+ case('truncated shard refuses model POST', truncated, 0),
+ case('malformed index refuses model POST', lambda p: (p/'model.safetensors.index.json').write_text('{'), 0),
+ case('missing mapped tensor refuses model POST', wrong_tensor, 0),
+ case('path traversal refuses model POST', traversal, 0),
+ case('complete indexed model permits mocked POST', complete, 1),
+ case('complete single shard permits mocked POST', lambda p: shard(p/'model.safetensors'), 1),
+ case('blank choice remains no-op', lambda p: None, 0, chosen=''),
+]
+print(json.dumps(results, indent=2))
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
new file mode 100644
index 0000000..f88601a
--- /dev/null
+++ b/verification/e2e-proof.json
@@ -0,0 +1,48 @@
+{
+ "intent": "Refuse model-load requests for configuration-only or incomplete on-disk model directories; isolated adoption candidate only",
+ "risk_tier": "R1",
+ "ticket": "TK-11401",
+ "correlation_id": "TK-11401/tail/M-02912",
+ "timestamp": "2026-09-10T23:49:39.363605+00:00",
+ "base_commit": "4c16796",
+ "environment": "Isolated git worktree. Actual review.sh copied to temporary fixture app; curl and timeout intercepted by offline executables. No active source or scheduler edited.",
+ "preconditions": [
+ "Real Qwen3.8 directory has config/index but zero shards",
+ "TK11325 records intentional user-authorized removal",
+ "No active related model ticket; prior owner notified M02912"
+ ],
+ "commands": [
+ "python3 test-model-preflight.py",
+ "bash -n review.sh",
+ "git diff --check",
+ "python3 verify-model-weights.py ~/.exo/models/mlx-community--Qwen3.8-27B-8bit"
+ ],
+ "assertions": [
+ {
+ "verdict": "PASS",
+ "boundary": "actual wrapper -> model POST",
+ "details": "9 scenarios: config stub, missing shard, truncated shard, malformed index, wrong tensor, traversal all refuse; complete indexed/single-shard allow mocked request; blank choice no-op"
+ },
+ {
+ "verdict": "PASS",
+ "boundary": "actual model filesystem",
+ "details": "Real configuration/index-only model refused missing shard with exit1; no write or model request"
+ },
+ {
+ "verdict": "PASS",
+ "boundary": "syntax/diff",
+ "details": "bash -n and git diff --check pass"
+ },
+ {
+ "verdict": "SKIP",
+ "boundary": "runtime adoption and live inference",
+ "details": "Explicitly excluded; requires owner acceptance / scheduled runtime change approval. Artifact completeness only, not a production-fix claim."
+ }
+ ],
+ "rollback": "Revert candidate commit or do not adopt candidate; no live change to undo",
+ "limitations": [
+ "Structural shard/header/index completeness does not authenticate publisher checksums or prove model engine compatibility",
+ "Preflight cannot prevent concurrent deletion after check; runtime errors remain possible"
+ ],
+ "verdict": "PASS isolated candidate; operational issue remains held until adoption"
+}
diff --git a/verify-model-weights.py b/verify-model-weights.py
new file mode 100644
index 0000000..d592770
--- /dev/null
+++ b/verify-model-weights.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Offline structural preflight: a config directory is not a downloaded model.
+
+Read safetensors headers, never model payloads. This is completeness validation,
+not publisher authentication or a guarantee that the inference engine supports it.
+"""
+import json
+from pathlib import Path
+import struct
+import sys
+
+
+def verify(root):
+ root = Path(root).resolve(strict=True)
+ config = json.loads((root / 'config.json').read_text())
+ if not isinstance(config, dict) or not config:
+ raise ValueError('missing model configuration')
+ index = root / 'model.safetensors.index.json'
+ if index.exists():
+ mapping = json.loads(index.read_text()).get('weight_map', {})
+ if not isinstance(mapping, dict) or not mapping:
+ raise ValueError('empty weight map')
+ else:
+ mapping = None
+ names = set(mapping.values()) if mapping else {'model.safetensors'}
+ tensors = {}
+ for name in names:
+ if not isinstance(name, str) or Path(name).name != name or not name.endswith('.safetensors'):
+ raise ValueError('invalid shard filename')
+ shard = root / name
+ if shard.resolve().parent != root or not shard.is_file():
+ raise ValueError('missing or outside-root shard: ' + name)
+ size = shard.stat().st_size
+ with shard.open('rb') as stream:
+ length = stream.read(8)
+ if len(length) != 8:
+ raise ValueError('truncated shard: ' + name)
+ header_size = struct.unpack('<Q', length)[0]
+ if header_size < 2 or header_size > 100_000_000 or header_size + 8 > size:
+ raise ValueError('invalid shard header: ' + name)
+ header = json.loads(stream.read(header_size))
+ payload_size = size - 8 - header_size
+ entries = {key: value for key, value in header.items() if key != '__metadata__'}
+ if not entries or payload_size <= 0:
+ raise ValueError('empty shard: ' + name)
+ ranges = []
+ for tensor, info in entries.items():
+ start, end = info['data_offsets']
+ if type(start) is not int or type(end) is not int or not 0 <= start <= end <= payload_size:
+ raise ValueError('invalid tensor offsets: ' + name)
+ ranges.append((start, end))
+ cursor = 0
+ for start, end in sorted(ranges):
+ if start != cursor:
+ raise ValueError('noncontiguous tensor payload: ' + name)
+ cursor = end
+ if cursor != payload_size:
+ raise ValueError('incomplete or extra tensor payload: ' + name)
+ tensors[name] = entries
+ if mapping:
+ for tensor, name in mapping.items():
+ if tensor not in tensors[name]:
+ raise ValueError('index tensor missing from shard: ' + tensor)
+ return len(names)
+
+
+if __name__ == '__main__':
+ try:
+ print('weight preflight PASS:', verify(sys.argv[1]), 'complete shard(s)')
+ except (OSError, ValueError, KeyError, TypeError, IndexError, struct.error) as error:
+ print('weight preflight REFUSED:', error)
+ sys.exit(1)
← 4c16796 auto-data-snapshot: 2026-09-03T04:44:57 (1 data files) — dat
·
back to Local Model Leaderboard Watch
·
Validate tensor shape byte lengths before allowing model loa 156aa46 →