← back to Local Model Leaderboard Watch
Validate tensor shape byte lengths before allowing model load
156aa467d29c5f65a77973bde94eeb287940e244 · 2026-09-10 17:41:34 -0700 · Steve Abrams
Files touched
M review.shM test-model-preflight.pyM verification/e2e-proof.jsonM verify-model-weights.py
Diff
commit 156aa467d29c5f65a77973bde94eeb287940e244
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 17:41:34 2026 -0700
Validate tensor shape byte lengths before allowing model load
---
review.sh | 5 +++--
test-model-preflight.py | 34 +++++++++++++++++++++++++++++-----
verification/e2e-proof.json | 15 +++++++++------
verify-model-weights.py | 43 +++++++++++++++++++++++++++++++++++++++----
4 files changed, 80 insertions(+), 17 deletions(-)
diff --git a/review.sh b/review.sh
index e2bd382..02f9d31 100755
--- a/review.sh
+++ b/review.sh
@@ -5,6 +5,7 @@
# (research + report + gated recommendation). READ-ONLY: never downloads, never changes the cluster.
set -uo pipefail
DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+EXO_MODELS_DIR="${EXO_MODELS_DIR:-$HOME/.exo/models}"
TS="$(date '+%FT%T%z')"
echo "[$TS] === review start ===" >> "$DIR/data/run.log"
@@ -19,7 +20,7 @@ try:
print(list(inst.values())[0]['MlxRingInstance']['shardAssignments']['modelId'] if inst else 'none loaded')
except Exception as e: print('state-unreachable')" 2>/dev/null
echo "## Downloaded models (~/.exo/models):"
- ls -la ~/.exo/models 2>/dev/null | awk '{print $5, $NF}' | grep -viE '^\s*$|caches|\.$'
+ ls -la "$EXO_MODELS_DIR" 2>/dev/null | awk '{print $5, $NF}' | grep -viE '^\s*$|caches|\.$'
echo "## Hardware: 96GB M3 Ultra (primary) + 2x 32GB nodes; target = fits ~55GB weights single-node."
} > "$DIR/data/local-state.txt" 2>&1
@@ -35,7 +36,7 @@ rc=$?
CHOSEN="$(head -1 "$DIR/data/chosen-model.txt" 2>/dev/null | tr -d '[:space:]')"
if [ -n "$CHOSEN" ]; then
DNAME="$(printf '%s' "$CHOSEN" | sed 's#/#--#g')"
- if python3 "$DIR/verify-model-weights.py" "$HOME/.exo/models/$DNAME" >> "$DIR/data/run.log" 2>&1; then
+ if python3 "$DIR/verify-model-weights.py" "$EXO_MODELS_DIR/$DNAME" >> "$DIR/data/run.log" 2>&1; then
LOADED="$(curl -s -m8 http://127.0.0.1:52415/state 2>/dev/null | python3 -c "import json,sys
try:
d=json.load(sys.stdin); i=d.get('instances',{})
diff --git a/test-model-preflight.py b/test-model-preflight.py
index 40ccb48..cd65e70 100644
--- a/test-model-preflight.py
+++ b/test-model-preflight.py
@@ -10,9 +10,11 @@ import tempfile
SOURCE = Path(__file__).resolve().parent
-def shard(path):
- header = json.dumps({'weight': {'dtype': 'F32', 'shape': [1], 'data_offsets': [0, 4]}}).encode()
- path.write_bytes(struct.pack('<Q', len(header)) + header + b'\0'*4)
+def shard(path, dtype='F32', shape=None, size=4, header=None):
+ if header is None:
+ header = {'weight': {'dtype': dtype, 'shape': [1] if shape is None else shape, 'data_offsets': [0, size]}}
+ encoded = json.dumps(header).encode()
+ path.write_bytes(struct.pack('<Q', len(encoded)) + encoded + b'\0'*size)
def case(name, setup, expected_posts, chosen='org/model'):
@@ -32,14 +34,16 @@ def case(name, setup, expected_posts, chosen='org/model'):
curl.write_text('#!/usr/bin/env python3\nimport json,os,sys\nwith open(os.environ["REQUEST_LOG"],"a") as f: f.write(json.dumps(sys.argv[1:])+"\\n")\nprint("{\\"instances\\": {}}")\n')
curl.chmod(0o755)
timeout = bins/'timeout'; timeout.write_text('#!/bin/sh\nexit 0\n'); timeout.chmod(0o755)
- env = dict(os.environ, HOME=str(home), PATH=str(bins)+':'+os.environ['PATH'], REQUEST_LOG=str(log))
+ env = dict(os.environ, EXO_MODELS_DIR=str(model.parent), PATH=str(bins)+':'+os.environ['PATH'], REQUEST_LOG=str(log))
result = subprocess.run(['bash', str(app/'review.sh')], env=env, capture_output=True, text=True, timeout=10)
calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
posts = [args for args in calls if any('/v1/chat/completions' in arg for arg in args)]
assert result.returncode == 0, (name, result.stderr)
assert len(posts) == expected_posts, (name, calls, (app/'data/run.log').read_text())
+ run_log = (app/'data/run.log').read_text()
+ assert 'Traceback' not in run_log, (name, run_log)
if not expected_posts and chosen:
- assert 'REFUSED' in (app/'data/run.log').read_text(), name
+ assert 'REFUSED' in run_log, name
return {'name': name, 'verdict': 'PASS', 'model_load_requests': len(posts), 'external_calls': 'all mocked'}
@@ -67,6 +71,26 @@ results = [
case('path traversal refuses model POST', traversal, 0),
case('complete indexed model permits mocked POST', complete, 1),
case('complete single shard permits mocked POST', lambda p: shard(p/'model.safetensors'), 1),
+ case('shape payload mismatch refuses model POST', lambda p: shard(p/'model.safetensors', shape=[1000]), 0),
+ case('unknown dtype refuses model POST', lambda p: shard(p/'model.safetensors', dtype='NEW_UNKNOWN'), 0),
+ case('negative dimension refuses model POST', lambda p: shard(p/'model.safetensors', shape=[-1]), 0),
+ case('boolean dimension refuses model POST', lambda p: shard(p/'model.safetensors', shape=[True]), 0),
+ case('nonobject index refuses without traceback', lambda p: (p/'model.safetensors.index.json').write_text('[]'), 0),
+ case('nonobject header refuses without traceback', lambda p: shard(p/'model.safetensors', header=[]), 0),
+ case('nonobject tensor refuses without traceback', lambda p: shard(p/'model.safetensors', header={'weight': None}), 0),
+ case('unhashable index value refuses without traceback', lambda p: (p/'model.safetensors.index.json').write_text('{"weight_map":{"weight":[]}}'), 0),
+ case('zero-size tensor permits mocked POST', lambda p: shard(p/'model.safetensors', shape=[0,1000], size=0), 1),
+ case('scalar tensor permits mocked POST', lambda p: shard(p/'model.safetensors', shape=[]), 1),
case('blank choice remains no-op', lambda p: None, 0, chosen=''),
]
+# Exercise each supported dtype against the real wrapper, including packed widths.
+for dtype, count, size in [
+ ('BOOL', 1, 1), ('I8', 1, 1), ('U8', 1, 1), ('I16', 1, 2), ('U16', 1, 2),
+ ('F16', 1, 2), ('BF16', 1, 2), ('I32', 1, 4), ('U32', 1, 4), ('F32', 1, 4),
+ ('I64', 1, 8), ('U64', 1, 8), ('F64', 1, 8), ('C64', 1, 8),
+ ('F8_E4M3', 1, 1), ('F8_E5M2', 1, 1), ('F8_E8M0', 1, 1),
+ ('F8_E4M3FNUZ', 1, 1), ('F8_E5M2FNUZ', 1, 1),
+ ('F4', 2, 1), ('F6_E2M3', 4, 3), ('F6_E3M2', 4, 3),
+]:
+ results.append(case('supported dtype '+dtype, lambda p, d=dtype, c=count, s=size: shard(p/'model.safetensors', dtype=d, shape=[c], size=s), 1))
print(json.dumps(results, indent=2))
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
index f88601a..cec2537 100644
--- a/verification/e2e-proof.json
+++ b/verification/e2e-proof.json
@@ -3,9 +3,9 @@
"risk_tier": "R1",
"ticket": "TK-11401",
"correlation_id": "TK-11401/tail/M-02912",
- "timestamp": "2026-09-10T23:49:39.363605+00:00",
+ "timestamp": "2026-09-11T00:41:34.548708+00:00",
"base_commit": "4c16796",
- "environment": "Isolated git worktree. Actual review.sh copied to temporary fixture app; curl and timeout intercepted by offline executables. No active source or scheduler edited.",
+ "environment": "Isolated git worktree. Actual review.sh copied to temporary fixture app; curl and timeout intercepted by offline executables. EXO_MODELS_DIR points to fixture directory; HOME is unchanged. No active source or scheduler edited.",
"preconditions": [
"Real Qwen3.8 directory has config/index but zero shards",
"TK11325 records intentional user-authorized removal",
@@ -21,7 +21,7 @@
{
"verdict": "PASS",
"boundary": "actual wrapper -> model POST",
- "details": "9 scenarios: config stub, missing shard, truncated shard, malformed index, wrong tensor, traversal all refuse; complete indexed/single-shard allow mocked request; blank choice no-op"
+ "details": "41 actual-wrapper offline scenarios PASS: malformed/missing/truncated shards, wrong tensor map, shape/dtype payload mismatch, unknown dtype, negative/bool dimensions, invalid object schemas and traversal refuse with zero model POST and no traceback; valid scalar/zero-size tensors and22 official dtype widths pass mocked POST; blank no-op."
},
{
"verdict": "PASS",
@@ -41,8 +41,11 @@
],
"rollback": "Revert candidate commit or do not adopt candidate; no live change to undo",
"limitations": [
- "Structural shard/header/index completeness does not authenticate publisher checksums or prove model engine compatibility",
- "Preflight cannot prevent concurrent deletion after check; runtime errors remain possible"
+ "Validates local tensor layout, dtype/shape byte lengths and index references only; not publisher authentication, full safetensors conformance or inference-engine compatibility.",
+ "Unknown dtype names fail closed and require explicit support review.",
+ "Preflight cannot prevent concurrent deletion after check; runtime errors remain possible."
],
- "verdict": "PASS isolated candidate; operational issue remains held until adoption"
+ "verdict": "PASS isolated candidate; operational issue remains held until adoption",
+ "review_fix": "Acceptance reviewer proved F32 shape[1000] with4-byte offsets/payload incorrectly passed6afbb80. Followup rejects mismatch, validates malformed object schemas without traceback, supports zero-size tensors, replaces HOME override with scoped EXO_MODELS_DIR.",
+ "reference": "https://github.com/safetensors/safetensors/blob/main/safetensors/src/tensor.rs (Dtype::bitsize verified September10,2026)"
}
diff --git a/verify-model-weights.py b/verify-model-weights.py
index d592770..ea6e34c 100644
--- a/verify-model-weights.py
+++ b/verify-model-weights.py
@@ -5,11 +5,24 @@ Read safetensors headers, never model payloads. This is completeness validation,
not publisher authentication or a guarantee that the inference engine supports it.
"""
import json
+import math
from pathlib import Path
import struct
import sys
+# Official safetensors Dtype::bitsize; unknown future types fail closed.
+# https://github.com/safetensors/safetensors/blob/main/safetensors/src/tensor.rs
+DTYPE_BITS = {
+ 'BOOL': 8, 'U8': 8, 'I8': 8, 'F8_E5M2': 8, 'F8_E4M3': 8,
+ 'F8_E8M0': 8, 'F8_E4M3FNUZ': 8, 'F8_E5M2FNUZ': 8,
+ 'I16': 16, 'U16': 16, 'F16': 16, 'BF16': 16,
+ 'I32': 32, 'U32': 32, 'F32': 32,
+ 'I64': 64, 'U64': 64, 'F64': 64, 'C64': 64,
+ 'F4': 4, 'F6_E2M3': 6, 'F6_E3M2': 6,
+}
+
+
def verify(root):
root = Path(root).resolve(strict=True)
config = json.loads((root / 'config.json').read_text())
@@ -17,12 +30,18 @@ def verify(root):
raise ValueError('missing model configuration')
index = root / 'model.safetensors.index.json'
if index.exists():
- mapping = json.loads(index.read_text()).get('weight_map', {})
+ index_data = json.loads(index.read_text())
+ if not isinstance(index_data, dict):
+ raise ValueError('index must be a JSON object')
+ mapping = index_data.get('weight_map', {})
if not isinstance(mapping, dict) or not mapping:
raise ValueError('empty weight map')
else:
mapping = None
- names = set(mapping.values()) if mapping else {'model.safetensors'}
+ names = list(mapping.values()) if mapping else ['model.safetensors']
+ if any(not isinstance(name, str) for name in names):
+ raise ValueError('shard filenames must be strings')
+ names = set(names)
tensors = {}
for name in names:
if not isinstance(name, str) or Path(name).name != name or not name.endswith('.safetensors'):
@@ -39,15 +58,31 @@ def verify(root):
if header_size < 2 or header_size > 100_000_000 or header_size + 8 > size:
raise ValueError('invalid shard header: ' + name)
header = json.loads(stream.read(header_size))
+ if not isinstance(header, dict):
+ raise ValueError('shard header must be a JSON object: ' + name)
payload_size = size - 8 - header_size
entries = {key: value for key, value in header.items() if key != '__metadata__'}
- if not entries or payload_size <= 0:
+ if not entries:
raise ValueError('empty shard: ' + name)
ranges = []
for tensor, info in entries.items():
- start, end = info['data_offsets']
+ if not isinstance(info, dict):
+ raise ValueError('tensor metadata must be a JSON object: ' + tensor)
+ dtype = info.get('dtype')
+ if not isinstance(dtype, str) or dtype not in DTYPE_BITS:
+ raise ValueError('unsupported tensor dtype: ' + str(dtype))
+ shape = info.get('shape')
+ if not isinstance(shape, list) or any(type(d) is not int or d < 0 for d in shape):
+ raise ValueError('tensor shape must contain nonnegative integers: ' + tensor)
+ offsets = info.get('data_offsets')
+ if not isinstance(offsets, list) or len(offsets) != 2:
+ raise ValueError('tensor offsets must contain two integers: ' + tensor)
+ start, end = offsets
if type(start) is not int or type(end) is not int or not 0 <= start <= end <= payload_size:
raise ValueError('invalid tensor offsets: ' + name)
+ expected_bits = math.prod(shape) * DTYPE_BITS[dtype]
+ if expected_bits % 8 or expected_bits // 8 != end - start:
+ raise ValueError('tensor shape/dtype byte-length mismatch: ' + tensor)
ranges.append((start, end))
cursor = 0
for start, end in sorted(ranges):
← 6fe8096 Refuse model loads when local weight shards are incomplete
·
back to Local Model Leaderboard Watch
·
Record independent verification of adopted model guard c7d4b71 →