← back to Wallco Ai
elements: add prod-flavored extract.py with path rewriter + incremental persist
0be62b7b4482dc7e5b11dc12039156a905d06ea4 · 2026-05-29 15:21:12 -0700 · steve
Files touched
A scripts/elements_extract_prod.py
Diff
commit 0be62b7b4482dc7e5b11dc12039156a905d06ea4
Author: steve <steve@designerwallcoverings.com>
Date: Fri May 29 15:21:12 2026 -0700
elements: add prod-flavored extract.py with path rewriter + incremental persist
---
scripts/elements_extract_prod.py | 309 +++++++++++++++++++++++++++++++++++++++
1 file changed, 309 insertions(+)
diff --git a/scripts/elements_extract_prod.py b/scripts/elements_extract_prod.py
new file mode 100644
index 0000000..31783c0
--- /dev/null
+++ b/scripts/elements_extract_prod.py
@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+"""elements/extract.py — PROD variant.
+
+Differences from the Mac2 canonical at ~/.claude/skills/elements/scripts/extract.py:
+ - WALLCO_ROOT = /root/public-projects/wallco-ai (prod path layout)
+ - Path rewriter: many prod PG rows have local_path pointing at Mac2's
+ /Users/stevestudio2/Projects/wallco-ai/... — rewrite to the prod-native
+ /root/public-projects/wallco-ai/... so the bake can read the source PNG.
+ - psql invoked via `sudo -u postgres` since the bake script runs as root and
+ pg_hba peer-auth is by OS user.
+
+Otherwise identical pipeline. Read-only against source PNGs; idempotent;
+never touches is_published.
+"""
+import argparse
+import json
+import multiprocessing as mp
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+WALLCO_ROOT = Path('/root/public-projects/wallco-ai')
+OUT_DIR = WALLCO_ROOT / 'data' / 'elements'
+TARGET_PX = 512
+CROP_FRAC = 0.75
+PALETTE_K = 6
+
+# Path rewriter — prod PG carries Mac2 paths; rewrite to prod-native.
+MAC2_PREFIX = '/Users/stevestudio2/Projects/wallco-ai/'
+PROD_PREFIX = '/root/public-projects/wallco-ai/'
+
+
+def rewrite_path(p):
+ if p and p.startswith(MAC2_PREFIX):
+ return PROD_PREFIX + p[len(MAC2_PREFIX):]
+ return p
+
+
+def psql(sql):
+ """Run psql as postgres user; return stdout."""
+ r = subprocess.run(
+ ['sudo', '-u', 'postgres', 'psql', 'dw_unified', '-At', '-F', '\t', '-c', sql],
+ check=True, capture_output=True, text=True,
+ )
+ return r.stdout
+
+
+def psql_file(sql_path):
+ r = subprocess.run(
+ ['sudo', '-u', 'postgres', 'psql', 'dw_unified', '-At', '-q',
+ '-v', 'ON_ERROR_STOP=1', '-f', sql_path],
+ capture_output=True, text=True,
+ )
+ return r
+
+
+def fetch_qualifying(limit=None, ids=None, sample_per_category=None, force=False):
+ where = [
+ 'is_published = TRUE',
+ '(user_removed IS NULL OR user_removed = FALSE)',
+ 'local_path IS NOT NULL',
+ "(notes IS NULL OR (notes NOT LIKE '%EDGES_FAIL:%'"
+ " AND notes NOT LIKE '%EDGES_SCAN_ERROR:%'"
+ " AND notes NOT LIKE '%BLEED_GHOST_REGEN%'"
+ " AND notes NOT LIKE '%SETTLEMENT BLOCK%'"
+ " AND notes NOT LIKE '%BLEED_GUARD:%'))",
+ ]
+ if ids:
+ where = [f"id IN ({','.join(str(int(i)) for i in ids)})"]
+ if not force:
+ where.append("(element_motif_path IS NULL OR element_ref_path IS NULL)")
+
+ if sample_per_category:
+ sql = f"""
+ WITH q AS (
+ SELECT id, local_path, category,
+ ROW_NUMBER() OVER (PARTITION BY category ORDER BY id) AS rn
+ FROM all_designs
+ WHERE {' AND '.join(where)}
+ )
+ SELECT id, local_path FROM q WHERE rn <= {int(sample_per_category)}
+ ORDER BY category, id;
+ """
+ else:
+ sql = (
+ "SELECT id, local_path FROM all_designs WHERE "
+ + ' AND '.join(where)
+ + " ORDER BY id DESC"
+ )
+ if limit:
+ sql += f' LIMIT {int(limit)}'
+ sql += ';'
+
+ out = []
+ for line in psql(sql).splitlines():
+ if not line.strip():
+ continue
+ parts = line.split('\t', 1)
+ if len(parts) == 2:
+ try:
+ out.append((int(parts[0]), rewrite_path(parts[1])))
+ except ValueError:
+ pass
+ return out
+
+
+def detect_bg_color(arr):
+ import numpy as np
+ H, W = arr.shape[:2]
+ pad = max(8, min(H, W) // 16)
+ corners = np.concatenate([
+ arr[:pad, :pad, :].reshape(-1, 3),
+ arr[:pad, -pad:, :].reshape(-1, 3),
+ arr[-pad:, :pad, :].reshape(-1, 3),
+ arr[-pad:, -pad:, :].reshape(-1, 3),
+ ])
+ bucketed = (corners // 32 * 32).astype(np.uint8)
+ flat = bucketed.view(np.dtype((np.void, 3))).ravel()
+ _, counts = np.unique(flat, return_counts=True)
+ top_bucket = np.unique(flat)[np.argmax(counts)]
+ bgr = np.frombuffer(top_bucket.tobytes(), dtype=np.uint8)
+ return tuple(int(x) for x in bgr), float(counts.max()) / len(bucketed)
+
+
+def bake_one(job):
+ design_id, local_path = job
+ try:
+ from PIL import Image
+ import numpy as np
+ except ImportError:
+ return {'id': design_id, 'ok': False, 'error': 'PIL or numpy missing'}
+
+ src = Path(local_path)
+ if not src.exists():
+ return {'id': design_id, 'ok': False, 'error': f'file_missing: {local_path}'}
+
+ motif_path = OUT_DIR / f'{design_id}_motif.png'
+ ref_path = OUT_DIR / f'{design_id}_ref512.png'
+
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
+ try:
+ img = Image.open(src).convert('RGB')
+ except Exception as e:
+ return {'id': design_id, 'ok': False, 'error': f'image_decode_fail: {e}'}
+ W, H = img.size
+ if min(W, H) < 256:
+ return {'id': design_id, 'ok': False, 'error': f'image_too_small {W}x{H}'}
+
+ crop_side = int(min(W, H) * CROP_FRAC)
+ cx, cy = W // 2, H // 2
+ half = crop_side // 2
+ cropped = img.crop((cx - half, cy - half, cx + half, cy + half))
+ arr = np.asarray(cropped)
+
+ bg_color, bg_dominance = detect_bg_color(arr)
+
+ diff = arr.astype(np.int16) - np.array(bg_color, dtype=np.int16)
+ dist = np.abs(diff).max(axis=-1)
+ bg_mask = dist <= 28
+
+ dense_pattern = bg_dominance < 0.25
+
+ if dense_pattern:
+ rgba = np.dstack([arr, np.full(arr.shape[:2], 255, dtype=np.uint8)])
+ else:
+ alpha = np.where(bg_mask, 0, 255).astype(np.uint8)
+ rgba = np.dstack([arr, alpha])
+
+ motif_img = Image.fromarray(rgba, mode='RGBA').resize(
+ (TARGET_PX, TARGET_PX), Image.LANCZOS
+ )
+
+ white_bg = Image.new('RGB', motif_img.size, (255, 255, 255))
+ white_bg.paste(motif_img, mask=motif_img.split()[3])
+ ref_img = white_bg
+
+ motif_img.save(motif_path, 'PNG', optimize=True)
+ ref_img.save(ref_path, 'PNG', optimize=True)
+
+ return {
+ 'id': design_id,
+ 'ok': True,
+ 'motif_path': str(motif_path),
+ 'ref_path': str(ref_path),
+ 'bg_color': '#{:02x}{:02x}{:02x}'.format(*bg_color),
+ 'bg_dominance': round(bg_dominance, 3),
+ 'dense_pattern': dense_pattern,
+ 'motif_bytes': motif_path.stat().st_size,
+ 'ref_bytes': ref_path.stat().st_size,
+ }
+
+
+def persist_paths(results):
+ successes = [r for r in results if r.get('ok')]
+ if not successes:
+ return 0
+
+ sql_parts = [
+ 'BEGIN;',
+ 'CREATE TEMP TABLE _elem_apply (id BIGINT PRIMARY KEY, motif TEXT NOT NULL, ref TEXT NOT NULL);',
+ ]
+ CHUNK = 400
+ for i in range(0, len(successes), CHUNK):
+ batch = successes[i:i + CHUNK]
+ values = ','.join(
+ f"({r['id']}, '{r['motif_path']}', '{r['ref_path']}')"
+ for r in batch
+ )
+ sql_parts.append(f"INSERT INTO _elem_apply (id, motif, ref) VALUES {values};")
+ sql_parts.append(
+ "UPDATE all_designs ad "
+ "SET element_motif_path = ea.motif, element_ref_path = ea.ref "
+ "FROM _elem_apply ea WHERE ad.id = ea.id;"
+ )
+ sql_parts.append('SELECT count(*) FROM _elem_apply;')
+ sql_parts.append('COMMIT;')
+
+ import tempfile
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.sql', delete=False) as tf:
+ tf.write('\n'.join(sql_parts))
+ sql_path = tf.name
+ # postgres user needs to read this; we wrote it as root
+ os.chmod(sql_path, 0o644)
+
+ r = psql_file(sql_path)
+ if r.returncode != 0:
+ print(f'psql update failed: {r.stderr.strip()}', file=sys.stderr)
+ return 0
+ return len(successes)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--id', type=int, action='append')
+ ap.add_argument('--all', action='store_true')
+ ap.add_argument('--sample-per-category', type=int, default=None)
+ ap.add_argument('--limit', type=int, default=None)
+ ap.add_argument('--workers', type=int, default=8)
+ ap.add_argument('--force', action='store_true')
+ args = ap.parse_args()
+
+ if not (args.id or args.all or args.sample_per_category or args.limit):
+ ap.error('Pass --id, --all, --sample-per-category, or --limit')
+
+ targets = fetch_qualifying(
+ limit=args.limit,
+ ids=args.id,
+ sample_per_category=args.sample_per_category,
+ force=args.force,
+ )
+ print(f'targets: {len(targets)}', flush=True)
+ if not targets:
+ print('nothing to do', flush=True)
+ return
+
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
+ t0 = time.time()
+ counts = {'ok': 0, 'err': 0, 'dense': 0, 'missing': 0}
+ results: list[dict] = []
+
+ pending_persist: list[dict] = []
+ PERSIST_EVERY = 200
+ total_persisted = 0
+ with mp.Pool(args.workers) as pool:
+ for i, res in enumerate(pool.imap_unordered(bake_one, targets, chunksize=4), 1):
+ results.append(res)
+ if res.get('ok'):
+ counts['ok'] += 1
+ pending_persist.append(res)
+ if res.get('dense_pattern'):
+ counts['dense'] += 1
+ else:
+ counts['err'] += 1
+ if 'file_missing' in str(res.get('error', '')):
+ counts['missing'] += 1
+ # Persist in batches so a late crash doesn't lose all the work
+ if len(pending_persist) >= PERSIST_EVERY:
+ total_persisted += persist_paths(pending_persist)
+ pending_persist = []
+ if i % 100 == 0 or i == len(targets):
+ rate = i / max(0.001, time.time() - t0)
+ eta = (len(targets) - i) / max(0.001, rate)
+ print(
+ f' [{i:>5}/{len(targets)}] ok={counts["ok"]} err={counts["err"]} '
+ f'missing={counts["missing"]} dense={counts["dense"]} '
+ f'rate={rate:.1f}/s eta={eta:.0f}s pg_persisted={total_persisted}',
+ flush=True,
+ )
+
+ if pending_persist:
+ total_persisted += persist_paths(pending_persist)
+ written = total_persisted
+ elapsed = time.time() - t0
+ print('', flush=True)
+ print(f'extract done in {elapsed:.1f}s — baked={counts["ok"]} err={counts["err"]} '
+ f'missing={counts["missing"]} dense_passthrough={counts["dense"]} '
+ f'pg_updated={written}', flush=True)
+
+ sample = [r for r in results if r.get('ok')][:5]
+ for r in sample:
+ print(f" id={r['id']:>6} bg={r['bg_color']} dom={r['bg_dominance']} "
+ f"motif={r['motif_bytes']//1024}KB ref={r['ref_bytes']//1024}KB"
+ f"{' [dense passthrough]' if r['dense_pattern'] else ''}", flush=True)
+
+
+if __name__ == '__main__':
+ main()
← c12ba4f publish-gate: raise execSync 90s→240s and urllib 120s→210s f
·
back to Wallco Ai
·
admin/rooms list: filter out soft-deleted rooms (AND removed c42d61a →