← back to Rentv Ad Engine
scheduled auto-refresh: refresh.sh (deploy only on NEW episodes, extract-frames promo miner, atomic rollback, wobble-tolerant guard) + harvest preserves co-owned podcast manifest + launchd plist
28ca24ea33bcaca9b103847fea4a334d86ee7168 · 2026-08-07 15:40:18 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M scripts/harvest-transcripts.pyA scripts/refresh.sh
Diff
commit 28ca24ea33bcaca9b103847fea4a334d86ee7168
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Aug 7 15:40:18 2026 -0700
scheduled auto-refresh: refresh.sh (deploy only on NEW episodes, extract-frames promo miner, atomic rollback, wobble-tolerant guard) + harvest preserves co-owned podcast manifest + launchd plist
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/harvest-transcripts.py | 16 +++++++++--
scripts/refresh.sh | 63 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 77 insertions(+), 2 deletions(-)
diff --git a/scripts/harvest-transcripts.py b/scripts/harvest-transcripts.py
index 72679da..28cffe5 100644
--- a/scripts/harvest-transcripts.py
+++ b/scripts/harvest-transcripts.py
@@ -109,10 +109,22 @@ def main():
print(f" [{i}/{len(items)}] ok={ok} thin={thin} none={none}")
time.sleep(0.35) # polite
+ # PRESERVE non-Vimeo-library entries (e.g. SoundCloud podcast episodes upserted by
+ # transcribe-soundcloud.py) — manifest.json is co-owned; never clobber the other source.
+ lib_ids = {v["id"] for v in items}
+ try:
+ prev = json.load(open(MANIFEST)).get("videos", [])
+ extra = [v for v in prev if v.get("id") not in lib_ids]
+ if extra:
+ manifest += extra
+ print(f"[harvest] preserved {len(extra)} non-library (e.g. podcast) manifest entries")
+ except Exception:
+ pass
+
json.dump({"generated_at": time.strftime("%Y-%m-%d %H:%M:%S"),
- "total": len(items), "rich": ok, "thin": thin, "none": none,
+ "total": len(manifest), "rich": ok, "thin": thin, "none": none,
"videos": manifest}, open(MANIFEST, "w"), indent=1)
- print(f"[harvest] DONE rich={ok} thin={thin} none={none} skipped={skip} -> {MANIFEST}")
+ print(f"[harvest] DONE rich={ok} thin={thin} none={none} skipped={skip} total={len(manifest)} -> {MANIFEST}")
if __name__ == "__main__":
diff --git a/scripts/refresh.sh b/scripts/refresh.sh
new file mode 100755
index 0000000..bcfe0d5
--- /dev/null
+++ b/scripts/refresh.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Scheduled auto-refresh for the RENTV Ad Engine — so new CRE Talk episodes flow in
+# automatically. Full LOCAL pipeline ($0: Vimeo auto-captions + poster-OCR + Ollama + regex).
+#
+# Deploys ONLY when harvest finds genuinely NEW videos (new episodes drop into rentv's own
+# vimeo-library.json cron → this picks them up). On idle runs it no-ops — it never churns the
+# live data on qwen LLM wobble. Atomic: snapshots outputs, rolls back on any failure or a
+# significant shrink, so LOCAL and PROD can never be left regressed. Targeted rsync (no
+# --delete) so the prod-only raw podcast audio is never wiped. Logs to data/refresh.log.
+set -uo pipefail
+cd "$(dirname "$0")/.."
+export PATH="/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH"
+LOG="data/refresh.log"; exec 3>>"$LOG"
+say(){ echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee /dev/fd/3; }
+vids(){ python3 -c "import json;print(len(json.load(open('data/manifest.json'))['videos']))" 2>/dev/null || echo 0; }
+cnt(){ python3 -c "import json;d=json.load(open('$1'));print(len(d.get('$2',[])))" 2>/dev/null || echo 0; }
+REMOTE="root@45.61.58.125:/root/public-projects/rentv-ad-engine"
+
+say "── refresh start ──"
+BK=$(mktemp -d)
+cp data/advertisers.json data/guests.json data/adairings.json data/rentvreview.json \
+ data/frames.json data/synopses.json data/manifest.json "$BK/" 2>/dev/null || true
+restore(){ cp "$BK"/*.json data/ 2>/dev/null || true; }
+run(){ say "→ $1"; if ! "${@:2}" >>"$LOG" 2>&1; then say "✗ FAILED: $1 — rolling back"; restore; rm -rf "$BK"; exit 1; fi; }
+
+V0=$(vids)
+run "harvest transcripts (new Vimeo VTTs)" python3 scripts/harvest-transcripts.py
+V1=$(vids)
+say "videos: ${V0} → ${V1}"
+
+# DEPLOY ONLY ON NEW EPISODES — idle runs never touch prod (avoids LLM-wobble churn)
+if [ "$V1" -le "$V0" ]; then
+ say "no new videos — nothing to publish."; restore; rm -rf "$BK"; say "── refresh done (no-op) ──"; exit 0
+fi
+say "★ ${V1}-${V0}=$((V1-V0)) NEW video(s) — running full rebuild."
+
+ADV0=$(cnt data/advertisers.json advertisers); AIR0=$(cnt data/adairings.json airings)
+run "poster OCR (new videos)" python3 scripts/ocr-posters.py
+run "mine promo advertisers (poster OCR)" python3 scripts/extract-frames.py
+run "extract advertisers + guests" python3 scripts/extract.py
+run "clip-ads (podcast badges/adclips + merge)" python3 scripts/clip-ads.py
+run "harvest The REview (authoritative)" python3 scripts/harvest-rentvreview.py
+run "reconcile REview (dates + verify)" python3 scripts/reconcile-review.py
+run "build as-aired ad timeline" python3 scripts/build-adairings.py
+ADV1=$(cnt data/advertisers.json advertisers); AIR1=$(cnt data/adairings.json airings)
+say "counts: advertisers ${ADV0}→${ADV1} · airings ${AIR0}→${AIR1}"
+
+# GUARD: tolerate small LLM wobble (±3 adv / ±5 air) but roll back a real collapse.
+if [ "$ADV1" -lt "$((ADV0-3))" ] || [ "$AIR1" -lt "$((AIR0-5))" ] || [ "$ADV1" -lt 5 ]; then
+ say "✗ GUARD TRIPPED — rebuild collapsed (adv ${ADV0}→${ADV1}, air ${AIR0}→${AIR1}); rolling back, NOT deploying."
+ restore; rm -rf "$BK"; exit 2
+fi
+rm -rf "$BK"
+
+say "→ deploy regenerated data to prod (targeted, no --delete)"
+rsync -az --exclude 'soundcloud/audio' --exclude 'transcripts' data/ "$REMOTE/data/" >>"$LOG" 2>&1
+rsync -az public/img/posters/ "$REMOTE/public/img/posters/" >>"$LOG" 2>&1
+ssh -o ConnectTimeout=15 root@45.61.58.125 'pm2 restart rentv-ad-engine --update-env >/dev/null 2>&1' >>"$LOG" 2>&1
+CODE=$(curl -s -o /dev/null -w '%{http_code}' -u 'admin:DW2024!' https://rentv.agentabrams.com/advertisers/api/health)
+say "prod health after deploy: $CODE"
+git add data/ public/img/posters >/dev/null 2>&1
+git commit -q -m "auto-refresh: +$((V1-V0)) episode(s) — adv ${ADV0}→${ADV1}, airings ${AIR0}→${AIR1}" >/dev/null 2>&1 || true
+say "── refresh done (deployed +$((V1-V0)) episode(s)) ──"
← 8993674 add Analytics as an in-page tab in the directory (iframe of
·
back to Rentv Ad Engine
·
add refresh launchd plist 6782e69 →