← back to Allnewsdaily
allnewsdaily: PROXY_URL-aware + gzip + early-abort fetch in check-live.js (TK-11340)
046bf14108e0165e4c9c0d84132ab65ede0fb314 · 2026-09-09 23:46:52 -0700 · Steve Abrams
- Optional residential/ISP proxy egress via PROXY_URL (lazy https-proxy-agent dep;
no-op + no dep needed when unset, so Mac2 dev path is unchanged).
- Accept-Encoding gzip/br + local decompress: ~1.65MB -> ~0.38MB per fetch.
- Early-abort at live marker / 900KB cap: ~0.2MB/fetch worst case.
Verified locally (no proxy): 20 outlets, 13 live, 0 errors, 2.6s.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015acgmNP9spwqQLjfFcf1Qs
Files touched
M package.jsonM scripts/check-live.js
Diff
commit 046bf14108e0165e4c9c0d84132ab65ede0fb314
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 9 23:46:52 2026 -0700
allnewsdaily: PROXY_URL-aware + gzip + early-abort fetch in check-live.js (TK-11340)
- Optional residential/ISP proxy egress via PROXY_URL (lazy https-proxy-agent dep;
no-op + no dep needed when unset, so Mac2 dev path is unchanged).
- Accept-Encoding gzip/br + local decompress: ~1.65MB -> ~0.38MB per fetch.
- Early-abort at live marker / 900KB cap: ~0.2MB/fetch worst case.
Verified locally (no proxy): 20 outlets, 13 live, 0 errors, 2.6s.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015acgmNP9spwqQLjfFcf1Qs
---
package.json | 3 +-
scripts/check-live.js | 79 ++++++++++++++++++++++++++++++++++++++++-----------
2 files changed, 65 insertions(+), 17 deletions(-)
diff --git a/package.json b/package.json
index 6209578..4b5b86e 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,8 @@
"check-live": "node scripts/check-live.js"
},
"dependencies": {
- "express": "^4.21.1"
+ "express": "^4.21.1",
+ "https-proxy-agent": "^7.0.6"
},
"engines": {
"node": ">=18"
diff --git a/scripts/check-live.js b/scripts/check-live.js
index 8e72b1e..3482b16 100644
--- a/scripts/check-live.js
+++ b/scripts/check-live.js
@@ -10,6 +10,25 @@
const fs = require('fs');
const path = require('path');
const https = require('https');
+const zlib = require('zlib');
+
+// Optional egress through a residential/ISP proxy. Prod (Kamatera) is a datacenter IP that
+// YouTube walls; routing this fetch through a residential/ISP proxy gets prod the real page,
+// which lets us retire the Mac2 residential-IP push (TK-11340). When PROXY_URL is UNSET the
+// fetch behaves exactly as before (Mac2 dev path) — and the https-proxy-agent dep is required
+// LAZILY, so only the proxied host (prod) ever needs it installed.
+const PROXY_URL = process.env.PROXY_URL || '';
+let proxyAgent = null;
+if (PROXY_URL) {
+ const { HttpsProxyAgent } = require('https-proxy-agent');
+ proxyAgent = new HttpsProxyAgent(PROXY_URL);
+}
+
+// Live markers — presence means the channel is actively streaming. Used to detect liveness
+// AND to EARLY-ABORT: we stop the download the instant we've seen enough, transferring
+// ~0.2 MB instead of the full ~1.65 MB page (decisive on metered/proxy bandwidth).
+const LIVE_MARKER = /"hlsManifestUrl":"https:\/\/manifest\.googlevideo\.com|"isLive":true|"isLiveNow":true/;
+const MAX_DECODED = 900_000; // live markers appear by ~750 KB; also bounds not-live pages
const OUTLETS_PATH = path.join(__dirname, '..', 'data', 'outlets.json');
const STATUS_PATH = path.join(__dirname, '..', 'data', 'live-status.json');
@@ -18,15 +37,31 @@ const UA =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36';
-function fetchHtml(url, timeoutMs = 8000) {
+function fetchHtml(url, timeoutMs = 10000) {
return new Promise((resolve, reject) => {
- const req = https.get(
- url,
+ let done = false;
+ let req;
+ const settle = (fn, arg) => {
+ if (done) return;
+ done = true;
+ try { req && req.destroy(); } catch (_) {}
+ fn(arg);
+ };
+ const u = new URL(url);
+ req = https.request(
{
+ hostname: u.hostname,
+ port: u.port || 443,
+ path: u.pathname + u.search,
+ method: 'GET',
+ agent: proxyAgent || undefined,
headers: {
'User-Agent': UA,
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml',
+ // Ask for compression — cuts the transferred page ~4.4x (1.65 MB -> ~0.38 MB),
+ // then we decompress locally. Bandwidth billed by a proxy = the compressed bytes.
+ 'Accept-Encoding': 'gzip, deflate, br',
// Bypass YouTube's consent/cookie interstitial (served to datacenter IPs like prod,
// which otherwise returns a consent page with no live markers → false "not live").
Cookie: 'CONSENT=YES+1; SOCS=CAI',
@@ -34,28 +69,40 @@ function fetchHtml(url, timeoutMs = 8000) {
},
res => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
- // Follow one hop
- fetchHtml(new URL(res.headers.location, url).toString(), timeoutMs).then(resolve, reject);
+ // Follow one hop (reuses the proxy agent via module scope).
res.resume();
+ if (!done) {
+ done = true;
+ fetchHtml(new URL(res.headers.location, url).toString(), timeoutMs).then(resolve, reject);
+ }
return;
}
if (res.statusCode !== 200) {
res.resume();
- return reject(new Error('status ' + res.statusCode));
+ return settle(reject, new Error('status ' + res.statusCode));
}
- let body = '';
- res.setEncoding('utf8');
- res.on('data', d => {
- body += d;
- if (body.length > 4_000_000) {
- req.destroy(new Error('body too big'));
- }
+ const enc = (res.headers['content-encoding'] || '').toLowerCase();
+ const stream =
+ enc === 'gzip' ? res.pipe(zlib.createGunzip())
+ : enc === 'br' ? res.pipe(zlib.createBrotliDecompress())
+ : enc === 'deflate' ? res.pipe(zlib.createInflate())
+ : res;
+ let text = '';
+ stream.on('data', d => {
+ text += d.toString('utf8');
+ // Early-abort: we have what we need (live markers), or the page is long enough
+ // to conclude it's NOT live — either way stop transferring.
+ if (LIVE_MARKER.test(text) || text.length > MAX_DECODED) settle(resolve, text);
});
- res.on('end', () => resolve(body));
+ stream.on('end', () => settle(resolve, text));
+ stream.on('error', e => settle(reject, e));
+ res.on('error', e => settle(reject, e));
}
);
- req.setTimeout(timeoutMs, () => req.destroy(new Error('timeout')));
- req.on('error', reject);
+ req.setTimeout(timeoutMs, () => settle(reject, new Error('timeout')));
+ // Swallow the post-abort 'aborted'/'destroy' error we cause on early-abort; surface real ones.
+ req.on('error', e => { if (!done) settle(reject, e); });
+ req.end();
});
}
← 47a0b36 allnewsdaily: add self-verifying deploy-outlets.sh (rsync ou
·
back to Allnewsdaily
·
front page: Drudge-style 3-column article layout 85e4126 →