← back to Allnewsdaily
scripts/check-live.js
207 lines
#!/usr/bin/env node
// Polls each outlet's YouTube /channel/<id>/live URL.
// YouTube returns the channel's currently-live broadcast page (200 + watch-page HTML)
// or a "no live stream" placeholder. We detect liveness by looking for the
// hlsManifestUrl / "isLive":true marker in the inline ytInitialPlayerResponse JSON.
//
// Writes data/live-status.json keyed by outlet.id:
// { "<id>": { isLive: bool, videoId: "...", checkedAt: ISO, error?: "..." } }
const fs = require('fs');
const path = require('path');
const https = require('https');
const zlib = require('zlib');
// Optional egress through a residential/ISP proxy. Prod (Kamatera) is a datacenter IP that
// YouTube walls; routing this fetch through a residential/ISP proxy gets prod the real page,
// which lets us retire the Mac2 residential-IP push (TK-11340). When PROXY_URL is UNSET the
// fetch behaves exactly as before (Mac2 dev path) — and the https-proxy-agent dep is required
// LAZILY, so only the proxied host (prod) ever needs it installed.
const PROXY_URL = process.env.PROXY_URL || '';
let proxyAgent = null;
if (PROXY_URL) {
const { HttpsProxyAgent } = require('https-proxy-agent');
proxyAgent = new HttpsProxyAgent(PROXY_URL);
}
// Live markers — presence means the channel is actively streaming. Used to detect liveness
// AND to EARLY-ABORT: we stop the download the instant we've seen enough, transferring
// ~0.2 MB instead of the full ~1.65 MB page (decisive on metered/proxy bandwidth).
const LIVE_MARKER = /"hlsManifestUrl":"https:\/\/manifest\.googlevideo\.com|"isLive":true|"isLiveNow":true/;
const MAX_DECODED = 900_000; // live markers appear by ~750 KB; also bounds not-live pages
const OUTLETS_PATH = path.join(__dirname, '..', 'data', 'outlets.json');
const STATUS_PATH = path.join(__dirname, '..', 'data', 'live-status.json');
const UA =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36';
function fetchHtml(url, timeoutMs = 10000, agent = proxyAgent) {
return new Promise((resolve, reject) => {
let done = false;
let req;
const settle = (fn, arg) => {
if (done) return;
done = true;
try { req && req.destroy(); } catch (_) {}
fn(arg);
};
const u = new URL(url);
req = https.request(
{
hostname: u.hostname,
port: u.port || 443,
path: u.pathname + u.search,
method: 'GET',
agent: agent || undefined,
headers: {
'User-Agent': UA,
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml',
// Ask for compression — cuts the transferred page ~4.4x (1.65 MB -> ~0.38 MB),
// then we decompress locally. Bandwidth billed by a proxy = the compressed bytes.
'Accept-Encoding': 'gzip, deflate, br',
// Bypass YouTube's consent/cookie interstitial (served to datacenter IPs like prod,
// which otherwise returns a consent page with no live markers → false "not live").
Cookie: 'CONSENT=YES+1; SOCS=CAI',
},
},
res => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
// Follow one hop (reuses the proxy agent via module scope).
res.resume();
if (!done) {
done = true;
fetchHtml(new URL(res.headers.location, url).toString(), timeoutMs).then(resolve, reject);
}
return;
}
if (res.statusCode !== 200) {
res.resume();
return settle(reject, new Error('status ' + res.statusCode));
}
const enc = (res.headers['content-encoding'] || '').toLowerCase();
const stream =
enc === 'gzip' ? res.pipe(zlib.createGunzip())
: enc === 'br' ? res.pipe(zlib.createBrotliDecompress())
: enc === 'deflate' ? res.pipe(zlib.createInflate())
: res;
let text = '';
stream.on('data', d => {
text += d.toString('utf8');
// Early-abort: we have what we need (live markers), or the page is long enough
// to conclude it's NOT live — either way stop transferring.
if (LIVE_MARKER.test(text) || text.length > MAX_DECODED) settle(resolve, text);
});
stream.on('end', () => settle(resolve, text));
stream.on('error', e => settle(reject, e));
res.on('error', e => settle(reject, e));
}
);
req.setTimeout(timeoutMs, () => settle(reject, new Error('timeout')));
// Swallow the post-abort 'aborted'/'destroy' error we cause on early-abort; surface real ones.
req.on('error', e => { if (!done) settle(reject, e); });
req.end();
});
}
function detectLive(html) {
// Marker 1 — hls manifest is present only when actively streaming
// Anchor on the live SIGNAL itself, not its position relative to "videoDetails".
// YouTube serves different ytInitialPlayerResponse layouts to different egress IPs
// (residential Mac2 vs the residential/ISP proxy prod now uses) — through the proxy the
// primary "isLive":true can sit ~370 KB *before* the "videoDetails" key, so the old
// proximity regex /"videoDetails":\{[^}]*"isLive":true/ missed every live channel and
// reported 0/20 live. These three markers match LIVE_MARKER exactly, so any marker that
// early-aborts the download is re-confirmed here. Verified no false positive: non-live
// channels (Fox/MSNBC/BBC) carry zero "isLive":true through the proxy (TK-11340).
if (
/"hlsManifestUrl":\s*"https:\/\/manifest\.googlevideo\.com/.test(html) ||
/"isLive":true/.test(html) ||
/"isLiveNow":true/.test(html)
) {
const m = html.match(/"videoId":\s*"([A-Za-z0-9_-]{11})"/);
return { isLive: true, videoId: m ? m[1] : null };
}
return { isLive: false, videoId: null };
}
async function checkOne(outlet) {
if (outlet.liveCheck !== 'youtube' || !outlet.youtube) {
return { id: outlet.id, isLive: false, videoId: null, skipped: true };
}
const url = `https://www.youtube.com/channel/${outlet.youtube}/live?gl=US&hl=en`;
// DIRECT-FIRST, PROXY-FALLBACK (TK-11340): prod's datacenter IP gets the real YouTube
// watch page MOST of the time — the consent/bot wall is intermittent — so try direct
// (free, no proxy bandwidth) and only fall back to the residential proxy when the page
// is walled. A real watch page (live OR not) always carries ytInitialPlayerResponse; a
// consent/bot wall does not — that's the reliable "walled" signal. This also degrades
// gracefully: if the proxy is ever down/lapsed, direct still covers every unwalled fetch
// instead of a hard 0-live outage.
try {
let html = await fetchHtml(url, 10000, null); // direct (no proxy)
if (!/ytInitialPlayerResponse/.test(html) && proxyAgent) {
html = await fetchHtml(url, 10000, proxyAgent); // walled → residential fallback
}
const { isLive, videoId } = detectLive(html);
return { id: outlet.id, isLive, videoId };
} catch (e) {
// Direct threw (timeout/reset — often itself a soft block): try the proxy before giving up.
if (proxyAgent) {
try {
const html = await fetchHtml(url, 10000, proxyAgent);
const { isLive, videoId } = detectLive(html);
return { id: outlet.id, isLive, videoId };
} catch (e2) {
return { id: outlet.id, isLive: false, videoId: null, error: e2.message };
}
}
return { id: outlet.id, isLive: false, videoId: null, error: e.message };
}
}
async function runBatch(outlets, concurrency = 6) {
const results = {};
let idx = 0;
async function worker() {
while (idx < outlets.length) {
const o = outlets[idx++];
const r = await checkOne(o);
results[o.id] = {
isLive: r.isLive,
videoId: r.videoId,
checkedAt: new Date().toISOString(),
...(r.error ? { error: r.error } : {}),
};
}
}
await Promise.all(Array.from({ length: concurrency }, worker));
return results;
}
async function main() {
const outlets = JSON.parse(fs.readFileSync(OUTLETS_PATH, 'utf8'));
const checkable = outlets.filter(o => o.liveCheck === 'youtube' && o.youtube);
console.log(`[check-live] checking ${checkable.length} of ${outlets.length} outlets`);
const t0 = Date.now();
const results = await runBatch(outlets);
const liveCount = Object.values(results).filter(r => r.isLive).length;
console.log(`[check-live] done in ${Date.now() - t0}ms — ${liveCount} live`);
// Merge with prior so we keep history if any outlet errored this tick
let prior = {};
try {
prior = JSON.parse(fs.readFileSync(STATUS_PATH, 'utf8'));
} catch (_) {}
const merged = { ...prior, ...results };
fs.writeFileSync(STATUS_PATH, JSON.stringify(merged, null, 2));
}
main().catch(e => {
console.error('[check-live] fatal', e);
process.exit(1);
});