← back to Allnewsdaily
scripts/short/upload-youtube.mjs
252 lines
#!/usr/bin/env node
// upload-youtube.mjs — resumable YouTube upload for the allnewsdaily daily-Short pipeline (TK-11342, STAGE 4).
// Node v26, raw fetch, NO googleapis dependency.
//
// Exports: async uploadShort({ file, title, description, tags, privacyStatus='unlisted', thumbnail }) -> { videoId, url }
//
// Cody/DTD red-team hardening (2026-09-09):
// - TIMEOUTS (#5): every network call has an AbortController deadline (no silent hang).
// - RESUMABLE RESUME (#6): a dropped byte-PUT is RESUMED from the server's received
// offset (Content-Range: bytes */size → 308 Range) — it NEVER re-POSTs the init,
// so a bad-network retry costs 0 extra quota units instead of another 1,600.
// - THUMBNAIL (#4): uploads data/short/thumb.jpg via thumbnails.set (best-effort).
// - CLI guard fixed for the .mjs rename.
//
// CLI: node scripts/short/upload-youtube.mjs --file data/short/out.mp4 --title "…" [--dry-run] …
import { readFileSync, existsSync, statSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { join, isAbsolute, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
const SECRETS_ENV = join(homedir(), 'Projects', 'secrets-manager', '.env');
const APP_ENV = join(homedir(), 'Projects', 'allnewsdaily', '.env');
const PROJECT_ROOT = join(homedir(), 'Projects', 'allnewsdaily');
const TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token';
const RESUMABLE_ENDPOINT = 'https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet,status';
const THUMBNAIL_ENDPOINT = 'https://www.googleapis.com/upload/youtube/v3/thumbnails/set';
const CATEGORY_ID = '25'; // News & Politics
const PUT_ATTEMPTS = 3; // resume tries for the byte upload
const T_SHORT = 30000; // token/init/status/thumbnail timeout
const T_PUT = 180000; // byte-PUT timeout
// fetch with an AbortController deadline
async function tfetch(url, opts = {}, ms = T_SHORT) {
const ac = new AbortController();
const to = setTimeout(() => ac.abort(), ms);
try { return await fetch(url, { ...opts, signal: ac.signal }); }
finally { clearTimeout(to); }
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
function parseEnv(path) {
const out = {};
if (!existsSync(path)) return out;
for (const raw of readFileSync(path, 'utf8').split('\n')) {
const line = raw.trim();
if (!line || line.startsWith('#')) continue;
const eq = line.indexOf('=');
if (eq === -1) continue;
let val = line.slice(eq + 1).trim();
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) val = val.slice(1, -1);
out[line.slice(0, eq).trim()] = val;
}
return out;
}
function loadCreds() {
const secrets = parseEnv(SECRETS_ENV);
const app = parseEnv(APP_ENV);
return {
clientId: secrets.YOUTUBE_CLIENT_ID || process.env.YOUTUBE_CLIENT_ID,
clientSecret: secrets.YOUTUBE_CLIENT_SECRET || process.env.YOUTUBE_CLIENT_SECRET,
refreshToken: app.YOUTUBE_REFRESH_TOKEN || secrets.YOUTUBE_REFRESH_TOKEN || process.env.YOUTUBE_REFRESH_TOKEN,
};
}
async function refreshAccessToken({ clientId, clientSecret, refreshToken }) {
if (!clientId || !clientSecret) throw new Error(`Missing YOUTUBE_CLIENT_ID/SECRET in ${SECRETS_ENV}`);
if (!refreshToken) throw new Error(`Missing YOUTUBE_REFRESH_TOKEN (run youtube-auth.mjs first to authorize; saved to ${APP_ENV})`);
const body = new URLSearchParams({ client_id: clientId, client_secret: clientSecret, refresh_token: refreshToken, grant_type: 'refresh_token' });
const res = await tfetch(TOKEN_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: body.toString() });
const json = await res.json().catch(() => ({}));
if (!res.ok || !json.access_token) {
// A revoked/expired refresh token surfaces here as invalid_grant — make it loud.
throw new Error(`Access-token refresh failed (${res.status}): ${json.error || ''} ${json.error_description || ''}${json.error === 'invalid_grant' ? ' — refresh token revoked/expired; re-run youtube-auth.mjs' : ''}`);
}
return json.access_token;
}
function buildMetadata({ title, description, tags, privacyStatus }) {
const tagArr = Array.isArray(tags) ? tags
: typeof tags === 'string' && tags.length ? tags.split(',').map((t) => t.trim()).filter(Boolean) : [];
return {
snippet: { title: title || '', description: description || '', tags: tagArr, categoryId: CATEGORY_ID },
status: { privacyStatus: privacyStatus || 'unlisted', selfDeclaredMadeForKids: false },
};
}
const resolveFile = (file) => { if (!file) throw new Error('file is required'); return isAbsolute(file) ? file : join(PROJECT_ROOT, file); };
// Query how many bytes the resumable session has already received (0 on a fresh session).
async function queryOffset(location, size, accessToken) {
const res = await tfetch(location, {
method: 'PUT',
headers: { Authorization: `Bearer ${accessToken}`, 'Content-Range': `bytes */${size}`, 'Content-Length': '0' },
}, T_SHORT);
if (res.status === 200 || res.status === 201) { const j = await res.json().catch(() => ({})); return { done: true, json: j }; }
if (res.status === 308) {
const range = res.headers.get('range'); // e.g. "bytes=0-262143"
const m = range && range.match(/bytes=0-(\d+)/);
return { done: false, offset: m ? Number(m[1]) + 1 : 0 };
}
return { done: false, offset: 0 }; // unknown → restart the bytes (still same session, no extra quota)
}
// Best-effort custom thumbnail (a failure here must NOT fail the upload).
async function setThumbnail(videoId, file, accessToken) {
try {
const abs = resolveFile(file);
if (!existsSync(abs)) return { ok: false, note: 'thumb not found' };
const bytes = await readFile(abs);
const res = await tfetch(`${THUMBNAIL_ENDPOINT}?videoId=${videoId}`, {
method: 'POST',
headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'image/jpeg', 'Content-Length': String(bytes.length) },
body: bytes,
}, T_SHORT);
return { ok: res.ok, note: res.ok ? 'set' : `HTTP ${res.status}` };
} catch (e) { return { ok: false, note: e.message }; }
}
export async function uploadShort({ file, title, description, tags, privacyStatus = 'unlisted', thumbnail }) {
const abs = resolveFile(file);
if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
const size = statSync(abs).size;
const creds = loadCreds();
const accessToken = await refreshAccessToken(creds);
const metadata = buildMetadata({ title, description, tags, privacyStatus });
// Step 1 — init the resumable session (ONCE; never re-POSTed on retry → no quota waste).
const initRes = await tfetch(RESUMABLE_ENDPOINT, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json; charset=UTF-8',
'X-Upload-Content-Type': 'video/*',
'X-Upload-Content-Length': String(size),
},
body: JSON.stringify(metadata),
}, T_SHORT);
if (!initRes.ok) { const t = await initRes.text().catch(() => ''); throw new Error(`Resumable init failed (${initRes.status}): ${t.slice(0, 400)}`); }
const location = initRes.headers.get('location');
if (!location) throw new Error('Resumable init returned no Location header');
// Step 2 — PUT the bytes, RESUMING from the server's offset on any failure.
const bytes = await readFile(abs);
let offset = 0;
let lastErr = null;
for (let attempt = 1; attempt <= PUT_ATTEMPTS; attempt++) {
try {
const slice = offset > 0 ? bytes.subarray(offset) : bytes;
const putRes = await tfetch(location, {
method: 'PUT',
headers: { 'Content-Type': 'video/*', 'Content-Length': String(size - offset), 'Content-Range': `bytes ${offset}-${size - 1}/${size}` },
body: slice,
}, T_PUT);
if (putRes.ok) {
const j = await putRes.json().catch(() => ({}));
if (j.id) return await finalize(j.id, thumbnail, accessToken);
throw new Error(`Byte upload OK but no video id: ${JSON.stringify(j).slice(0, 300)}`);
}
if (putRes.status === 308) { // incomplete — resume from reported offset
const range = putRes.headers.get('range');
const m = range && range.match(/bytes=0-(\d+)/);
offset = m ? Number(m[1]) + 1 : offset;
continue;
}
const t = await putRes.text().catch(() => '');
throw new Error(`Byte upload failed (${putRes.status}): ${t.slice(0, 300)}`);
} catch (e) {
lastErr = e;
if (attempt === PUT_ATTEMPTS) break;
await sleep(1500 * attempt); // backoff
const q = await queryOffset(location, size, accessToken).catch(() => ({ done: false, offset }));
if (q.done && q.json && q.json.id) return await finalize(q.json.id, thumbnail, accessToken);
offset = q.offset ?? offset;
}
}
throw new Error(`Byte upload failed after ${PUT_ATTEMPTS} attempts: ${lastErr && lastErr.message}`);
}
// Exposed for the post-publish delete-canary (needs a fresh access token + privacy update).
export async function refreshedAccessToken() { return refreshAccessToken(loadCreds()); }
// Change an EXISTING video's privacy (requires youtube.force-ssl scope). Used by the delete-canary
// to auto-unlist a video whose source(s) got retracted. Reversible (set back to 'public').
export async function setVideoPrivacy(videoId, privacyStatus, accessToken) {
const token = accessToken || (await refreshedAccessToken());
const res = await tfetch('https://www.googleapis.com/youtube/v3/videos?part=status', {
method: 'PUT',
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
body: JSON.stringify({ id: videoId, status: { privacyStatus, selfDeclaredMadeForKids: false } }),
}, T_SHORT);
const j = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(`setVideoPrivacy(${videoId}→${privacyStatus}) failed (${res.status}): ${JSON.stringify(j).slice(0, 300)}`);
return j.status ? j.status.privacyStatus : privacyStatus;
}
async function finalize(videoId, thumbnail, accessToken) {
let thumb = { ok: false, note: 'skipped' };
if (thumbnail) thumb = await setThumbnail(videoId, thumbnail, accessToken);
return { videoId, url: `https://youtu.be/${videoId}`, thumbnail: thumb };
}
// --- CLI -------------------------------------------------------------------
function parseArgs(argv) {
const args = { dryRun: false };
for (let i = 2; i < argv.length; i++) {
const a = argv[i]; const next = () => argv[++i];
switch (a) {
case '--file': args.file = next(); break;
case '--title': args.title = next(); break;
case '--description': args.description = next(); break;
case '--tags': args.tags = next(); break;
case '--thumbnail': args.thumbnail = next(); break;
case '--privacy': case '--privacyStatus': args.privacyStatus = next(); break;
case '--dry-run': args.dryRun = true; break;
default: break;
}
}
return args;
}
async function cli() {
const args = parseArgs(process.argv);
if (!args.file) { console.error('Usage: node scripts/short/upload-youtube.mjs --file <path> --title "…" [--thumbnail p] [--privacy unlisted] [--dry-run]'); process.exit(1); }
const privacyStatus = args.privacyStatus || 'unlisted';
const metadata = buildMetadata({ title: args.title, description: args.description, tags: args.tags, privacyStatus });
const abs = resolveFile(args.file);
const exists = existsSync(abs);
const size = exists ? statSync(abs).size : 0;
if (args.dryRun) {
console.log('\n[upload-youtube] --dry-run — NO upload performed.\n');
console.log(`File: ${abs} exists=${exists}${exists ? ` (${size} bytes)` : ' (WARNING: not found)'}`);
console.log(`Endpoints:\n 1. POST ${RESUMABLE_ENDPOINT}\n 2. PUT <Location> (resumable, Content-Range)\n thumb: POST ${THUMBNAIL_ENDPOINT}?videoId=…\n token: POST ${TOKEN_ENDPOINT}`);
console.log('\nMetadata:\n' + JSON.stringify(metadata, null, 2));
console.log(`\ncategoryId=${CATEGORY_ID}, selfDeclaredMadeForKids=false, privacyStatus=${privacyStatus}\n`);
return;
}
try {
const r = await uploadShort({ file: args.file, title: args.title, description: args.description, tags: args.tags, privacyStatus, thumbnail: args.thumbnail });
console.log(`\n[upload-youtube] Uploaded. videoId=${r.videoId}\n ${r.url}\n thumbnail: ${r.thumbnail.note}\n`);
} catch (e) { console.error(`\n[upload-youtube] FAILED: ${e.message}\n`); process.exit(1); }
}
// Run the CLI only when invoked directly (extension-agnostic — survives the .js→.mjs rename).
if (process.argv[1] && process.argv[1] === fileURLToPath(import.meta.url)) cli();