← back to Rentv Website

scripts/pull-videos.mjs

77 lines

#!/usr/bin/env node
// Live-data engine for The REview (rentv-v1 video section): pulls the REAL
// rentvreview.com video catalog on a schedule and writes data/videos.json,
// which /api/videos serves and public/review.html renders in the RENTV shell.
// $0 (plain fetch, no browser). Mirrors pull-news.mjs exactly.
import { writeFileSync, mkdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const HERE = dirname(fileURLToPath(import.meta.url));
const OUT = join(HERE, '..', 'data'); mkdirSync(OUT, { recursive: true });
const SRC = 'https://www.rentvreview.com';

async function fetchDecoded(url) {
  const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 RENTV-v1' }, signal: AbortSignal.timeout(20000) });
  const buf = Buffer.from(await r.arrayBuffer());
  let s = buf.toString('utf8');
  if ((s.match(/�/g) || []).length > 5) s = buf.toString('latin1'); // WIN-1252 fallback
  return s;
}
const clean = (t) => (t || '')
  .replace(/&/g, '&').replace(/'/g, "'").replace(/&#0?39;/g, "'").replace(/’/gi, "’")
  .replace(/‘/gi, "‘").replace(/"/g, '"').replace(/&#x201[cd];/gi, '"')
  .replace(/&#x0?D;/gi, ' ').replace(/&#x0?A;/gi, ' ').replace(/&#1[03];/g, ' ')
  .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

// Walk the homepage, splitting on each preview block, and attach the nearest
// preceding <h4> heading (JLL TV / Most Recent / Most Viewed / Industry Events…)
// as the video's category rail.
function parse(html) {
  const seen = new Set(), items = [];
  const blocks = html.split('video-preview__thumbnail');
  for (let i = 1; i < blocks.length; i++) {
    const before = blocks[i - 1], b = blocks[i];
    const view = b.match(/\/Videos\/View\/(\d+)/);
    const yt = b.match(/youtube\.com\/vi\/([A-Za-z0-9_-]{6,})\//i);
    const title = b.match(/video-preview__text__title[^>]*>[\s\S]*?<div>([\s\S]*?)<\/div>/i);
    const desc = b.match(/video-preview__text__description[^>]*>([\s\S]*?)<\/div>/i);
    if (!view || !yt) continue;
    const id = view[1]; if (seen.has(id)) continue; seen.add(id);
    // nearest preceding <h4> = the rail this card sits under
    const heads = [...before.matchAll(/<h4[^>]*>([\s\S]*?)<\/h4>/gi)];
    const cat = heads.length ? clean(heads[heads.length - 1][1]) : 'Featured';
    items.push({
      id,
      yt: yt[1],
      title: clean(title ? title[1] : '') || 'RENTV Video',
      desc: clean(desc ? desc[1] : '').slice(0, 240),
      cat: cat || 'Featured',
      view_url: SRC + '/Videos/View/' + id,
      thumb: 'https://img.youtube.com/vi/' + yt[1] + '/hqdefault.jpg',
      embed: 'https://www.youtube.com/embed/' + yt[1],
    });
  }
  return items;
}

try {
  const html = await fetchDecoded(SRC + '/');
  const items = parse(html);
  if (!items.length) throw new Error('parsed 0 videos — markup may have changed');
  const cats = [...new Set(items.map(v => v.cat))];
  const payload = {
    source: 'rentvreview.com (The REview) — live',
    fetched_at: new Date().toISOString(),
    count: items.length,
    cats,
    items,
  };
  writeFileSync(join(OUT, 'videos.json'), JSON.stringify(payload, null, 1));
  console.log('OK: pulled ' + items.length + ' videos across ' + cats.length + ' rails: ' + cats.join(', '));
} catch (e) {
  console.error('pull-videos failed:', e.message);
  // never overwrite a good cache with nothing — leave last-good in place
  try { JSON.parse(readFileSync(join(OUT, 'videos.json'), 'utf8')); console.error('kept last-good videos.json'); } catch {}
  process.exit(1);
}