`fanout_portfolio.py`

155 lines
#!/usr/bin/env python3
"""
Portfolio-wide GSC owner-grant fan-out (Steve-authorized 2026-08-17, entire portfolio).

Grants the ga-allsites SA verified-owner + sc-domain Search Console access across
EVERY GoDaddy + Cloudflare domain we control DNS for. Two-phase for speed + quota
safety, resumable via a JSONL log (rerun to pick up where a throttle left off).

  Phase 1  mint DNS-TXT token + write apex TXT (Cloudflare or GoDaddy)   [no sleeps]
  ---      one propagation wait
  Phase 2  verify SA as owner + add sc-domain:<domain>                    [429-backoff]

Reversible: delete the google-site-verification TXT + remove the SA user in GSC.

Usage:  python3 fanout_portfolio.py            # resume/run the full union
        python3 fanout_portfolio.py --phase2   # only re-run verify+add (TXT already written)
"""
import json
import pathlib
import sys
import threading
import time
import urllib.error
import urllib.parse
from concurrent.futures import ThreadPoolExecutor, as_completed

import grant_microsites_gsc as G

_LOG_LOCK = threading.Lock()
WORKERS = 10  # parallel verify+add; 429-backoff absorbs quota bumps

HERE = pathlib.Path(__file__).parent
PORT = HERE / "cache" / "_portfolio_domains.json"
LOG = HERE / "cache" / "_fanout_results.jsonl"
PROP_WAIT = 45  # one-shot DNS propagation wait between phases


def load_done():
    """domains already fully GRANTED in a prior run (resume)."""
    done = {}
    if LOG.exists():
        for line in LOG.read_text().splitlines():
            if not line.strip():
                continue
            r = json.loads(line)
            done[r["domain"]] = r["result"]
    return done


def logline(domain, result, token=""):
    with _LOG_LOCK, LOG.open("a") as f:
        f.write(json.dumps({"domain": domain, "result": result, "token": token}) + "\n")


def verify_and_add(domain, gtoken):
    """Phase-2 verify owner + add sc-domain, with 429 backoff. Returns result str."""
    for attempt in range(4):
        try:
            G.gget("https://www.googleapis.com/siteVerification/v1/webResource"
                   "?verificationMethod=DNS_TXT", gtoken,
                   {"site": {"type": "INET_DOMAIN", "identifier": domain}})
            break
        except urllib.error.HTTPError as e:
            if e.code == 429:
                time.sleep(20 * (attempt + 1))
                continue
            return f"VERIFY {e.code}: {e.read().decode()[:120]}"
    else:
        return "VERIFY throttled"
    site = urllib.parse.quote(f"sc-domain:{domain}", safe="")
    for attempt in range(4):
        try:
            G.gget(f"https://searchconsole.googleapis.com/webmasters/v3/sites/{site}",
                   gtoken, method="PUT")
            return "GRANTED"
        except urllib.error.HTTPError as e:
            if e.code == 409:
                return "GRANTED"  # already present
            if e.code == 429:
                time.sleep(20 * (attempt + 1))
                continue
            return f"OWNER-OK sites.add {e.code}"
    return "OWNER-OK sites.add throttled"


def main():
    phase2_only = "--phase2" in sys.argv
    domains = json.loads(PORT.read_text())["union"]
    creds = G.google_creds()
    gtoken = creds.token
    cftok = G.cf_token()
    gd = G.godaddy_creds()
    done = load_done()

    todo = [d for d in domains if done.get(d) != "GRANTED"]
    print(f"portfolio: {len(domains)} domains | already GRANTED: "
          f"{sum(1 for v in done.values() if v=='GRANTED')} | to process: {len(todo)}", flush=True)

    tokens = {}
    if not phase2_only:
        print("=== PHASE 1: mint token + write TXT ===", flush=True)
        for i, d in enumerate(todo, 1):
            try:
                tok = G.gget("https://www.googleapis.com/siteVerification/v1/token", gtoken,
                             {"verificationMethod": "DNS_TXT",
                              "site": {"type": "INET_DOMAIN", "identifier": d}})["token"]
                state = G.write_txt(d, tok, cftok, gd)
                tokens[d] = tok
                if state == "no-host":
                    logline(d, "SKIP no-host")
                    tokens.pop(d, None)
                if i % 25 == 0:
                    print(f"  phase1 {i}/{len(todo)} ...", flush=True)
            except urllib.error.HTTPError as e:
                logline(d, f"PHASE1 {e.code}: {e.read().decode()[:100]}")
            except Exception as e:  # noqa: BLE001
                logline(d, f"PHASE1 ERR: {e}")
        print(f"phase1 done: {len(tokens)} TXT written/present. waiting {PROP_WAIT}s ...", flush=True)
        time.sleep(PROP_WAIT)
    else:
        tokens = {d: True for d in todo}  # assume TXT already there

    print(f"=== PHASE 2: verify owner + add sc-domain ({WORKERS} parallel) ===", flush=True)
    p2 = [d for d in todo if d in tokens]
    granted = 0
    done_n = 0

    def _work(d):
        res = verify_and_add(d, gtoken)
        logline(d, res, tokens.get(d, "") if isinstance(tokens.get(d), str) else "")
        return d, res

    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        futs = {ex.submit(_work, d): d for d in p2}
        for fut in as_completed(futs):
            d, res = fut.result()
            done_n += 1
            if res == "GRANTED":
                granted += 1
            if done_n % 25 == 0:
                print(f"  phase2 {done_n}/{len(p2)} ... (+{granted} granted)", flush=True)

    # summary
    final = load_done()
    from collections import Counter
    c = Counter(v.split(":")[0].split(" ")[0] for v in final.values())
    print("=== SUMMARY ===", flush=True)
    for k, n in c.most_common():
        print(f"  {k:20} {n}", flush=True)
    print(f"total logged: {len(final)}", flush=True)


if __name__ == "__main__":
    main()