`src/conrad/normalize.py`

146 lines
"""Date, title, subject and name normalisation."""
from __future__ import annotations

import calendar
import re
import unicodedata

MONTHS = {m.lower(): i for i, m in enumerate(calendar.month_abbr) if m}
MONTHS.update({m.lower(): i for i, m in enumerate(calendar.month_name) if m})
MONTHS["sept"] = 9


def _mon(word: str | None) -> int | None:
    if not word:
        return None
    return MONTHS.get(word.lower().rstrip(".")) or MONTHS.get(word.lower()[:3])


def _last(y: int, m: int) -> int:
    return calendar.monthrange(y, m)[1]


def _iso(y, m=None, d=None) -> str:
    return f"{y:04d}" + (f"-{m:02d}" if m else "") + (f"-{d:02d}" if d else "")


def parse_date(text: str | None) -> dict:
    """Parse free-text archival dates into {date_exact, date_start, date_end, year, estimate}.

    date_exact is only set when a full day is known. Ranges set date_start/date_end.
    """
    out = {"date_exact": None, "date_start": None, "date_end": None, "year": None, "estimate": False}
    if not text:
        return out
    t = str(text).strip()
    t = re.sub(r"\[(publication date|date created|printed later)\]", "", t, flags=re.I).strip()
    if re.search(r"\?|circa|ca\.|approximately", t, re.I):
        out["estimate"] = True
    # ISO: 1989-07-16 / 1969-03 / 1969
    m = re.fullmatch(r"\[?(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?\]?", t)
    if m:
        y, mo, d = int(m[1]), int(m[2]) if m[2] else None, int(m[3]) if m[3] else None
        return _fill(out, y, mo, d)
    # "1984 Mar. 13" / "1984 March" / "Mar 13, 1984" / "March 1964"
    m = re.search(r"(\d{4})\s+([A-Za-z]{3,9})\.?(?:\s+(\d{1,2}))?", t)
    if m and _mon(m[2]):
        return _fill(out, int(m[1]), _mon(m[2]), int(m[3]) if m[3] else None)
    m = re.search(r"([A-Za-z]{3,9})\.?\s+(?:(\d{1,2}),?\s+)?(\d{4})", t)
    if m and _mon(m[1]):
        return _fill(out, int(m[3]), _mon(m[1]), int(m[2]) if m[2] else None)
    # "between 1971 and 1980", "1950s-1963", "1967-1968", "1950s"
    yrs = [int(y) for y in re.findall(r"(1[89]\d\d|20[0-2]\d)", t)]
    decade = re.search(r"(1[89]\d0|20[0-2]0)s", t)
    if yrs:
        lo, hi = min(yrs), max(yrs)
        if decade and int(decade[1]) == lo and len(yrs) == 1:
            hi = lo + 9
        if lo == hi:
            return _fill(out, lo, None, None)
        out.update(date_start=f"{lo}-01-01", date_end=f"{hi}-12-31", year=None)
        return out
    return out


def _fill(out: dict, y: int, mo: int | None, d: int | None) -> dict:
    out["year"] = y
    if mo and d:
        out["date_exact"] = _iso(y, mo, d)
        out["date_start"] = out["date_end"] = out["date_exact"]
    elif mo:
        out["date_start"], out["date_end"] = _iso(y, mo, 1), _iso(y, mo, _last(y, mo))
    else:
        out["date_start"], out["date_end"] = f"{y}-01-01", f"{y}-12-31"
    return out


def norm_title(t: str | None) -> str:
    if not t:
        return ""
    t = unicodedata.normalize("NFKD", t)
    t = re.sub(r"--\s*news item\.?", "", t, flags=re.I)
    t = t.lower().replace("…", " ").replace("&", " and ")
    t = re.sub(r"[^a-z0-9 ]+", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return "" if t in ("untitled", "untitled drawing", "") else t


def clean_subject(s: str) -> str:
    s = re.sub(r"--\s*\d{4}-\d{4}\.?$", "", s.strip())
    s = re.sub(r",?\s*\d{4}-(\d{4})?\.?$", "", s)
    return s.rstrip(" .,").strip()


GENERIC_SUBJECTS = re.compile(r"^(editorial cartoons|political cartoons|drawings|caricatures|cartoons|"
                              r"american|pen and ink drawings|ink drawings|drawings--american|proofs|prints|graphic arts)\b", re.I)

PERSON_RE = re.compile(r"^([A-Z][A-Za-z'.\-]+(?: [A-Z][A-Za-z'.\-]+)?),\s+([A-Z][A-Za-z.\- ]+?)(?:\s*\(|,?\s*\d{4}|,?--|,?$)")

PRESIDENTS = {
    "truman": "Truman, Harry S.", "eisenhower": "Eisenhower, Dwight D.", "kennedy, john": "Kennedy, John F.",
    "johnson, lyndon": "Johnson, Lyndon B.", "nixon, richard": "Nixon, Richard M.", "ford, gerald": "Ford, Gerald R.",
    "carter, jimmy": "Carter, Jimmy", "reagan, ronald": "Reagan, Ronald", "bush, george h": "Bush, George H. W.",
    "clinton, bill": "Clinton, Bill", "bush, george w": "Bush, George W.", "obama": "Obama, Barack",
}
PRESIDENT_WORDS = {  # free-text mentions in titles/scope notes
    r"\btruman\b": "Truman, Harry S.", r"\beisenhower|\bike\b": "Eisenhower, Dwight D.",
    r"\bjfk\b|john f\.? kennedy": "Kennedy, John F.", r"\blbj\b|lyndon": "Johnson, Lyndon B.",
    r"\bnixon": "Nixon, Richard M.", r"gerald ford|\bford administration": "Ford, Gerald R.",
    r"jimmy carter|\bcarter administration": "Carter, Jimmy", r"\breagan": "Reagan, Ronald",
    r"\bclinton\b": "Clinton, Bill", r"\bobama": "Obama, Barack",
}


def is_president(name: str) -> bool:
    n = name.lower()
    return any(n.startswith(k) for k in PRESIDENTS) or name in PRESIDENTS.values()


def person_from_heading(h: str) -> str | None:
    """'Nixon, Richard M. (Richard Milhous), 1913-1994' -> 'Nixon, Richard M.'; non-person headings -> None."""
    h = h.strip()
    for k, v in PRESIDENTS.items():
        if h.lower().startswith(k):
            return v
    m = PERSON_RE.match(h) or PERSON_RE.match(clean_subject(h))
    if not m:
        return None
    first = m[2].strip()
    # reject corporate/geographic headings that happen to contain a comma
    if re.search(r"\b(Inc|Company|Corporation|University|Department|Association|Party|Committee|Council|"
                 r"County|State|City|Republic|Church|Industry|Office|Court|Commission|Bureau|Army|Navy)\b", h):
        return None
    if len(first) < 2 or first.lower() in {"the"}:
        return None
    return f"{m[1]}, {first}".strip()


def presidents_in_text(text: str | None) -> list[str]:
    if not text:
        return []
    return sorted({v for k, v in PRESIDENT_WORDS.items() if re.search(k, text, re.I)})


def decade(y: int | None) -> str:
    return f"{y // 10 * 10}s" if y else "undated"