"""Collect Vancouver Park Board meeting minutes (and in-camera releases) by year.

parkboardmeetings.vancouver.ca sits behind Cloudflare, which (as of July 2026)
lets exactly ONE request per fresh headless-Chrome session through before
flagging the session. So this collector brute-forces politely: a fresh stealth
session per year index to harvest links, then a fresh session per PDF, letting
Chrome's own download machinery grab the file as the session's first request.
Slow (~10 s/doc) but deterministic and resume-safe: re-running skips anything
already on disk.

Output layout:
    data/minutes/<year>/<basename>.pdf
    data/minutes/<year>/<basename>.txt   (PyMuPDF text extraction)
    data/minutes/manifest.json           (all harvested links + status)

Usage:
    python collect_minutes.py                    # 2019-2026
    python collect_minutes.py --years 2017 2018  # extend backwards
    python collect_minutes.py --harvest-only     # just rebuild the manifest
"""
from __future__ import annotations

import argparse
import json
import re
import shutil
import sys
import tempfile
import time
from pathlib import Path

import fitz  # PyMuPDF
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager

sys.path.insert(0, str(Path(__file__).parent.parent))
from selenium_helpers import make_options

BASE = "https://parkboardmeetings.vancouver.ca"
HERE = Path(__file__).parent
OUT_DIR = HERE / "data" / "minutes"

# Link-text patterns worth downloading
WANTED_TEXT = re.compile(r"minutes|decision|report", re.I)

_service = None  # reuse the resolved chromedriver binary across sessions


def new_driver(download_dir: Path | None = None):
    global _service
    if _service is None:
        _service = Service(ChromeDriverManager().install())
    opts = make_options(headless=True, stealth=True)
    if download_dir is not None:
        opts.add_experimental_option(
            "prefs",
            {
                "download.default_directory": str(download_dir),
                "download.prompt_for_download": False,
                "plugins.always_open_pdf_externally": True,
            },
        )
    return webdriver.Chrome(service=_service, options=opts)


def harvest_year(year: int) -> list[dict]:
    """Fresh session; the year index is its first (and only) request."""
    d = new_driver()
    try:
        d.get(f"{BASE}/{year}/index.htm")
        time.sleep(4)
        if "cloudflare" in d.title.lower():
            print(f"  {year}: BLOCKED", file=sys.stderr)
            return []
        seen, out = set(), []
        for a in d.find_elements(By.TAG_NAME, "a"):
            href = (a.get_attribute("href") or "").replace("http://", "https://")
            text = (a.text or "").strip()
            if not href.lower().endswith(".pdf") or href in seen:
                continue
            if not (WANTED_TEXT.search(text) or "MINUTES" in href.upper() or "/documents/IC" in href):
                continue
            seen.add(href)
            out.append({"year": year, "text": text, "url": href})
        return out
    finally:
        d.quit()


def download_pdf(url: str, dest: Path, timeout: int = 45) -> str:
    """Fresh session; the PDF itself is the session's first request."""
    if dest.exists() and dest.stat().st_size > 0:
        return "cached"
    tmp = Path(tempfile.mkdtemp(prefix="pb_dl_"))
    d = new_driver(download_dir=tmp)
    try:
        d.get(url)
        deadline = time.time() + timeout
        got = None
        while time.time() < deadline:
            pdfs = [f for f in tmp.iterdir() if f.suffix.lower() == ".pdf"]
            partial = [f for f in tmp.iterdir() if f.suffix == ".crdownload"]
            if pdfs and not partial:
                got = pdfs[0]
                break
            time.sleep(1)
        if got is None:
            return "timeout (blocked?)"
        dest.parent.mkdir(parents=True, exist_ok=True)
        shutil.move(str(got), str(dest))
        return "ok"
    finally:
        d.quit()
        shutil.rmtree(tmp, ignore_errors=True)


def extract_text(pdf_path: Path) -> str:
    txt_path = pdf_path.with_suffix(".txt")
    if txt_path.exists():
        return "cached"
    try:
        doc = fitz.open(pdf_path)
        pages = [f"--- page {i + 1} ---\n{p.get_text()}" for i, p in enumerate(doc)]
        txt_path.write_text("\n".join(pages), encoding="utf-8")
        return "ok"
    except Exception as e:  # keep going; note failures in manifest
        return f"extract failed: {e}"


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--years", nargs="*", type=int, default=list(range(2019, 2027)))
    ap.add_argument("--harvest-only", action="store_true")
    args = ap.parse_args()

    links = []
    for year in args.years:
        year_links = harvest_year(year)
        print(f"{year}: {len(year_links)} PDFs listed", file=sys.stderr)
        links.extend(year_links)

    if not args.harvest_only:
        for i, link in enumerate(links, 1):
            name = link["url"].rsplit("/", 1)[-1]
            dest = OUT_DIR / str(link["year"]) / name
            status = download_pdf(link["url"], dest)
            link["file"] = str(dest.relative_to(HERE)) if dest.exists() else None
            link["download"] = status
            if dest.exists():
                link["extract"] = extract_text(dest)
            print(f"  [{i}/{len(links)}] {name}: {status}", file=sys.stderr)

    OUT_DIR.mkdir(parents=True, exist_ok=True)
    manifest_path = OUT_DIR / "manifest.json"
    manifest_path.write_text(json.dumps(links, indent=2), encoding="utf-8")
    ok = sum(1 for m in links if m.get("download") in ("ok", "cached"))
    print(f"\nManifest: {manifest_path}  ({ok}/{len(links)} downloaded)")


if __name__ == "__main__":
    main()