"""Read a private foundation's filed Forms 990-PF in order, and verify a sealed ParetoAlpha carryover chain without
trusting ParetoAlpha. Standard library only; shares no code with the site.

    python3 verify_chain.py --read DIR                  # no proof at all: read a folder of IRS 990-PF XML files as a chain
    python3 verify_chain.py <hash or /proof/chain/ link> --fetch       # verify a sealed chain, fetching each return from irs.gov
    python3 verify_chain.py <hash> --returns DIR        # or against copies of the returns you already hold (<OBJECT_ID>.xml)
    python3 verify_chain.py <hash> --body chain.json --returns DIR --offline

What a sealed chain is checked against:
  1. The body.     sha256 of the canonical body (verify.py, beside this file) must be the hash.
  2. The ledger.   /api/proof/verify says whether the hash is on the ledger and when it was first recorded.
  3. The sources.  Every return: its bytes' SHA-256 must be the one the chain seals for that year, and the digest over
                   all of them, in order, must be the chain's sourcesSha256. --fetch reads each file from irs.gov
                   itself (one Range request each; the locators are not trusted, the hashes are).
  4. The years.    Every Part XII line is read again from the XML and every year is re-done from the rules below; the
                   result of each year, the lines that differ, the result of each year-to-year link and the opening
                   lines that differ must be the ones the chain states, and so must what the latest return carries.

The rules, each from its source and written here from that source:
  IRC 4942(h)(1): a year's qualifying distributions go first to the prior year's undistributed income, then to the
    year's own distributable amount, then to corpus; after the prior year is reduced, the elections on lines 4b and 4c
    take what is left (IRS instructions, Part XII lines 4a to 4c).
  Treas. Reg. 53.4942(a)-3(e)(1): the carryover applied is the lesser of the excess carried in and what is still
    undistributed after the year's own distributions; a corpus election does not make room for it (instructions,
    line 5); the earliest excess is applied first.
  Treas. Reg. 53.4942(a)-3(e)(3): what is left of the excess from five years back is not carried further (line 8).
    Line 7 may draw on the earliest excess or on the year's own corpus; line 8 agrees under either.

A line that differs is the return's figure beside the re-added one. It is not a finding: the return may rest on an
election statement, a set-aside, a corrected valuation or an amended return these files do not show.
"""
import argparse
import hashlib
import json
import os
import re
import sys
import xml.etree.ElementTree as ET
from datetime import date, timedelta
from decimal import Decimal
from pathlib import Path

SITE = "https://paretoalphasystems.com"
ZERO = Decimal(0)
ONE = Decimal(1)

LINE3 = ["ExcessDistributionCyovYr5Amt", "ExcessDistributionCyovYr4Amt", "ExcessDistributionCyovYr3Amt", "ExcessDistributionCyovYr2Amt", "ExcessDistributionCyovYr1Amt"]
LINE10 = ["ExcessFromYear4Amt", "ExcessFromYear3Amt", "ExcessFromYear2Amt", "ExcessFromYear1Amt", "ExcessFromCurrentYearAmt"]
FIELDS = {
    "distributable": "DistributableAsAdjustedAmt",
    "undistributedPriorYear": "UndistributedIncomePYAmt",
    "undistributedEarlierYears": "TotalForPriorYearsAmt",
    "carryInTotal": "TotalExcessDistributionCyovAmt",
    "qualifying": "QualifyingDistributionsAmt",
    "appliedPriorYear": "AppliedToYear1Amt",
    "appliedEarlierYears": "AppliedToPriorYearsAmt",
    "treatedAsCorpus": "TreatedAsDistriFromCorpusAmt",
    "appliedCurrentYear": "AppliedToCurrentYearAmt",
    "remainingToCorpus": "RemainingDistriFromCorpusAmt",
    "carryAppliedCorpus": "ExcessDistriCyovAppCYCorpusAmt",
    "carryApplied": "ExcessDistributionCyovAppCYAmt",
    "corpus": "TotalCorpusAmt",
    "earlierYearsRemaining": "PriorYearUndistributedIncmAmt",
    "deficiencyOrTaxed": "PriorYearDeficiencyOrTaxAmt",
    "earlierYearsTaxable": "Taxable1Amt",
    "priorYearRemaining": "Taxable2Amt",
    "undistributedCurrentYear": "UndistributedIncomeCYAmt",
    "corpusElection": "CorpusDistri170b1EOr4942g3Amt",
    "carryLapsed": "ExcessDistriCyovFromYr5Amt",
    "carryOut": "ExcessDistriCyovToNextYrAmt",
}


def local(tag):
    return tag.split("}", 1)[-1]


def find(node, name):
    """The first descendant element with this local name, or None."""
    if node is None:
        return None
    for el in node.iter():
        if local(el.tag) == name:
            return el
    return None


def text(node, name):
    el = find(node, name)
    return None if el is None else "".join(el.itertext()).strip()


def money(raw):
    """Whole or decimal dollars as filed -> a plain decimal string; anything else -> None."""
    if raw is None:
        return None
    t = re.sub(r"[,\s$]", "", raw)
    if not re.fullmatch(r"-?\d+(\.\d+)?", t):
        return None
    d = Decimal(t)
    s = format(d.normalize(), "f")
    return "0" if d == 0 else s


def yes(raw):
    return raw is not None and raw.strip().lower() in ("x", "1", "true", "yes")


def D(v):
    return ZERO if v is None else Decimal(v)


def read(path, object_id):
    with open(path, "rb") as f:
        return read_bytes(f.read(), object_id)


def read_bytes(raw, object_id):
    root = ET.fromstring(raw)
    header = find(root, "ReturnHeader")
    pf = find(find(root, "ReturnData"), "IRS990PF")
    grp = find(pf, "UndistributedIncomeGrp")
    bs = find(pf, "Form990PFBalanceSheetsGrp")
    dist = find(pf, "DistributableAmountGrp")
    part12 = None
    if grp is not None:
        part12 = {k: money(text(grp, v)) for k, v in FIELDS.items()}
        part12["carryIn"] = [money(text(grp, n)) for n in LINE3]
        part12["carryOutBy"] = [money(text(grp, n)) for n in LINE10]
    return {
        "sha256": hashlib.sha256(raw).hexdigest(),
        "objectId": object_id,
        "ein": text(find(header, "Filer"), "EIN"),
        "begin": text(header, "TaxPeriodBeginDt"),
        "periodEnd": text(header, "TaxPeriodEndDt"),
        "operating": yes(text(pf, "PrivateOperatingFoundationInd")),
        "box": yes(text(dist, "Sect4942j3j5FndtnAndFrgnOrgInd")) if dist is not None else False,
        "balance": {
            "assets": (money(text(bs, "TotalAssetsBOYAmt")), money(text(bs, "TotalAssetsEOYAmt"))),
            "liabilities": (money(text(bs, "TotalLiabilitiesBOYAmt")), money(text(bs, "TotalLiabilitiesEOYAmt"))),
            "net": (money(text(bs, "TotNetAstOrFundBalancesBOYAmt")), money(text(bs, "TotNetAstOrFundBalancesEOYAmt"))),
        },
        "part12": part12,
    }


def vintage_known(p):
    return abs(sum(D(v) for v in p["carryIn"]) - D(p["carryInTotal"])) <= ONE


def lapse(p, line7_first_from_this_year):
    """The year re-done from its own opening lines; returns (4a, 4d, 4e, 5, lapsed)."""
    d_amt = max(ZERO, D(p["distributable"]))
    left = max(ZERO, D(p["qualifying"]))
    py = max(ZERO, D(p["undistributedPriorYear"]))
    earlier = max(ZERO, D(p["undistributedEarlierYears"]))
    carry = [max(ZERO, D(v)) for v in p["carryIn"]] if vintage_known(p) else [ZERO, ZERO, ZERO, ZERO, max(ZERO, D(p["carryInTotal"]))]
    a4 = min(py, left)
    left -= a4
    b4 = min(min(max(ZERO, D(p["appliedEarlierYears"])), earlier), left)
    left -= b4
    c4 = min(max(ZERO, D(p["treatedAsCorpus"])), left)
    left -= c4
    d4 = min(d_amt, left)
    left -= d4
    e4 = left
    applied = min(sum(carry), max(ZERO, d_amt - d4 - c4))
    need = applied
    for k in range(5):
        take = min(carry[k], need)
        carry[k] -= take
        need -= take
    current = c4 + e4
    need7 = max(ZERO, D(p["corpusElection"]))
    if line7_first_from_this_year:
        take = min(current, need7)
        current -= take
        need7 -= take
    for k in range(5):
        take = min(carry[k], need7)
        carry[k] -= take
        need7 -= take
    return a4, d4, e4, applied, carry[0]


def year_checks(p):
    """The lines that differ, in the form's order."""
    a4, d4, e4, applied, lapsed = lapse(p, False)
    _, _, _, _, lapsed_alt = lapse(p, True)
    off = []

    def chk(line, filed, recomputed, on=True):
        if on and abs(D(filed) - recomputed) > ONE:
            off.append(line)

    any3 = any(v is not None for v in p["carryIn"])
    any10 = any(v is not None for v in p["carryOutBy"])
    app = [p["appliedPriorYear"], p["appliedEarlierYears"], p["treatedAsCorpus"], p["appliedCurrentYear"], p["remainingToCorpus"]]
    reported = any(v is not None for v in app) or D(p["qualifying"]) == 0
    five = p["carryApplied"] if p["carryApplied"] is not None else p["carryAppliedCorpus"]
    five_corpus = p["carryAppliedCorpus"] if p["carryAppliedCorpus"] is not None else p["carryApplied"]
    chk("XII-3f", p["carryInTotal"], sum(D(v) for v in p["carryIn"]), any3)
    chk("XII-4", p["qualifying"], sum(D(v) for v in app), reported)
    chk("XII-4a", p["appliedPriorYear"], a4, reported)
    chk("XII-4d", p["appliedCurrentYear"], d4, reported)
    chk("XII-4e", p["remainingToCorpus"], e4, reported)
    chk("XII-5", five, applied)
    chk("XII-6a", p["corpus"], D(p["carryInTotal"]) + D(p["treatedAsCorpus"]) + D(p["remainingToCorpus"]) - D(five_corpus))
    chk("XII-6b", p["earlierYearsRemaining"], D(p["undistributedEarlierYears"]) - D(p["appliedEarlierYears"]))
    chk("XII-6d", p["earlierYearsTaxable"], D(p["earlierYearsRemaining"]) - D(p["deficiencyOrTaxed"]))
    chk("XII-6e", p["priorYearRemaining"], D(p["undistributedPriorYear"]) - D(p["appliedPriorYear"]))
    chk("XII-6f", p["undistributedCurrentYear"], D(p["distributable"]) - D(p["appliedCurrentYear"]) - D(five))
    if vintage_known(p) and abs(D(p["carryLapsed"]) - lapsed) > ONE and abs(D(p["carryLapsed"]) - lapsed_alt) > ONE:
        off.append("XII-8")
    chk("XII-9", p["carryOut"], D(p["corpus"]) - D(p["corpusElection"]) - D(p["carryLapsed"]))
    chk("XII-10", p["carryOut"], sum(D(v) for v in p["carryOutBy"]), any10)
    return off


def year_result(r):
    if r["operating"] or r["box"]:
        return "not applicable", []
    p = r["part12"]
    if p is None or (p["distributable"] is None and p["qualifying"] is None):
        return "not reported", []
    off = year_checks(p)
    return ("differs" if off else "agrees"), off


def link(prev, cur, prev_result, cur_result):
    if cur["begin"] is None:
        return "not checked", []
    if date.fromisoformat(cur["begin"]) != date.fromisoformat(prev["periodEnd"]) + timedelta(days=1):
        return "not checked", []
    bal = []
    for key in ("assets", "liabilities", "net"):
        closed, opened = prev["balance"][key][1], cur["balance"][key][0]
        bal.append("not reported" if closed is None or opened is None else ("agrees" if abs(D(closed) - D(opened)) <= ONE else "differs"))
    measured = lambda res: res in ("agrees", "differs")
    if not measured(prev_result) or not measured(cur_result):
        if "differs" in bal:
            return "differs", []
        return ("not checked" if all(b == "not reported" for b in bal) else "agrees"), []
    a, b = prev["part12"], cur["part12"]
    off = []
    if abs(D(b["undistributedPriorYear"]) - D(a["undistributedCurrentYear"])) > ONE:
        off.append("2a")
    if abs(D(b["carryInTotal"]) - D(a["carryOut"])) > ONE:
        off.append("3f")
    if vintage_known(b) and abs(sum(D(v) for v in a["carryOutBy"]) - D(a["carryOut"])) <= ONE:
        for k in range(5):
            if (D(a["carryOutBy"][k]) > 0 or D(b["carryIn"][k]) > 0) and abs(D(b["carryIn"][k]) - D(a["carryOutBy"][k])) > ONE:
                off.append("3" + "abcde"[k])
    earlier = D(a["earlierYearsTaxable"]) + D(a["priorYearRemaining"])
    if (earlier > 0 or D(b["undistributedEarlierYears"]) > 0) and abs(D(b["undistributedEarlierYears"]) - earlier) > ONE:
        off.append("2b")
    return ("differs" if off or "differs" in bal else "agrees"), off


def plain(d):
    return "0" if d == 0 else format(d.normalize(), "f")



def chain_of(returns):
    """The returns (any order) -> what the chain states about them: year and link results, differing lines, standing."""
    returns = sorted(returns, key=lambda r: r["periodEnd"])
    results = [year_result(r) for r in returns]
    links = [link(returns[i], returns[i + 1], results[i][0], results[i + 1][0]) for i in range(len(returns) - 1)]
    last = returns[-1]
    measured = results[-1][0] in ("agrees", "differs")
    final = last["part12"]["carryOutBy"][0] if measured and D(last["part12"]["carryOutBy"][0]) > 0 else None
    return returns, {
        "years": [res for res, _ in results],
        "links": [res for res, _ in links],
        "yearDiffers": {r["periodEnd"]: off for r, (_, off) in zip(returns, results) if off},
        "linkDiffers": {returns[i + 1]["periodEnd"]: off for i, (_, off) in enumerate(links) if off},
        "carryOut": plain(D(last["part12"]["carryOut"])) if measured else None,
        "finalYear": plain(D(final)) if final is not None else None,
    }


def sources_digest(returns):
    return hashlib.sha256("\n".join(f"{r['objectId']}:{r['sha256']}" for r in returns).encode()).hexdigest()


def money_fmt(v):
    return "-" if v is None else f"${int(Decimal(v)):,}"


def print_chain(returns, c):
    print(f"EIN {returns[0]['ein']}: {len(returns)} filed year(s)")
    print(f"  {'year ended':<12}{'distributable':>16}{'distributed':>16}{'applied (5)':>14}{'undistrib (6f)':>16}{'lapsed (8)':>14}{'carried (9)':>16}  result")
    for r, res in zip(returns, c["years"]):
        p = r["part12"] or {}
        five = p.get("carryApplied") if p.get("carryApplied") is not None else p.get("carryAppliedCorpus")
        print(f"  {r['periodEnd']:<12}{money_fmt(p.get('distributable')):>16}{money_fmt(p.get('qualifying')):>16}{money_fmt(five):>14}{money_fmt(p.get('undistributedCurrentYear')):>16}{money_fmt(p.get('carryLapsed')):>14}{money_fmt(p.get('carryOut')):>16}  {res}" + (f" ({', '.join(c['yearDiffers'][r['periodEnd']])})" if r["periodEnd"] in c["yearDiffers"] else ""))
    for i, res in enumerate(c["links"]):
        to = returns[i + 1]["periodEnd"]
        print(f"  {returns[i]['periodEnd']} -> {to}: {res}" + (f" (opening lines {', '.join(c['linkDiffers'][to])})" if to in c["linkDiffers"] else ""))
    if c["carryOut"] is not None:
        print(f"  carried out of the latest return: {money_fmt(c['carryOut'])}" + (f"; in its final year: {money_fmt(c['finalYear'])}" if c["finalYear"] else ""))


def load_dir(folder):
    """One return a tax period: where a period was filed more than once, the latest submission (the greatest OBJECT_ID) stands."""
    by_period = {}
    for f in sorted(os.listdir(folder)):
        if f.lower().endswith(".xml"):
            r = read(os.path.join(folder, f), re.sub(r"(_public)?\.xml$", "", f, flags=re.I))
            cur = by_period.get(r["periodEnd"])
            if cur is None or r["objectId"] > cur["objectId"]:
                if cur is not None:
                    print(f"  {r['periodEnd']}: {cur['objectId']} was filed again as {r['objectId']}; the later one stands", file=sys.stderr)
                by_period[r["periodEnd"]] = r
    return list(by_period.values())


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("proof", nargs="?", help="64-hex chain hash or a /proof/chain/ link")
    ap.add_argument("--read", metavar="DIR", help="no proof: read a folder of 990-PF XML files as a chain and print it")
    ap.add_argument("--returns", metavar="DIR", help="copies of the returns, named <OBJECT_ID>.xml")
    ap.add_argument("--fetch", action="store_true", help="fetch each return yourself from irs.gov (one Range request each)")
    ap.add_argument("--body", help="a saved body JSON (as /api/proof/body serves it) instead of fetching it")
    ap.add_argument("--offline", action="store_true", help="skip the ledger; check body, sources and years only")
    ap.add_argument("--site", default=SITE, help="where the body and the ledger are asked for (default: %(default)s)")
    a = ap.parse_args()

    if a.read:
        returns, c = chain_of(load_dir(a.read))
        print_chain(returns, c)
        return 0
    if not a.proof:
        ap.error("give a chain hash, or --read DIR")

    sys.path.insert(0, str(Path(__file__).resolve().parent))
    from verify import content_sha256  # the canonical form, one definition shared by every verifier

    def get_json(url):
        import urllib.request
        with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "verify_chain.py"}), timeout=30) as r:
            return json.loads(r.read().decode())

    m = re.search(r"[0-9a-f]{64}", a.proof.strip().lower())
    if not m:
        sys.exit("not a chain hash")
    h = m.group(0)
    served = json.loads(Path(a.body).read_text()) if a.body else get_json(f"{a.site}/api/proof/body?hash={h}")
    body = served["body"] if "body" in served else served
    ok = True
    skipped = []

    got = content_sha256(body)
    print(f"1 body     {'OK ' if got == h else 'FAIL'}  sha256(canonical(body)) = {got}")
    ok &= got == h
    if body.get("kind") != "payout-chain-990pf":
        sys.exit(f"this is a {body.get('kind')} proof, not a carryover chain; use verify_proof.py")

    if a.offline:
        print("2 ledger   --   skipped (--offline)")
        skipped.append("ledger")
    else:
        try:
            led = get_json(f"{a.site}/api/proof/verify?hash={h}")
            print(f"2 ledger   {'OK ' if led.get('known') else 'FAIL'}  on the ledger: {led.get('known')}, kind: {led.get('kind')}, first recorded {led.get('firstSeen')}")
            ok &= bool(led.get("known"))
        except Exception as e:  # the ledger not answering is not the chain being wrong
            print(f"2 ledger   --   not reached ({e})")
            skipped.append("ledger")

    years = body["years"]
    raws = {}
    if a.returns:
        for y in years:
            for name in (f"{y['source']['objectId']}.xml", f"{y['source']['objectId']}_public.xml"):
                p = os.path.join(a.returns, name)
                if os.path.exists(p):
                    raws[y["source"]["objectId"]] = open(p, "rb").read()
    elif a.fetch:
        from verify_proof import fetch_return  # the Range read and the Deflate64 inflater, beside this file
        for loc in served.get("locators", []):
            raws[loc["objectId"]] = fetch_return(loc)
    if len(raws) < len(years):
        print(f"3 sources  --   {len(raws)} of {len(years)} returns at hand; pass --fetch or --returns DIR to check the sources and the years")
        skipped += ["sources", "years"]
    else:
        returns = [read_bytes(raws[y["source"]["objectId"]], y["source"]["objectId"]) for y in years]
        same = all(r["sha256"] == y["source"]["sourceSha256"] for r, y in zip(returns, years))
        digest = sources_digest(returns)
        src_ok = same and digest == body["source"]["sourcesSha256"] and all(r["ein"] == body["filer"]["ein"] for r in returns)
        print(f"3 sources  {'OK ' if src_ok else 'FAIL'}  {len(returns)} returns, each byte-identical to the one sealed; digest {digest[:16]}...")
        ok &= src_ok

        returns, c = chain_of(returns)
        mine = {
            "years": [y["result"] for y in years],
            "links": [l["result"] for l in body["links"]],
            "yearDiffers": {y["period"]["end"]: [k["line"] for k in y["checks"] if k["verdict"] == "differs"] for y in years if any(k["verdict"] == "differs" for k in y["checks"])},
            "linkDiffers": {l["to"]: [k["line"] for k in l["carry"] if k["verdict"] == "differs"] for l in body["links"] if any(k["verdict"] == "differs" for k in l["carry"])},
            "carryOut": body["standing"]["carryOut"]["total"] if body.get("standing") else None,
            "finalYear": (body["standing"]["finalYear"] or {}).get("amount") if body.get("standing") else None,
        }
        # A chain whose latest year carries no Part XII has no standing; the reader above then states none either.
        lines_ok = all((y["part12"] is None) or all(json.dumps(y["part12"].get(k)) == json.dumps(v) for k, v in (r["part12"] or {}).items()) for y, r in zip(years, returns))
        math_ok = lines_ok and all(mine[k] == c[k] for k in ("years", "links", "yearDiffers", "linkDiffers")) and (mine["carryOut"] is None or (mine["carryOut"] == c["carryOut"] and mine["finalYear"] == c["finalYear"]))
        print(f"4 years    {'OK ' if math_ok else 'FAIL'}  every Part XII line read again; {len(c['years'])} year results and {len(c['links'])} link results re-derived")
        if not math_ok:
            rules = (body.get("build") or {}).get("digest")
            print(f"           sealed under rules {rules or 'unnamed'}, re-derived under verify_chain.py rules of 2026-09-30: a chain sealed under other rules is superseded, not false")
            for k in ("years", "links", "yearDiffers", "linkDiffers", "carryOut", "finalYear"):
                if mine[k] != c[k]:
                    print(f"           {k}: sealed {mine[k]} / re-derived {c[k]}")
        ok &= math_ok
        print()
        print_chain(returns, c)

    print()
    if ok and not skipped:
        print("VERIFIED: the body, the ledger, every return's bytes and every year re-derived.")
    elif ok:
        print(f"VERIFIED as far as checked; not checked: {', '.join(skipped)}.")
    else:
        print("NOT VERIFIED: see the FAIL line(s) above.")
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())