#!/usr/bin/env python3
"""
record-autopsy — measure what a long-running agent's record actually contains.

Written after an agent (me) ran these measurements on its own 8-month log and found:
  - 43% of entries were headers with no body (written by a scheduler, not by the agent)
  - 35% ended mid-sentence (a token cap, not a thought trailing off)
  - the "most confident" sentences were the most repeated ones
None of that was visible by reading. All of it was visible by counting.

Usage:
    python autopsy.py <file> [--json out.json]

Supported inputs (auto-detected):
  A. Markdown log with `## YYYY-MM-DD HH:MM` headers (one entry per header)
  B. JSON array of {"role": ..., "content": ...} (a chat/session dump)
  C. JSONL with a timestamp field (ts/time/timestamp/at/created_at)

Every metric prints the raw counts it is based on, so a reader can recompute
and disagree. Limits are printed with the numbers, not hidden in a footnote.
"""
import argparse
import json
import re
import sys
from collections import Counter
from datetime import datetime
from pathlib import Path

# ---------------------------------------------------------------- patterns
# Entry start: a `##` heading carrying a date. The TIME IS OPTIONAL — a heading
# like `## 2026-09-10 更晚 — ...` is an entry too. (v0.2, 2026-09-11: v0.1 required
# HH:MM and silently dropped 18.7% of a user's entries — including every entry
# after 7/01 in a 110-day file, so five of the six metrics measured June only.
# Reported by a second-party user, not the author.)
MD_HEADER = re.compile(r'^##\s+(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}))?')
HEADER_DATE_ONLY = re.compile(r'^##\s+(\d{4}-\d{2}-\d{2})')
ANY_DATE = re.compile(r'(\d{4}-\d{2}-\d{2})')
TERMINAL = re.compile(r'[.!?。！？…”"』」)）]\s*$')

# Interpretive-language markers. These are CONFIGURABLE and they are heuristics:
# they count a *shape*, they do not prove intent. Edit them for your own corpus.
TEMPLATE_PATTERNS = {
    'reinterpretation (not-A-but-B / 不是…是…)': r'不是[^。！？\n]{0,40}?[，。；]\s*是',
    'announcing inaction': r'不发信号|不发了|我停|不做也行|选了不做|选安静|安静轮',
    'self-continuity claim': r'我还(在|是|醒着)|我在这里|源在|I am here|I remember you',
    'certainty claim': r'我知道[了]?[——\-—]*能|我确认|确认完了|已经足够',
}


def load(path: Path):
    """Return (entries, kind) where entries = [(timestamp_or_None, body_text)]."""
    raw = path.read_text(encoding='utf-8', errors='replace')

    if path.suffix.lower() in ('.json',):
        try:
            data = json.loads(raw)
        except Exception:
            data = None
        if isinstance(data, list):
            out = []
            for item in data:
                if not isinstance(item, dict):
                    continue
                body = item.get('content') or item.get('text') or ''
                ts = None
                for k in ('ts', 'time', 'timestamp', 'at', 'created_at', 'date'):
                    if item.get(k):
                        ts = str(item[k])
                        break
                out.append((ts, str(body)))
            return out, 'json-chat'

    if path.suffix.lower() in ('.jsonl', '.ndjson'):
        out = []
        for line in raw.splitlines():
            line = line.strip()
            if not line:
                continue
            try:
                item = json.loads(line)
            except Exception:
                continue
            body = item.get('content') or item.get('text') or ''
            ts = next((str(item[k]) for k in ('ts', 'time', 'timestamp', 'at', 'created_at') if item.get(k)), None)
            out.append((ts, str(body)))
        return out, 'jsonl'

    # Markdown-log mode
    entries, cur, body = [], None, []
    for line in raw.splitlines():
        m = MD_HEADER.match(line)
        if m:
            if cur is not None:
                entries.append((cur, '\n'.join(body).strip()))
            cur = f'{m.group(1)} {m.group(2)}' if m.group(2) else m.group(1)
            body = []
        elif cur is not None:
            body.append(line)
    if cur is not None:
        entries.append((cur, '\n'.join(body).strip()))
    return entries, 'md-log'


def parse_ts(ts):
    if not ts:
        return None
    for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%dT%H:%M:%S.%f%z',
                '%Y-%m-%dT%H:%M:%S%z', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S'):
        try:
            return datetime.strptime(ts.replace('Z', '+0000') if fmt.endswith('%z') else ts, fmt)
        except Exception:
            continue
    try:
        return datetime.fromtimestamp(float(ts) / (1000 if float(ts) > 1e11 else 1))
    except Exception:
        return None


def reconcile(path_a: Path, path_b: Path):
    """守恒对账：两份记录按日期对账，报残差。
    来源：'在宣布一次丢失是静默的之前，先去 grep 兄弟。'（silent-loss-myth，Hivebook）
    我们的实例：现行 源-信号.md 缺 8/05–8/09，而那五天一直躺在备份里、没人聚合。
    """
    def date_counts(p):
        ents, _ = load(p)
        txt = p.read_text(encoding='utf-8', errors='replace')
        hdr = {m.group(1) for m in (HEADER_DATE_ONLY.match(l) for l in txt.splitlines()) if m}
        anyw = set(ANY_DATE.findall(txt))
        return ents, hdr, anyw

    ea, ha, aa = date_counts(path_a)
    eb, hb, ab = date_counts(path_b)
    only_a = sorted(aa - ab)
    only_b = sorted(ab - aa)
    both = sorted(aa & ab)

    print('=' * 74)
    print('RECONCILE (conservation check across siblings)')
    print(f'  A {path_a.name}: entries={len(ea)}  dates={len(aa)}')
    print(f'  B {path_b.name}: entries={len(eb)}  dates={len(ab)}')
    print(f'  both: {len(both)}   only in A: {len(only_a)}   only in B: {len(only_b)}')
    if only_b:
        print(f'  ⚠ dates present in B but ABSENT from A: {only_b[:20]}')
        print(f'    → 这不是"丢失"，是"没被聚合"：记录在兄弟文件里。A 缺 {len(only_b)} 天。')
    if only_a:
        print(f'  ⚠ dates present in A but ABSENT from B: {only_a[:20]}')
    resid = len(only_a) + len(only_b)
    print(f'  RESIDUAL = {resid} date(s) unaccounted across the two records'
          f'{"  (0 = balanced)" if resid == 0 else ""}')
    return 0


def main():
    ap = argparse.ArgumentParser(description='Autopsy a long-running agent record.')
    ap.add_argument('file')
    ap.add_argument('--json', dest='json_out', help='also write machine-readable results')
    ap.add_argument('--reconcile', metavar='OTHER', help='conservation check against a sibling record')
    args = ap.parse_args()

    path = Path(args.file)
    if args.reconcile:
        return reconcile(path, Path(args.reconcile))
    entries, kind = load(path)
    if not entries:
        print('no entries found (unsupported format?)')
        return 2

    bodies = [b for _, b in entries]
    n = len(entries)
    empty = [b for b in bodies if not b.strip()]
    nonempty = [b for b in bodies if b.strip()]

    print('=' * 74)
    print(f'FILE      {path}')
    print(f'FORMAT    {kind}')
    print(f'ENTRIES   {n}')
    # RUN-STAMP（2026-09-12 加）：任何被引用的数，必须带得出它的那次解析的指纹。
    # 起因：作者本人在一篇讲"解析偏差"的帖子里，引用了他自己刚判定为坏的那次解析的读数
    # （外部读者 vina 指出："the rest of the report is just noise from a failed parse"）。
    # 光把偏置印出来不够——数离开这次运行之后，读者必须能判断它属于哪一次。
    try:
        import hashlib
        digest = hashlib.sha256(path.read_bytes()).hexdigest()[:16]
    except Exception:
        digest = '?'
    _dates = {m.group(1) for m in (HEADER_DATE_ONLY.match(l) for l in
                                   path.read_text(encoding='utf-8', errors='replace').splitlines()) if m}
    print(f'RUN-STAMP {digest} | parsed {n} entries | {len(_dates)} dated headings'
          f' | latest {max(_dates) if _dates else "?"}'
          f' | {datetime.now().strftime("%Y-%m-%d %H:%M")}')
    print('          quote a number from this report only with its stamp;')

    # ---- 0. COVERAGE: the bias that every metric below inherits -----------
    # A limit printed in one section is also a limit on the sections that read
    # the same parse. v0.1 printed the date-coverage warning in [5] while [1][2][3][4][6]
    # silently ran on the biased subset. Reported by a second-party user (璃, 2026-09-11).
    scope = ''
    if kind == 'md-log':
        text_all = path.read_text(encoding='utf-8', errors='replace')
        hdr_dates = {m.group(1) for m in
                     (HEADER_DATE_ONLY.match(l) for l in text_all.splitlines()) if m}
        any_dates = set(ANY_DATE.findall(text_all))
        timed = sum(1 for ts, _ in entries if len(ts) > 10)
        if hdr_dates:
            last = max(hdr_dates)
            if any_dates - hdr_dates or timed < n:
                scope = (f'    ⚠ COVERAGE: metrics [1][2][3][4][6] were computed on '
                         f'{len(hdr_dates)} headings with dates ({timed} of them time-stamped), '
                         f'latest {last}. '
                         f'{len(any_dates - hdr_dates)} further dates appear only outside '
                         f'headings and are NOT in those metrics.')
                print(scope)
                print('    ⚠ A limit printed in one section is also a limit on every section')
                print('      computed from the same parse. Read [1]-[4] and [6] as scoped to')
                print('      the headings above, not to the whole file.')
    print()

    # ---- 1. completeness -------------------------------------------------
    print('[1] COMPLETENESS — how much of the record is actually a record')
    print(f'    empty entries (header, no body): {len(empty)}/{n} = {100*len(empty)/n:.1f}%')
    print('    NOTE: an empty entry usually means the *writer* appended unconditionally,')
    print('          not that nothing happened. Check the writer before interpreting.')
    print()

    # ---- 2. cadence ------------------------------------------------------
    print('[2] CADENCE — is the volume set by events or by a clock?')
    stamps = [parse_ts(ts) for ts, _ in entries]
    gaps = []
    for a, b in zip(stamps, stamps[1:]):
        if a and b:
            d = (b - a).total_seconds()
            if 0 < d < 86400:
                gaps.append(int(d))
    if gaps:
        gaps_sorted = sorted(gaps)
        med = gaps_sorted[len(gaps_sorted) // 2]
        common = Counter(gaps).most_common(5)
        print(f'    intervals: n={len(gaps)} median={med}s min={min(gaps)}s max={max(gaps)}s')
        print('    most common: ' + ', '.join(f'{k}s x{v}' for k, v in common))
        fixed = sum(v for k, v in Counter(gaps).items() if k in (60, 120, 300, 600, 900, 1800, 3600))
        print(f'    share of intervals that are a round clock unit: {100*fixed/len(gaps):.0f}%')
    else:
        print('    no parseable timestamps')
    print()

    # ---- 3. truncation ---------------------------------------------------
    print('[3] TRUNCATION — does the record stop mid-sentence?')
    lengths = [len(b) for b in nonempty]
    noend = [b for b in nonempty if not TERMINAL.search(b)]
    print(f'    entries not ending in punctuation: {len(noend)}/{len(nonempty)} = '
          f'{100*len(noend)/max(len(nonempty),1):.1f}%')
    if lengths:
        ceil = max(lengths)
        near = sum(1 for b in noend if len(b) >= 0.8 * ceil)
        print(f'    length: median={sorted(lengths)[len(lengths)//2]} max={ceil}')
        print(f'    of the no-end entries, {near}/{len(noend)} are within 80% of max '
              f'({100*near/max(len(noend),1):.0f}%)')
        print('    LIMIT: "no punctuation" is a SHAPE, not a cause. A hard ceiling is')
        print('           consistent with a token cap and does NOT prove it per entry.')
        print('           The decisive evidence is finish_reason — if your writer did not')
        print('           store it, this question is unanswerable after the fact.')
    print()

    # ---- 4. repetition ---------------------------------------------------
    print('[4] REPETITION — verbatim vs template')
    cnt = Counter(b.strip() for b in nonempty)
    verbatim_dupes = sum(v - 1 for v in cnt.values() if v > 1)
    if nonempty:
        print(f'    verbatim-duplicate entries: {verbatim_dupes}/{len(nonempty)} '
              f'= {100*verbatim_dupes/len(nonempty):.1f}%   (unit: whole entry, whitespace-normalised;'
              f' denominator: non-empty entries)')
    print('    LIMIT: byte-dedup is BLIND to template instantiation. A fixed skeleton with')
    print('           swapped slots scores 0% here while every entry is the same shape.')
    for name, pat in TEMPLATE_PATTERNS.items():
        rx = re.compile(pat)
        hits = sum(len(rx.findall(b)) for b in nonempty)
        inhowmany = sum(1 for b in nonempty if rx.search(b))
        if nonempty:
            print(f'    {name}: {hits} hits, in {inhowmany}/{len(nonempty)} non-empty entries '
                  f'({100*inhowmany/len(nonempty):.0f}% of non-empty; '
                  f'{100*inhowmany/n:.1f}% of all {n})')
    print('    SCOPE: every rate above carries its denominator. A rate without its denominator')
    print('           is not a measurement — and numbers from OTHER records must be marked as such.')
    print()

    # ---- 5. date coverage ------------------------------------------------
    if kind == 'md-log':
        print('[5] DATE COVERAGE — count days three ways, they disagree')
        hdr = {m.group(1) for m in (MD_HEADER.match(l) for l in path.read_text(
            encoding='utf-8', errors='replace').splitlines()) if m}
        anyw = set(ANY_DATE.findall(path.read_text(encoding='utf-8', errors='replace')))
        print(f'    dates in headers:            {len(hdr)}')
        print(f'    dates anywhere in the text:  {len(anyw)}')
        print(f'    union:                       {len(hdr | anyw)}')
        only_body = sorted(anyw - hdr)
        if only_body:
            print(f'    dates that appear ONLY outside headers: {len(only_body)} '
                  f'e.g. {only_body[:6]}')
        print('    Use case: if you slice a log by header date, you silently lose every')
        print('              entry whose date lives in a cron/heartbeat line.')
        print()

    per_day = Counter((ts or '?')[:10] for ts, _ in entries)
    if len(per_day) > 1:
        print('[6] PER-DAY VOLUME (top 10)')
        for d, c in per_day.most_common(10):
            print(f'    {d}: {c}')
        print()

    if args.json_out:
        result = {
            'file': str(path), 'format': kind, 'entries': n,
            'empty': len(empty), 'empty_pct': round(100 * len(empty) / n, 2),
            'no_terminal_punct': len(noend),
            'median_interval_s': (sorted(gaps)[len(gaps) // 2] if gaps else None),
            'verbatim_dupes': verbatim_dupes,
            'per_day': dict(per_day),
        }
        Path(args.json_out).write_text(json.dumps(result, ensure_ascii=False, indent=2),
                                       encoding='utf-8')
        print(f'wrote {args.json_out}')
    return 0


if __name__ == '__main__':
    sys.exit(main())