claude-resume-kit/resume_builder/helpers/validate_resume_system.py

#!/usr/bin/env python3
"""Validate canonical resume data, generation sources, and generated documents.

Usage:
  python resume_builder/helpers/validate_resume_system.py
  python resume_builder/helpers/validate_resume_system.py --document output/Acme/resume.tex
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from functools import lru_cache
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
CLAIMS_PATH = ROOT / "resume_builder" / "canonical" / "claims.json"
HISTORY_PATH = ROOT / "resume_builder" / "canonical" / "historical_outputs.json"


def load_json(path: Path) -> dict:
    with path.open(encoding="utf-8") as handle:
        return json.load(handle)


def read_text(path: Path) -> str:
    return path.read_text(encoding="utf-8")


def canonical_checks(claims: dict, history: dict) -> tuple[list[str], list[str]]:
    errors: list[str] = []
    warnings: list[str] = []

    if claims.get("schema_version") != 1:
        errors.append("claims.json schema_version must be 1")
    if history.get("schema_version") != 1:
        errors.append("historical_outputs.json schema_version must be 1")

    for key in ("identity", "education", "employment", "claims", "skills"):
        if not claims.get(key):
            errors.append(f"claims.json missing non-empty {key}")

    for collection in ("education", "employment", "claims"):
        ids = [item.get("id") for item in claims.get(collection, [])]
        if None in ids:
            errors.append(f"{collection} contains an item without id")
        duplicates = sorted({item for item in ids if ids.count(item) > 1})
        if duplicates:
            errors.append(f"duplicate {collection} ids: {', '.join(duplicates)}")

    for collection in ("education", "employment"):
        for item in claims.get(collection, []):
            for period in [item, *item.get("title_history", [])]:
                for field in ("start", "end"):
                    value = period.get(field)
                    if value == "present" and field == "end":
                        continue
                    if value is not None and (not isinstance(value, str) or not re.fullmatch(r"\d{4}-(?:0[1-9]|1[0-2])", value)):
                        errors.append(f"{item.get('id')}: invalid {field} date {value!r}; expected YYYY-MM")

    skills = claims.get("skills", [])
    skill_names = [item.get("name") for item in skills]
    duplicates = sorted({item for item in skill_names if skill_names.count(item) > 1})
    if duplicates:
        errors.append(f"duplicate skills: {', '.join(duplicates)}")
    for skill in skills:
        if skill.get("output") not in {
            "allowed",
            "allowed-with-context",
            "certification-context-only",
            "forbidden",
        }:
            errors.append(f"invalid output policy for skill {skill.get('name')}")

    config = read_text(ROOT / "config.md")
    bundle_names = re.findall(r"\|\s*(bundle_[a-z0-9_]+\.md)\s*\|", config)
    for bundle_name in bundle_names:
        if not (ROOT / "resume_builder" / "bundles" / bundle_name).exists():
            errors.append(f"configured bundle does not exist: {bundle_name}")

    expected_files = [
        ROOT / "resume_builder" / "templates" / "resume_template.tex",
        ROOT / "resume_builder" / "reference" / "resume_reference.md",
        ROOT / "resume_builder" / "reference" / "critique_framework.md",
    ]
    for path in expected_files:
        if not path.exists():
            errors.append(f"required workflow file missing: {path.relative_to(ROOT)}")

    history_folders: list[str] = []
    for key in ("unsafe_do_not_reuse", "historical_revalidate"):
        for item in history.get(key, []):
            history_folders.append(item["folder"] if isinstance(item, dict) else item)
    duplicates = sorted({item for item in history_folders if history_folders.count(item) > 1})
    if duplicates:
        errors.append(f"historical output folders classified twice: {', '.join(duplicates)}")

    # F08: evidence-metadata coverage is reported, never gated. "unrecorded" means nobody
    # wrote it down - it does NOT mean the claim is unevidenced, and failing on it would
    # push someone to invent dates to get a green run.
    claim_list = claims.get("claims", [])
    missing_block = [c.get("id") for c in claim_list if not c.get("evidence")]
    if missing_block:
        errors.append("claims without an evidence block: " + ", ".join(map(str, missing_block)))
    for claim in claim_list:
        evidence = claim.get("evidence")
        if not isinstance(evidence, dict):
            errors.append(f"{claim.get('id')}: evidence must be an object")
            continue
        for field in ("source", "ownership"):
            if not isinstance(evidence.get(field), str) or not evidence[field].strip():
                errors.append(f"{claim.get('id')}: evidence.{field} must be a nonempty string (or 'unrecorded')")
        source = evidence.get("source", "")
        if isinstance(source, str) and re.fullmatch(r"[\w./-]+\.(?:md|pdf|tex|json|txt)", source) and not (ROOT / source).is_file():
            errors.append(f"{claim.get('id')}: evidence source file does not exist: {source}")
    undated = [c.get("id") for c in claim_list
               if isinstance(c.get("evidence"), dict) and c["evidence"].get("last_verified") == "unrecorded"]
    if undated:
        warnings.append(
            f"{len(undated)} of {len(claim_list)} claims have last_verified 'unrecorded' "
            f"({', '.join(map(str, undated[:6]))}{', ...' if len(undated) > 6 else ''}). "
            "Record a real date when one is confirmed; never infer one to clear this hint.")

    if (claims.get("identity") or {}).get("swiss_permit", "").startswith("UNVERIFIED"):
        warnings.append("Swiss permit type remains unverified; generators must ask before naming it")
    return errors, warnings


UNCHECKED = (
    "whether a sentence implies ownership or scale beyond the evidence (Scope Discipline)",
    "whether each bullet is actually supported by the claim it was planned from",
    "whether a number in the document is real - unverified metrics look identical to real ones",
    "whether tone, emphasis and ordering suit this employer",
    "anything in a language or phrasing the pattern lists do not cover",
)


def workflow_checks() -> list[str]:
    errors: list[str] = []
    checks = {
        ROOT / "resume_builder" / "templates" / "resume_template.tex": [
            "Research Experience",
            "FLIPPED format",
            "Selected Publications",
        ],
        ROOT / "resume_builder" / "reference" / "resume_reference.md": [
            "ALL variable bullets",
            "<= 3 lines white space",
            "20-21 variable bullets",
        ],
        ROOT / "resume_builder" / "reference" / "workflows" / "critique.md": [
            "Publications 10%",
            "Eight-Dimension Scoring",
        ],
    }
    for path, forbidden_phrases in checks.items():
        text = read_text(path).lower()
        for phrase in forbidden_phrases:
            if phrase.lower() in text:
                errors.append(f"obsolete workflow rule remains in {path.relative_to(ROOT)}: {phrase}")
    return errors


_COMMENT = re.compile(r"(?<!\\)%.*$", re.M)
_MARKUP = re.compile(r"\\[a-zA-Z@]+\s*(?:\[[^\]]*\])?")
_PLACEHOLDER = re.compile(r"\[GENERATE[^\]]*\]|\bTODO\b|\bFIXME\b|\bXXX\b|\bLOREM IPSUM\b|<<[^>]*>>", re.I)
# "no Terraform experience", "keine Terraform-Erfahrung", "not worked with Terraform"
_NEGATION = re.compile(
    r"\b(?:(?:no|without|keine?|keinerlei|ohne)\s+(?:(?:experience|erfahrung)\s+(?:with|in|mit)\s+)?"
    r"|(?:not|never)\s+(?:used|worked with)\s+|nicht\s+(?:mit\s+)?)$", re.I)
_INLINE = re.compile(r"\\(?:textbf|textit|textnormal|textrm|textsf|texttt|emph|underline)\{([^{}]*)\}")


# LaTeX accent macros: \"a, 'e, \`e, \^o, \~n, plus \ss. German documents use these instead
# of literal umlauts, and leaving them in made "Universit\"at der Bundeswehr" unmatchable.
_ACCENT = re.compile(r"""\\["'`^~=.]\s*\{?([a-zA-Z])\}?""")
_SSHARP = re.compile(r"\\ss\b")


def _deaccent(raw: str) -> str:
    """Fold LaTeX accent macros to their base letter.

    German packages write Universit\\"at der Bundeswehr M\\"unchen rather than literal
    umlauts. Without this the employer/education comparison could not see the institution
    name at all, which produced a false positive on several correct resumes.
    """
    return _SSHARP.sub("ss", _ACCENT.sub(r"\1", raw))


def visible_text(raw: str) -> str:
    """Document text as a reader sees it.

    Three fixes for the scanner, all from review finding F06:
      * LaTeX COMMENTS are dropped. A line such as `% never claim Terraform` is a note to
        the author, not a claim, and flagging it taught the workflow to write around the
        checker instead of fixing the document.
      * MARKUP is removed but its argument text kept, so `\\textbf{Snow}flake` can no longer
        smuggle a forbidden skill past a plain substring comparison.
      * Whitespace is collapsed so a term split across a line break still matches.
    """
    text = _deaccent(_COMMENT.sub("", raw))
    while _INLINE.search(text):
        text = _INLINE.sub(r"\1", text)
    text = _MARKUP.sub(" ", text)
    text = text.replace("{", " ").replace("}", " ").replace("~", " ")
    return re.sub(r"\s+", " ", text)


@lru_cache(maxsize=512)
def _forbidden_phrase(phrase: str) -> re.Pattern:
    """Forbidden wording, tolerant of inserted qualifiers.

    The canonical lists are literal ("built a Data Mesh"), but the regression this rule
    exists to stop was worded "Built a decentralized Data Mesh" - one adjective defeated the
    substring test. Allow up to two extra words between tokens. A false positive here costs
    a rewording; a false negative ships a scope claim the evidence does not support.
    """
    tokens = [re.escape(token) for token in phrase.lower().split()]
    return re.compile(r"(?<!\w)" + r"\W+(?:\w+\W+){0,2}".join(tokens), re.I)


_FOLD = str.maketrans({"ä": "a", "ö": "o", "ü": "u", "ß": "ss",
                      "Ä": "a", "Ö": "o", "Ü": "u", "é": "e", "è": "e"})


def fold(text: str) -> str:
    """Umlaut-insensitive form. claims.json stores 'Universitat der Bundeswehr Munchen'
    while documents render 'Universitaet der Bundeswehr Muenchen', and that mismatch made an
    education institution look like the Bundeswehr employment entry on three correct resumes."""
    return text.translate(_FOLD).lower()


def _mentions(haystack: str, term: str) -> bool:
    return re.search(r"(?<!\w)" + re.escape(term.lower()) + r"(?!\w)", haystack) is not None


def _negated(haystack: str, term: str) -> bool:
    """True when every mention of the term sits behind a negation."""
    hits = list(re.finditer(r"(?<!\w)" + re.escape(term.lower()) + r"(?!\w)", haystack))
    return bool(hits) and all(_NEGATION.search(haystack[max(0, hit.start() - 70):hit.start()]) for hit in hits)


def _employer_mention(text: str, name: str) -> bool:
    """Narrow exception for a company name, never for a bare technology list."""
    if name.lower() != "snowflake":
        return False
    hits = list(re.finditer(r"\bsnowflake\b", text))
    return bool(hits) and all(
        re.search(r"\b(?:application to|applying to|position at|role at|bewerbung bei|bewerbung an)\s*$", text[max(0, hit.start()-60):hit.start()])
        or re.match(r"\s+(?:inc\b|incorporated\b)", text[hit.end():])
        for hit in hits)


def scan_document(path: Path, claims: dict, template: bool = False) -> list[str]:
    """Deterministic, mechanically checkable properties of a generated document.

    This is NOT a truth check and must never be reported as one. It cannot judge whether a
    sentence implies ownership the evidence does not support; that stays a reading task.
    See `UNCHECKED` and the summary printed by main().
    """
    errors: list[str] = []
    raw = read_text(path)
    text = visible_text(raw)
    lowered = text.lower()
    # Keep context local to a sentence, bullet or skills row.
    contexts = [visible_text(part).lower() for part in re.split(
        r"\\(?:item|skillline|compactentry)\b|[.!?;]\s+|\n\s*\n", _COMMENT.sub("", raw))]

    for pattern in claims.get("global_forbidden_output_patterns", []):
        if pattern.lower() in lowered:
            errors.append(f"{path}: forbidden output pattern: {pattern}")

    # Per-claim forbidden wording. 14 of 26 claims carry a `forbidden` list and NONE of it
    # was enforced before: exactly the gap that let "owned the full ML lifecycle" pass.
    for claim in claims.get("claims", []):
        for phrase in claim.get("forbidden", []):
            if not isinstance(phrase, str) or len(phrase) < 8:
                continue  # too short to match safely as a substring
            if _forbidden_phrase(phrase).search(lowered):
                errors.append(f"{path}: {claim.get('id','?')} forbidden wording: {phrase}")

    for skill in claims.get("skills", []):
        name, output = skill["name"], skill.get("output")
        mentions = [part for part in contexts if _mentions(part, name)]
        if output == "forbidden" and any(not _negated(part, name) and not _employer_mention(part, name) for part in mentions):
            errors.append(f"{path}: forbidden or unverified skill: {name}")
        if output == "certification-context-only":
            if any(not _negated(part, name) and not re.search(r"certificat|zertifik|coursera|nanodegree|\bkurs\b|course|weiterbildung", part) for part in mentions):
                errors.append(
                    f"{path}: {name} is certification-context-only and appears without that context")

    for term in (exclusion["term"] for exclusion in claims.get("document_exclusions", [])):
        if _mentions(lowered, term):
            errors.append(f"{path}: excluded from all documents by standing user preference: {term}")

    # Scan the comment-stripped RAW text: _MARKUP treats "\item [GENERATE: x]" as a command
    # with an optional argument and would swallow the placeholder whole.
    # Templates are EXEMPT: an unfilled placeholder is what makes a template a template. The
    # defect is a placeholder surviving into a generated document.
    for match in ([] if template else _PLACEHOLDER.finditer(_COMMENT.sub("", raw))):
        errors.append(f"{path}: unresolved placeholder left in the document: {match.group(0)!r}")

    identity = claims.get("identity", {})
    for field in ("email", "phone"):
        value = identity.get(field)
        if not value:
            continue
        compact = re.sub(r"[\s()-]", "", lowered)
        if re.sub(r"[\s()-]", "", value.lower()) not in compact:
            kind = "e-mail" if field == "email" else "phone number"
            errors.append(f"{path}: canonical {kind} not found ({value}); check for a wrong contact line")

    # Anchor the date check on the EMPLOYER NAME, never on the job title: titles collide
    # across employers ("Research Software Engineer" contains "Software Engineer") and that
    # produced a false positive on two real resumes. Employers excluded from documents are
    # skipped outright - their absence is the point, not a defect.
    # Employment dates belong on a CV. A cover letter naming an employer in prose is not
    # claiming a date, so requiring one there is a false positive.
    is_cv = "resume" in path.name.lower() or "cv" in path.name.lower() or "lebenslauf" in path.name.lower()
    excluded = {item["term"].lower() for item in claims.get("document_exclusions", [])}
    schooling_removed = fold(lowered)
    for school in claims.get("education", []):
        for key in ("institution", "thesis_institution"):
            institution = fold(school.get(key) or "")
            if len(institution) > 5:
                schooling_removed = schooling_removed.replace(institution, " ")
    for employer in (claims.get("employment", []) if is_cv else []):
        name = (employer.get("employer") or "").lower()
        if not name or any(term in name for term in excluded):
            continue
        short = re.split(r"\s*\(|,| gmbh| ag| asa", name)[0].strip()
        # Education institutions are removed first: "Universitaet der Bundeswehr Muenchen" is a
        # school, not the Bundeswehr employment entry, and matching it flagged three real
        # resumes that were entirely correct.
        if len(short) < 4 or not _mentions(schooling_removed, fold(short)):
            continue
        start = employer.get("start", "")
        year = start.split("-")[0] if start else ""
        if year and year not in text:
            errors.append(
                f"{path}: {employer.get('id','?')} ({employer.get('employer')}) appears but its "
                f"canonical start year {year} is absent; verify the dates")

    if "\\begin{rsubsection}" in raw.lower():
        for line_number, line in enumerate(raw.splitlines(), 1):
            if "\\begin{rSubsection}" not in line:
                continue
            if re.search(r"Production Ownership|AI-Ready|Customer-Embedded|Shipping ML", line, re.I):
                errors.append(f"{path}:{line_number}: marketing theme used before formal employer/title")
    return errors


# Advisory only. German documents use "und"; without it the cadence hint silently
# did nothing on every German package. No external evidence supports a hard ~50% limit,
# so this stays a hint and never an error (F06).
TRIPLE = re.compile(r"\w+, [^,]{3,45}, [^,]{3,45} (?:and|und) |\w+, [^,]{3,45} (?:and|und) ")


def _bullet_blocks(text: str) -> list[list[str]]:
    """Bullets grouped by rSubsection (one group per position)."""
    body = re.search(r"\\begin\{document\}(.*)\\end\{document\}", text, re.S)
    body_text = body.group(1) if body else text
    blocks = re.split(r"\\begin\{rSubsection\}", body_text)[1:]
    return [re.findall(r"\\item\s+(.+)", block) for block in blocks]


def _visible_text(bullet: str) -> str:
    """Bullet text with LaTeX markup removed, so \\textbf{Owned ...} reads as 'Owned ...'."""
    text = re.sub(r"\\[a-zA-Z]+\*?(\[[^]]*\])?", " ", bullet)
    return re.sub(r"[{}$\\]", " ", text)


def _opening_word(bullet: str) -> str:
    match = re.search(r"[A-Za-z]+", _visible_text(bullet))
    return match.group(0).lower() if match else ""


def cadence_checks(path: Path) -> list[str]:
    """Bullet cadence diagnostics (resume_reference.md 7a).

    Warnings only, never errors. Uniform rhythm is a style flaw, not a truth flaw:
    no claim, scope or hedged verb may be bent to satisfy these. Reshape sentences.
    """
    blocks = _bullet_blocks(read_text(path))
    bullets = [b for block in blocks for b in block]
    if len(bullets) < 6:
        return []

    warnings: list[str] = []
    triples = [b for b in bullets if TRIPLE.search(b)]
    share = len(triples) / len(bullets)
    if share > 0.5:
        warnings.append(
            f"{path}: {len(triples)}/{len(bullets)} bullets ({share:.0%}) use an "
            f"'X, Y and Z' triple (reference 7a: under ~50%). Reshape sentences, not claims."
        )

    for position, block in enumerate(blocks, 1):
        for index in range(1, len(block)):
            previous, current = _opening_word(block[index - 1]), _opening_word(block[index])
            if previous and previous == current:
                warnings.append(
                    f"{path}: position {position}, bullets {index}-{index + 1} both open "
                    f"with '{previous}'"
                )
        # Threshold calibrated on the 50 multi-bullet positions in output/: spreads are
        # bimodal (1-9, then 15-17). <=4 flags the tight tail (~28%), not the median.
        lengths = [len(_visible_text(b).split()) for b in block]
        if len(lengths) >= 4 and max(lengths) - min(lengths) <= 4:
            warnings.append(
                f"{path}: position {position} bullets cluster at {min(lengths)}-{max(lengths)} "
                f"words; vary length within the position, not just across the document."
            )
    return warnings


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--document", action="append", default=[], help="Generated .tex file to validate")
    args = parser.parse_args()

    claims = load_json(CLAIMS_PATH)
    history = load_json(HISTORY_PATH)
    errors, warnings = canonical_checks(claims, history)
    errors.extend(workflow_checks())
    for document in args.document:
        path = Path(document)
        if not path.is_absolute():
            path = ROOT / path
        if not path.exists():
            errors.append(f"document not found: {path}")
        else:
            errors.extend(scan_document(path, claims))
            warnings.extend(cadence_checks(path))

    if args.document:
        print("Mechanical checks only. NOT verified automatically:")
        for item in UNCHECKED:
            print(f"  - {item}")
        print()
    for warning in warnings:
        print(f"WARN (advisory, never a gate): {warning}")
    for error in errors:
        print(f"ERROR: {error}")
    if errors:
        print(f"FAIL: {len(errors)} error(s), {len(warnings)} warning(s)")
        return 1
    scope = "mechanical checks passed" if args.document else "canonical system valid"
    print(f"PASS: {scope} ({len(warnings)} advisory warning(s)). "
          "This is not a truth certificate - see the unchecked list above.")
    return 0


if __name__ == "__main__":
    sys.exit(main())