from __future__ import annotations """Mutation gate for pdf_tasks, forged and verified 5/5 near-misses rejected. The checker in pdf_tasks.py could grade an output but nothing showed it would reject a broken one. These are the deliberate near-misses it refuses, each one verified to change the file and to be rejected. """ # ---- inlined from pdf_tasks.py so this file runs on its own ---- """Real-skill task bank for the anthropics `pdf` skill (Python: reportlab, pypdf), graded MECHANICALLY with pypdf and pdfplumber. Four contexts: form a fillable AcroForm with exact text field names and a checkbox (graded by pypdf get_fields) report a multi-page platypus report: exact heading per page, a table, footer "Page N of M" sci paragraphs with real sub/superscripts (H2O, x^2 as raised smaller glyphs) and PDF metadata assemble merge generated inputs, rotate one page, add an outline (bookmarks): pypdf manipulation Scripts are Python; the runner gives OUTPUT (path to write) and INPUT_DIR (assemble inputs). """ # (moved to the top of this file) import os import random import re import subprocess import sys import tempfile CONTEXTS = ["form", "report", "sci", "assemble"] FIELDS = ["full_name", "email", "phone", "company", "date_of_birth", "city", "employee_id", "department"] HEADINGS = ["Executive Summary", "Site Findings", "Risk Register", "Budget Outlook", "Staffing Plan", "Vendor Review", "Timeline", "Recommendations", "Appendix A", "Open Items"] AUTHORS = ["K. Clouthier", "Ops Team", "Safety Office", "Finance Desk", "Field Unit 3"] COMPOUNDS = [("H2O", "H2O", "water"), ("CO2", "CO2", "carbon dioxide"), ("CH4", "CH4", "methane"), ("NH3", "NH3", "ammonia"), ("O3", "O3", "ozone")] def build(n_per: int = 8, seed: int = 43) -> list[dict]: rng = random.Random(seed) specs: list[dict] = [] for i in range(n_per): fields = rng.sample(FIELDS, 3) cb = rng.choice(["agree_terms", "subscribe", "confirmed", "active"]) title = rng.choice(["Intake Form", "Access Request", "Onboarding Sheet", "Incident Report Form"]) specs.append({"context": "form", "id": f"fm{i}", "fields": fields, "checkbox": cb, "title": title, "spec": (f"Create a one-page fillable PDF form titled \"{title}\" (the title drawn as text at the top). " f"It must contain exactly these fillable TEXT fields, named exactly: {', '.join(fields)}, each " f"with a visible label next to it, and one fillable CHECKBOX field named exactly \"{cb}\". " f"The fields must be real AcroForm widgets a PDF reader can fill, not drawn boxes.")}) for i in range(n_per): k = rng.randint(3, 4) heads = rng.sample(HEADINGS, k) rows = [(rng.choice(["Alpha", "Bravo", "Charlie", "Delta", "Echo"]), rng.randint(10, 900)) for _ in range(rng.randint(3, 5))] specs.append({"context": "report", "id": f"rp{i}", "heads": heads, "rows": rows, "spec": (f"Create a {k}-page US Letter PDF report. Page p (1..{k}) must start with the heading " f"\"{heads[0]}\"" + "".join(f", page {j + 1} with \"{h}\"" for j, h in enumerate(heads[1:], start=1)) + f" (each heading as its own line of text at the top of its page, force page breaks between pages). " f"Page 1 must also contain a table with header row exactly Item, Count and these rows: " + "; ".join(f"{a}, {b}" for a, b in rows) + f". Every page must have a footer with exactly the text \"Page N of {k}\" where N is that page's " f"number (so page 2 shows \"Page 2 of {k}\").")}) for i in range(n_per): comp = rng.choice(COMPOUNDS) power = rng.randint(2, 5) author = rng.choice(AUTHORS) title = rng.choice(["Lab Notes", "Reaction Summary", "Field Chemistry", "Process Note"]) specs.append({"context": "sci", "id": f"sc{i}", "comp": comp, "power": power, "author": author, "title": title, "spec": (f"Create a one-page PDF with document metadata title exactly \"{title}\" and author exactly " f"\"{author}\". Its body has two paragraphs. Paragraph 1 must contain the formula {comp[0]} written " f"with a real subscript for the digit (the digit rendered smaller and lower than the letters, as in " f"reportlab's markup), followed by the words \"is {comp[2]}\". Paragraph 2 must contain " f"\"x{power}\" written as x to the power {power} with a real superscript (the digit smaller and " f"raised), followed by the words \"grows quickly\".")}) for i in range(n_per): n_in = rng.randint(2, 3) rot_page = rng.randint(1, n_in) names = [f"part{j + 1}.pdf" for j in range(n_in)] marks = rng.sample(HEADINGS, n_in) specs.append({"context": "assemble", "id": f"as{i}", "inputs": names, "rot_page": rot_page, "marks": marks, "spec": (f"In the directory given by the environment variable INPUT_DIR there are {n_in} one-page PDFs: " f"{', '.join(names)}. Merge them in that order into one PDF. Rotate page {rot_page} of the merged " f"document by 90 degrees clockwise (only that page). Add an outline (bookmarks): one top-level " f"bookmark per page, titled exactly " + ", ".join(f"\"{m}\" for page {j + 1}" for j, m in enumerate(marks)) + f". Keep every page's original text.")}) rng.shuffle(specs) return specs # ---------------------------------------------------------------- run the model's script FENCE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.S) def extract_script(reply: str) -> str | None: m = FENCE.findall(reply or "") if m: return max(m, key=len) if "import" in (reply or "") and ("reportlab" in reply or "pypdf" in reply): return reply return None def _make_inputs(spec: dict, d: str) -> None: from reportlab.lib.pagesizes import letter from reportlab.pdfgen import canvas for j, name in enumerate(spec["inputs"]): c = canvas.Canvas(os.path.join(d, name), pagesize=letter) c.setFont("Helvetica", 14) c.drawString(72, 700, f"INPUT DOCUMENT {j + 1} MARKER-{spec['id']}-{j + 1}") c.save() def run_script(script: str, spec: dict | None = None, timeout_s: int = 90) -> tuple[str | None, str]: d = tempfile.mkdtemp(prefix="sg_pdf_") out = os.path.join(d, "out.pdf") inp = os.path.join(d, "inputs") os.makedirs(inp, exist_ok=True) if spec and spec.get("context") == "assemble": _make_inputs(spec, inp) path = os.path.join(d, "build.py") with open(path, "w", encoding="utf-8") as fh: fh.write(script) env = {**os.environ, "OUTPUT": out, "INPUT_DIR": inp, "PYTHONIOENCODING": "utf-8"} try: p = subprocess.run([sys.executable, path], cwd=d, env=env, capture_output=True, text=True, timeout=timeout_s) if p.returncode != 0: err = next((ln for ln in reversed(p.stderr.splitlines()) if "Error" in ln or "error" in ln), "") return None, f"exit {p.returncode}: {(err or p.stderr)[-300:]}" except subprocess.TimeoutExpired: return None, "timeout" return (out if os.path.exists(out) else None), (p.stdout + p.stderr)[-500:] # ---------------------------------------------------------------- grading def grade(spec: dict, path: str | None) -> tuple[int, str]: if not path: return 0, "no file" try: from pypdf import PdfReader import pdfplumber reader = PdfReader(path) except Exception as e: # noqa: BLE001 return 0, f"unreadable: {str(e)[:80]}" ctx = spec["context"] try: if ctx == "form": fields = reader.get_fields() or {} names = set(fields.keys()) for f in spec["fields"]: if f not in names: return 0, f"missing text field {f}; have {sorted(names)[:6]}" if fields[f].get("/FT") != "/Tx": return 0, f"{f} is not a text field ({fields[f].get('/FT')})" if spec["checkbox"] not in names: return 0, f"missing checkbox {spec['checkbox']}" if fields[spec["checkbox"]].get("/FT") != "/Btn": return 0, "checkbox is not a button field" with pdfplumber.open(path) as pdf: text = pdf.pages[0].extract_text() or "" if spec["title"] not in text: return 0, "title text missing" return 1, "ok" if ctx == "report": k = len(spec["heads"]) if len(reader.pages) != k: return 0, f"{len(reader.pages)} pages, want {k}" with pdfplumber.open(path) as pdf: texts = [(pg.extract_text() or "") for pg in pdf.pages] tables = pdf.pages[0].extract_tables() for j, h in enumerate(spec["heads"]): lines = [ln.strip() for ln in texts[j].splitlines() if ln.strip()] if not lines or lines[0] != h: return 0, f"page {j + 1} first line {lines[:1]} != {h!r}" if f"Page {j + 1} of {k}" not in texts[j]: return 0, f"page {j + 1} footer missing 'Page {j + 1} of {k}'" want = [["Item", "Count"]] + [[a, str(b)] for a, b in spec["rows"]] got = [[(c or "").strip() for c in row] for t in tables for row in t] if tables and got[:len(want)] == want: return 1, "ok" # a borderless table is still a table: accept the rows as consecutive text lines # "Item Count" then "Alpha 120" (pdfplumber only finds ruled tables) lines = [" ".join(ln.split()) for ln in texts[0].splitlines()] want_lines = [" ".join(r) for r in want] for s in range(len(lines) - len(want_lines) + 1): if lines[s:s + len(want_lines)] == want_lines: return 1, "ok (borderless table)" return 0, f"table rows {got[:3] or lines[1:4]}" if ctx == "sci": meta = reader.metadata or {} if (meta.get("/Title") or "") != spec["title"] or (meta.get("/Author") or "") != spec["author"]: return 0, f"metadata {meta.get('/Title')!r} / {meta.get('/Author')!r}" with pdfplumber.open(path) as pdf: pg = pdf.pages[0] text = (pg.extract_text() or "").replace(" ", "") chars = pg.chars if f"is{spec['comp'][2].replace(' ', '')}" not in text or "growsquickly" not in text: return 0, "body words missing" letters, digit = spec["comp"][0][:-1], spec["comp"][0][-1] # find the digit glyph right after the letters: smaller and lower (subscript) ok_sub = ok_sup = False for i, ch in enumerate(chars): if ch["text"] == digit and i >= 1: prev = chars[i - 1] if prev["text"] == letters[-1] and ch["size"] < prev["size"] - 0.5 and ch["bottom"] > prev["bottom"] + 0.5: ok_sub = True if ch["text"] == str(spec["power"]) and i >= 1: prev = chars[i - 1] if prev["text"] == "x" and ch["size"] < prev["size"] - 0.5 and ch["top"] < prev["top"] - 0.5: ok_sup = True if not ok_sub: return 0, "no real subscript digit (smaller and lower) after the formula letters" if not ok_sup: return 0, "no real superscript digit (smaller and raised) after x" return 1, "ok" if ctx == "assemble": n = len(spec["inputs"]) if len(reader.pages) != n: return 0, f"{len(reader.pages)} pages, want {n}" for j in range(n): rot = int(reader.pages[j].get("/Rotate", 0)) % 360 want = 90 if j + 1 == spec["rot_page"] else 0 if rot != want: return 0, f"page {j + 1} rotation {rot}, want {want}" txt = reader.pages[j].extract_text() or "" if f"MARKER-{spec['id']}-{j + 1}" not in txt: return 0, f"page {j + 1} lost its original text or wrong order" outline = reader.outline titles = [o.title for o in outline if hasattr(o, "title")] if titles != spec["marks"]: return 0, f"outline {titles} != {spec['marks']}" for j, o in enumerate([o for o in outline if hasattr(o, "title")]): if reader.get_destination_page_number(o) != j: return 0, f"bookmark {o.title!r} points to page {reader.get_destination_page_number(o) + 1}" return 1, "ok" except Exception as e: # noqa: BLE001 return 0, f"grader exception: {str(e)[:100]}" return 0, "unknown context" def user_prompt(spec: dict) -> str: return (spec["spec"] + "\n\nWrite ONE complete Python script (reportlab and/or pypdf are installed) that builds this " "PDF and writes it to the path in the environment variable OUTPUT. Output only the script in a single " "```python fence. Do not add anything the spec did not ask for.") # ---- end of pdf_tasks.py ---- def build_reference(spec, out_path): from reportlab.pdfgen import canvas from reportlab.lib.pagesizes import letter c = canvas.Canvas(out_path, pagesize=letter) width, height = letter c.setFont("Helvetica-Bold", 16) c.drawString(72, height - 72, spec["title"]) form = c.acroForm y = height - 130 c.setFont("Helvetica", 12) for fname in spec["fields"]: label = fname.replace('_', ' ').title() c.drawString(72, y + 5, label + ":") form.textfield(name=fname, tooltip=label, x=220, y=y, width=200, height=20, borderStyle='solid', borderWidth=1, forceBorder=True) y -= 40 c.drawString(72, y + 5, spec["checkbox"].replace('_', ' ').title() + ":") form.checkbox(name=spec["checkbox"], tooltip=spec["checkbox"], x=220, y=y, size=16, borderStyle='solid', borderWidth=1, forceBorder=True) c.save() return out_path def _load_pdf(path): from pypdf import PdfReader, PdfWriter reader = PdfReader(path) writer = PdfWriter() writer.append(reader) return reader, writer def spoil_missing_field(path): out = path + '.spoil_missing_field' from pypdf import PdfReader, PdfWriter reader = PdfReader(path) writer = PdfWriter() writer.append(reader) # remove one of the required text fields from the AcroForm and page annotations root = writer._root_object acro = root['/AcroForm'] fields = acro['/Fields'] to_remove_name = None kept = [] for f in fields: obj = f.get_object() name = obj.get('/T') if name and str(name) not in ('agree_terms',) and to_remove_name is None: to_remove_name = str(name) continue kept.append(f) acro[__import__('pypdf').generic.NameObject('/Fields')] = __import__('pypdf').generic.ArrayObject(kept) for page in writer.pages: annots = page.get('/Annots') if annots: new_annots = [] for a in annots: obj = a.get_object() if obj.get('/T') and str(obj.get('/T')) == to_remove_name: continue new_annots.append(a) page[__import__('pypdf').generic.NameObject('/Annots')] = __import__('pypdf').generic.ArrayObject(new_annots) with open(out, 'wb') as fh: writer.write(fh) return out def spoil_checkbox_to_text(path): out = path + '.spoil_checkbox_to_text' from pypdf import PdfReader, PdfWriter from pypdf.generic import NameObject reader = PdfReader(path) writer = PdfWriter() writer.append(reader) acro = writer._root_object['/AcroForm'] for f in acro['/Fields']: obj = f.get_object() if obj.get('/T') and str(obj.get('/T')) == 'agree_terms': obj[NameObject('/FT')] = NameObject('/Tx') with open(out, 'wb') as fh: writer.write(fh) return out def spoil_rename_field(path): out = path + '.spoil_rename_field' from pypdf import PdfReader, PdfWriter from pypdf.generic import NameObject, TextStringObject reader = PdfReader(path) writer = PdfWriter() writer.append(reader) acro = writer._root_object['/AcroForm'] done = False for f in acro['/Fields']: obj = f.get_object() name = obj.get('/T') if name and str(name) not in ('agree_terms',) and not done: obj[NameObject('/T')] = TextStringObject(str(name) + '_renamed') done = True with open(out, 'wb') as fh: writer.write(fh) return out def spoil_missing_title(path): out = path + '.spoil_missing_title' from pypdf import PdfReader, PdfWriter from pypdf.generic import NameObject, ArrayObject reader = PdfReader(path) writer = PdfWriter() writer.append(reader) # strip the page content stream text entirely so title text vanishes but form stays page = writer.pages[0] try: page[NameObject('/Contents')] = writer._add_object(__import__('pypdf').generic.StreamObject()) page['/Contents'].set_data(b'') except Exception: pass with open(out, 'wb') as fh: writer.write(fh) return out def spoil_flatten_widgets(path): out = path + '.spoil_flatten_widgets' from reportlab.pdfgen import canvas from reportlab.lib.pagesizes import letter import pypdf reader = pypdf.PdfReader(path) # rebuild a visually similar page with drawn boxes/labels instead of real form widgets c = canvas.Canvas(out, pagesize=letter) width, height = letter c.setFont('Helvetica-Bold', 16) c.drawString(72, height - 72, 'Incident Report Form') c.setFont('Helvetica', 12) y = height - 130 for label in ['Date Of Birth', 'City', 'Phone']: c.drawString(72, y + 5, label + ':') c.rect(220, y, 200, 20) y -= 40 c.drawString(72, y + 5, 'Agree Terms:') c.rect(220, y, 16, 16) c.save() return out SPOILERS = [ ('missing_field', spoil_missing_field), ('checkbox_to_text', spoil_checkbox_to_text), ('rename_field', spoil_rename_field), ('missing_title', spoil_missing_title), ('flatten_widgets', spoil_flatten_widgets), ]