from __future__ import annotations
"""Mutation gate for pdf_tasks, forged and verified 5/5 near-misses rejected.
The checker in pdf_tasks.py could grade an output but nothing showed it would
reject a broken one. These are the deliberate near-misses it refuses, each one
verified to change the file and to be rejected.
"""
# ---- inlined from pdf_tasks.py so this file runs on its own ----
"""Real-skill task bank for the anthropics `pdf` skill (Python: reportlab, pypdf), graded
MECHANICALLY with pypdf and pdfplumber. Four contexts:
form a fillable AcroForm with exact text field names and a checkbox (graded by pypdf get_fields)
report a multi-page platypus report: exact heading per page, a table, footer "Page N of M"
sci paragraphs with real sub/superscripts (H2O, x^2 as raised smaller glyphs) and PDF metadata
assemble merge generated inputs, rotate one page, add an outline (bookmarks): pypdf manipulation
Scripts are Python; the runner gives OUTPUT (path to write) and INPUT_DIR (assemble inputs).
"""
# (moved to the top of this file)
import os
import random
import re
import subprocess
import sys
import tempfile
CONTEXTS = ["form", "report", "sci", "assemble"]
FIELDS = ["full_name", "email", "phone", "company", "date_of_birth", "city", "employee_id", "department"]
HEADINGS = ["Executive Summary", "Site Findings", "Risk Register", "Budget Outlook", "Staffing Plan",
"Vendor Review", "Timeline", "Recommendations", "Appendix A", "Open Items"]
AUTHORS = ["K. Clouthier", "Ops Team", "Safety Office", "Finance Desk", "Field Unit 3"]
COMPOUNDS = [("H2O", "H2O", "water"), ("CO2", "CO2", "carbon dioxide"),
("CH4", "CH4", "methane"), ("NH3", "NH3", "ammonia"), ("O3", "O3", "ozone")]
def build(n_per: int = 8, seed: int = 43) -> list[dict]:
rng = random.Random(seed)
specs: list[dict] = []
for i in range(n_per):
fields = rng.sample(FIELDS, 3)
cb = rng.choice(["agree_terms", "subscribe", "confirmed", "active"])
title = rng.choice(["Intake Form", "Access Request", "Onboarding Sheet", "Incident Report Form"])
specs.append({"context": "form", "id": f"fm{i}", "fields": fields, "checkbox": cb, "title": title,
"spec": (f"Create a one-page fillable PDF form titled \"{title}\" (the title drawn as text at the top). "
f"It must contain exactly these fillable TEXT fields, named exactly: {', '.join(fields)}, each "
f"with a visible label next to it, and one fillable CHECKBOX field named exactly \"{cb}\". "
f"The fields must be real AcroForm widgets a PDF reader can fill, not drawn boxes.")})
for i in range(n_per):
k = rng.randint(3, 4)
heads = rng.sample(HEADINGS, k)
rows = [(rng.choice(["Alpha", "Bravo", "Charlie", "Delta", "Echo"]), rng.randint(10, 900)) for _ in range(rng.randint(3, 5))]
specs.append({"context": "report", "id": f"rp{i}", "heads": heads, "rows": rows,
"spec": (f"Create a {k}-page US Letter PDF report. Page p (1..{k}) must start with the heading "
f"\"{heads[0]}\"" + "".join(f", page {j + 1} with \"{h}\"" for j, h in enumerate(heads[1:], start=1)) +
f" (each heading as its own line of text at the top of its page, force page breaks between pages). "
f"Page 1 must also contain a table with header row exactly Item, Count and these rows: "
+ "; ".join(f"{a}, {b}" for a, b in rows) +
f". Every page must have a footer with exactly the text \"Page N of {k}\" where N is that page's "
f"number (so page 2 shows \"Page 2 of {k}\").")})
for i in range(n_per):
comp = rng.choice(COMPOUNDS)
power = rng.randint(2, 5)
author = rng.choice(AUTHORS)
title = rng.choice(["Lab Notes", "Reaction Summary", "Field Chemistry", "Process Note"])
specs.append({"context": "sci", "id": f"sc{i}", "comp": comp, "power": power, "author": author, "title": title,
"spec": (f"Create a one-page PDF with document metadata title exactly \"{title}\" and author exactly "
f"\"{author}\". Its body has two paragraphs. Paragraph 1 must contain the formula {comp[0]} written "
f"with a real subscript for the digit (the digit rendered smaller and lower than the letters, as in "
f"reportlab's markup), followed by the words \"is {comp[2]}\". Paragraph 2 must contain "
f"\"x{power}\" written as x to the power {power} with a real superscript (the digit smaller and "
f"raised), followed by the words \"grows quickly\".")})
for i in range(n_per):
n_in = rng.randint(2, 3)
rot_page = rng.randint(1, n_in)
names = [f"part{j + 1}.pdf" for j in range(n_in)]
marks = rng.sample(HEADINGS, n_in)
specs.append({"context": "assemble", "id": f"as{i}", "inputs": names, "rot_page": rot_page, "marks": marks,
"spec": (f"In the directory given by the environment variable INPUT_DIR there are {n_in} one-page PDFs: "
f"{', '.join(names)}. Merge them in that order into one PDF. Rotate page {rot_page} of the merged "
f"document by 90 degrees clockwise (only that page). Add an outline (bookmarks): one top-level "
f"bookmark per page, titled exactly " + ", ".join(f"\"{m}\" for page {j + 1}" for j, m in enumerate(marks)) +
f". Keep every page's original text.")})
rng.shuffle(specs)
return specs
# ---------------------------------------------------------------- run the model's script
FENCE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.S)
def extract_script(reply: str) -> str | None:
m = FENCE.findall(reply or "")
if m:
return max(m, key=len)
if "import" in (reply or "") and ("reportlab" in reply or "pypdf" in reply):
return reply
return None
def _make_inputs(spec: dict, d: str) -> None:
from reportlab.lib.pagesizes import letter
from reportlab.pdfgen import canvas
for j, name in enumerate(spec["inputs"]):
c = canvas.Canvas(os.path.join(d, name), pagesize=letter)
c.setFont("Helvetica", 14)
c.drawString(72, 700, f"INPUT DOCUMENT {j + 1} MARKER-{spec['id']}-{j + 1}")
c.save()
def run_script(script: str, spec: dict | None = None, timeout_s: int = 90) -> tuple[str | None, str]:
d = tempfile.mkdtemp(prefix="sg_pdf_")
out = os.path.join(d, "out.pdf")
inp = os.path.join(d, "inputs")
os.makedirs(inp, exist_ok=True)
if spec and spec.get("context") == "assemble":
_make_inputs(spec, inp)
path = os.path.join(d, "build.py")
with open(path, "w", encoding="utf-8") as fh:
fh.write(script)
env = {**os.environ, "OUTPUT": out, "INPUT_DIR": inp, "PYTHONIOENCODING": "utf-8"}
try:
p = subprocess.run([sys.executable, path], cwd=d, env=env, capture_output=True, text=True, timeout=timeout_s)
if p.returncode != 0:
err = next((ln for ln in reversed(p.stderr.splitlines()) if "Error" in ln or "error" in ln), "")
return None, f"exit {p.returncode}: {(err or p.stderr)[-300:]}"
except subprocess.TimeoutExpired:
return None, "timeout"
return (out if os.path.exists(out) else None), (p.stdout + p.stderr)[-500:]
# ---------------------------------------------------------------- grading
def grade(spec: dict, path: str | None) -> tuple[int, str]:
if not path:
return 0, "no file"
try:
from pypdf import PdfReader
import pdfplumber
reader = PdfReader(path)
except Exception as e: # noqa: BLE001
return 0, f"unreadable: {str(e)[:80]}"
ctx = spec["context"]
try:
if ctx == "form":
fields = reader.get_fields() or {}
names = set(fields.keys())
for f in spec["fields"]:
if f not in names:
return 0, f"missing text field {f}; have {sorted(names)[:6]}"
if fields[f].get("/FT") != "/Tx":
return 0, f"{f} is not a text field ({fields[f].get('/FT')})"
if spec["checkbox"] not in names:
return 0, f"missing checkbox {spec['checkbox']}"
if fields[spec["checkbox"]].get("/FT") != "/Btn":
return 0, "checkbox is not a button field"
with pdfplumber.open(path) as pdf:
text = pdf.pages[0].extract_text() or ""
if spec["title"] not in text:
return 0, "title text missing"
return 1, "ok"
if ctx == "report":
k = len(spec["heads"])
if len(reader.pages) != k:
return 0, f"{len(reader.pages)} pages, want {k}"
with pdfplumber.open(path) as pdf:
texts = [(pg.extract_text() or "") for pg in pdf.pages]
tables = pdf.pages[0].extract_tables()
for j, h in enumerate(spec["heads"]):
lines = [ln.strip() for ln in texts[j].splitlines() if ln.strip()]
if not lines or lines[0] != h:
return 0, f"page {j + 1} first line {lines[:1]} != {h!r}"
if f"Page {j + 1} of {k}" not in texts[j]:
return 0, f"page {j + 1} footer missing 'Page {j + 1} of {k}'"
want = [["Item", "Count"]] + [[a, str(b)] for a, b in spec["rows"]]
got = [[(c or "").strip() for c in row] for t in tables for row in t]
if tables and got[:len(want)] == want:
return 1, "ok"
# a borderless table is still a table: accept the rows as consecutive text lines
# "Item Count" then "Alpha 120" (pdfplumber only finds ruled tables)
lines = [" ".join(ln.split()) for ln in texts[0].splitlines()]
want_lines = [" ".join(r) for r in want]
for s in range(len(lines) - len(want_lines) + 1):
if lines[s:s + len(want_lines)] == want_lines:
return 1, "ok (borderless table)"
return 0, f"table rows {got[:3] or lines[1:4]}"
if ctx == "sci":
meta = reader.metadata or {}
if (meta.get("/Title") or "") != spec["title"] or (meta.get("/Author") or "") != spec["author"]:
return 0, f"metadata {meta.get('/Title')!r} / {meta.get('/Author')!r}"
with pdfplumber.open(path) as pdf:
pg = pdf.pages[0]
text = (pg.extract_text() or "").replace(" ", "")
chars = pg.chars
if f"is{spec['comp'][2].replace(' ', '')}" not in text or "growsquickly" not in text:
return 0, "body words missing"
letters, digit = spec["comp"][0][:-1], spec["comp"][0][-1]
# find the digit glyph right after the letters: smaller and lower (subscript)
ok_sub = ok_sup = False
for i, ch in enumerate(chars):
if ch["text"] == digit and i >= 1:
prev = chars[i - 1]
if prev["text"] == letters[-1] and ch["size"] < prev["size"] - 0.5 and ch["bottom"] > prev["bottom"] + 0.5:
ok_sub = True
if ch["text"] == str(spec["power"]) and i >= 1:
prev = chars[i - 1]
if prev["text"] == "x" and ch["size"] < prev["size"] - 0.5 and ch["top"] < prev["top"] - 0.5:
ok_sup = True
if not ok_sub:
return 0, "no real subscript digit (smaller and lower) after the formula letters"
if not ok_sup:
return 0, "no real superscript digit (smaller and raised) after x"
return 1, "ok"
if ctx == "assemble":
n = len(spec["inputs"])
if len(reader.pages) != n:
return 0, f"{len(reader.pages)} pages, want {n}"
for j in range(n):
rot = int(reader.pages[j].get("/Rotate", 0)) % 360
want = 90 if j + 1 == spec["rot_page"] else 0
if rot != want:
return 0, f"page {j + 1} rotation {rot}, want {want}"
txt = reader.pages[j].extract_text() or ""
if f"MARKER-{spec['id']}-{j + 1}" not in txt:
return 0, f"page {j + 1} lost its original text or wrong order"
outline = reader.outline
titles = [o.title for o in outline if hasattr(o, "title")]
if titles != spec["marks"]:
return 0, f"outline {titles} != {spec['marks']}"
for j, o in enumerate([o for o in outline if hasattr(o, "title")]):
if reader.get_destination_page_number(o) != j:
return 0, f"bookmark {o.title!r} points to page {reader.get_destination_page_number(o) + 1}"
return 1, "ok"
except Exception as e: # noqa: BLE001
return 0, f"grader exception: {str(e)[:100]}"
return 0, "unknown context"
def user_prompt(spec: dict) -> str:
return (spec["spec"] + "\n\nWrite ONE complete Python script (reportlab and/or pypdf are installed) that builds this "
"PDF and writes it to the path in the environment variable OUTPUT. Output only the script in a single "
"```python fence. Do not add anything the spec did not ask for.")
# ---- end of pdf_tasks.py ----
def build_reference(spec, out_path):
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter
c = canvas.Canvas(out_path, pagesize=letter)
width, height = letter
c.setFont("Helvetica-Bold", 16)
c.drawString(72, height - 72, spec["title"])
form = c.acroForm
y = height - 130
c.setFont("Helvetica", 12)
for fname in spec["fields"]:
label = fname.replace('_', ' ').title()
c.drawString(72, y + 5, label + ":")
form.textfield(name=fname, tooltip=label, x=220, y=y, width=200, height=20,
borderStyle='solid', borderWidth=1, forceBorder=True)
y -= 40
c.drawString(72, y + 5, spec["checkbox"].replace('_', ' ').title() + ":")
form.checkbox(name=spec["checkbox"], tooltip=spec["checkbox"], x=220, y=y, size=16,
borderStyle='solid', borderWidth=1, forceBorder=True)
c.save()
return out_path
def _load_pdf(path):
from pypdf import PdfReader, PdfWriter
reader = PdfReader(path)
writer = PdfWriter()
writer.append(reader)
return reader, writer
def spoil_missing_field(path):
out = path + '.spoil_missing_field'
from pypdf import PdfReader, PdfWriter
reader = PdfReader(path)
writer = PdfWriter()
writer.append(reader)
# remove one of the required text fields from the AcroForm and page annotations
root = writer._root_object
acro = root['/AcroForm']
fields = acro['/Fields']
to_remove_name = None
kept = []
for f in fields:
obj = f.get_object()
name = obj.get('/T')
if name and str(name) not in ('agree_terms',) and to_remove_name is None:
to_remove_name = str(name)
continue
kept.append(f)
acro[__import__('pypdf').generic.NameObject('/Fields')] = __import__('pypdf').generic.ArrayObject(kept)
for page in writer.pages:
annots = page.get('/Annots')
if annots:
new_annots = []
for a in annots:
obj = a.get_object()
if obj.get('/T') and str(obj.get('/T')) == to_remove_name:
continue
new_annots.append(a)
page[__import__('pypdf').generic.NameObject('/Annots')] = __import__('pypdf').generic.ArrayObject(new_annots)
with open(out, 'wb') as fh:
writer.write(fh)
return out
def spoil_checkbox_to_text(path):
out = path + '.spoil_checkbox_to_text'
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject
reader = PdfReader(path)
writer = PdfWriter()
writer.append(reader)
acro = writer._root_object['/AcroForm']
for f in acro['/Fields']:
obj = f.get_object()
if obj.get('/T') and str(obj.get('/T')) == 'agree_terms':
obj[NameObject('/FT')] = NameObject('/Tx')
with open(out, 'wb') as fh:
writer.write(fh)
return out
def spoil_rename_field(path):
out = path + '.spoil_rename_field'
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, TextStringObject
reader = PdfReader(path)
writer = PdfWriter()
writer.append(reader)
acro = writer._root_object['/AcroForm']
done = False
for f in acro['/Fields']:
obj = f.get_object()
name = obj.get('/T')
if name and str(name) not in ('agree_terms',) and not done:
obj[NameObject('/T')] = TextStringObject(str(name) + '_renamed')
done = True
with open(out, 'wb') as fh:
writer.write(fh)
return out
def spoil_missing_title(path):
out = path + '.spoil_missing_title'
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, ArrayObject
reader = PdfReader(path)
writer = PdfWriter()
writer.append(reader)
# strip the page content stream text entirely so title text vanishes but form stays
page = writer.pages[0]
try:
page[NameObject('/Contents')] = writer._add_object(__import__('pypdf').generic.StreamObject())
page['/Contents'].set_data(b'')
except Exception:
pass
with open(out, 'wb') as fh:
writer.write(fh)
return out
def spoil_flatten_widgets(path):
out = path + '.spoil_flatten_widgets'
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter
import pypdf
reader = pypdf.PdfReader(path)
# rebuild a visually similar page with drawn boxes/labels instead of real form widgets
c = canvas.Canvas(out, pagesize=letter)
width, height = letter
c.setFont('Helvetica-Bold', 16)
c.drawString(72, height - 72, 'Incident Report Form')
c.setFont('Helvetica', 12)
y = height - 130
for label in ['Date Of Birth', 'City', 'Phone']:
c.drawString(72, y + 5, label + ':')
c.rect(220, y, 200, 20)
y -= 40
c.drawString(72, y + 5, 'Agree Terms:')
c.rect(220, y, 16, 16)
c.save()
return out
SPOILERS = [
('missing_field', spoil_missing_field),
('checkbox_to_text', spoil_checkbox_to_text),
('rename_field', spoil_rename_field),
('missing_title', spoil_missing_title),
('flatten_widgets', spoil_flatten_widgets),
]