from __future__ import annotations
"""Mutation gate for docx_tasks, forged and verified 6/6 near-misses rejected.
The checker in docx_tasks.py could grade an output but nothing showed it would
reject a broken one. These are the deliberate near-misses it refuses, each one
verified to change the file and to be rejected.
"""
# ---- inlined from docx_tasks.py so this file runs on its own ----
"""Real-skill task bank for the anthropics `docx` skill, graded MECHANICALLY.
The skill creates documents with docx-js (Node). The model writes a JS script; the
runner executes it with `docx` resolvable from external/node; Python reads the produced
.docx (python-docx + raw OOXML) and grades it. No model grades anything.
The skill states footguns explicitly, and the grader holds the file to them: bullets must
be numbering (no literal bullet characters), table widths in DXA (not percentage), shading
type CLEAR (not SOLID), US Letter when asked, landscape via section orientation. A tier
that ignores the skill's own rules fails. That is the point of this bank: on `xlsx` every
thinking tier was at the ceiling; `docx` has rules a careless model breaks.
Three contexts:
memo title heading, exact paragraphs, a real bullet list, US Letter
table heading + a DXA-width table, CLEAR-shaded header row, bold total row
landscape two sections, second landscape with a header and a table
"""
# (moved to the top of this file)
import os
import random
import re
import subprocess
import sys
import tempfile
CONTEXTS = ["memo", "table", "landscape"]
NODE_DIR = os.path.join(os.path.dirname(__file__), "..", "external", "node")
TOPICS = ["Quarterly Safety Review", "Vendor Onboarding Notes", "Site Access Policy",
"Fleet Maintenance Summary", "Training Attendance Report", "Incident Follow-up"]
SENTENCES = ["All contractors must sign in at the front desk before entering the yard.",
"The east gate will be closed for resurfacing from Monday to Wednesday.",
"Please submit expense forms by the fifth business day of each month.",
"Radio channel 4 is reserved for crane operations during lifts.",
"The new visitor badges expire at the end of the calendar year.",
"Report any damaged signage to facilities using the online form.",
"Overtime requires written approval from a supervisor in advance.",
"First aid kits are inspected on the first Friday of every month."]
ITEMS = ["Hard hats", "Safety boots", "Hi-vis vests", "Eye protection", "Gloves",
"Hearing protection", "Fall arrest harness", "Face shield"]
REGIONS = ["North", "South", "East", "West", "Central"]
def build(n_per: int = 8, seed: int = 31) -> list[dict]:
rng = random.Random(seed)
specs: list[dict] = []
for i in range(n_per):
title = rng.choice(TOPICS)
paras = rng.sample(SENTENCES, 3)
bullets = rng.sample(ITEMS, 3)
specs.append({"context": "memo", "id": f"mm{i}", "title": title, "paras": paras, "bullets": bullets,
"spec": (f"Create a US Letter Word document. First a Heading 1 with exactly the text \"{title}\". "
f"Then these three paragraphs, each its own paragraph, exact text:\n"
+ "\n".join(paras)
+ f"\nThen a bullet list with exactly these three items, in this order: {', '.join(bullets)}. "
f"Use a real bullet list (numbering), not bullet characters typed into the text.")})
for i in range(n_per):
rows = [(rng.choice(REGIONS), rng.randint(10, 400), rng.randint(1, 60)) for _ in range(rng.randint(3, 5))]
title = rng.choice(TOPICS)
specs.append({"context": "table", "id": f"tb{i}", "title": title, "rows": rows,
"spec": (f"Create a Word document with a Heading 1 with exactly the text \"{title}\", then a table. "
f"Header row, exactly: Region, Units, Incidents. Then these rows:\n"
+ "\n".join(f"{r}, {u}, {c}" for r, u, c in rows)
+ f"\nThen a final row with the text Total in the first cell, {sum(u for _, u, _ in rows)} in the second "
f"and {sum(c for _, _, c in rows)} in the third; every run of text in that final row must be bold. "
f"The header row cells must have a light grey shading. Table and cell widths must be set in DXA "
f"with columnWidths on the table and width on every cell, and the column widths must sum to the table width.")})
for i in range(n_per):
title = rng.choice(TOPICS)
para = rng.choice(SENTENCES)
header = f"{title} - Appendix"
rows = [(rng.choice(ITEMS), rng.randint(1, 99)) for _ in range(3)]
specs.append({"context": "landscape", "id": f"ls{i}", "title": title, "para": para, "header": header, "rows": rows,
"spec": (f"Create a Word document with two sections. Section one, portrait: a Heading 1 with exactly the text "
f"\"{title}\" and one paragraph with exactly the text \"{para}\". Section two, LANDSCAPE orientation, "
f"with a page header containing exactly the text \"{header}\", and a table with header row exactly "
f"Item, Count and these rows:\n" + "\n".join(f"{a}, {b}" for a, b in rows)
+ "\nUse DXA widths on the table and every cell.")})
rng.shuffle(specs)
return specs
# ---------------------------------------------------------------- run the model's script
FENCE = re.compile(r"```(?:javascript|js|typescript|ts)?\s*\n(.*?)```", re.S)
def extract_script(reply: str) -> str | None:
m = FENCE.findall(reply or "")
if m:
return max(m, key=len)
if "require(" in (reply or "") and "docx" in reply:
return reply
return None
def run_script(script: str, timeout_s: int = 60) -> tuple[str | None, str]:
d = tempfile.mkdtemp(prefix="sg_docx_")
out = os.path.join(d, "out.docx")
path = os.path.join(d, "build.js")
with open(path, "w", encoding="utf-8") as fh:
fh.write(script)
env = {**os.environ, "OUTPUT": out,
"NODE_PATH": os.path.abspath(os.path.join(NODE_DIR, "node_modules"))}
try:
p = subprocess.run(["node", path], cwd=d, env=env, capture_output=True, text=True, timeout=timeout_s)
log = (p.stdout + p.stderr)[-2000:]
if p.returncode != 0:
# the error line is near the TOP of node's stderr; the tail is only the stack
err = next((ln for ln in p.stderr.splitlines() if re.search(r"Error|error", ln) and "at " not in ln[:6]), "")
return None, f"exit {p.returncode}: {err[:300] or log[:300]}"
except subprocess.TimeoutExpired:
return None, "timeout"
except FileNotFoundError:
return None, "node not found"
return (out if os.path.exists(out) else None), log
# ---------------------------------------------------------------- grading
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
BULLET_CHARS = "•●◦‣⁃·-*"
def _ptext(p) -> str:
return "".join(r.text for r in p.runs).strip()
def _is_heading1(p) -> bool:
name = (p.style.name if p.style is not None else "") or ""
if name.lower().startswith("heading 1"):
return True
ppr = p._p.pPr
if ppr is not None and ppr.pStyle is not None:
return str(ppr.pStyle.val).lower() in ("heading1", "heading 1")
return False
def _has_numpr(p) -> bool:
ppr = p._p.pPr
return ppr is not None and ppr.numPr is not None
def _table_widths_dxa(t) -> tuple[bool, str]:
"""columnWidths on the table (tblGrid gridCol) AND width on every cell, all DXA."""
tbl = t._tbl
tblw = tbl.find(f"{W}tblPr/{W}tblW")
if tblw is not None and tblw.get(f"{W}type") not in (None, "dxa", "auto"):
return False, f"tblW type {tblw.get(f'{W}type')}"
grid = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol") if g.get(f"{W}w")]
if not grid:
return False, "no tblGrid columnWidths"
for tr in tbl.findall(f"{W}tr"):
for tc in tr.findall(f"{W}tc"):
tcw = tc.find(f"{W}tcPr/{W}tcW")
if tcw is None:
return False, "cell without width"
if tcw.get(f"{W}type") not in (None, "dxa"):
return False, f"cell width type {tcw.get(f'{W}type')}"
if tblw is not None and tblw.get(f"{W}type") == "dxa" and tblw.get(f"{W}w"):
if abs(sum(grid) - int(tblw.get(f"{W}w"))) > 2:
return False, f"columns sum {sum(grid)} != table width {tblw.get(f'{W}w')}"
return True, "ok"
def _cell_shading(tc) -> tuple[str | None, str | None]:
shd = tc._tc.find(f"{W}tcPr/{W}shd")
if shd is None:
return None, None
return shd.get(f"{W}val"), shd.get(f"{W}fill")
def grade(spec: dict, path: str | None) -> tuple[int, str]:
if not path:
return 0, "no file"
try:
import docx as pydocx
d = pydocx.Document(path)
except Exception as e: # noqa: BLE001
return 0, f"unreadable: {e}"
ctx = spec["context"]
try:
body_paras = [p for p in d.paragraphs]
texts = [_ptext(p) for p in body_paras]
for t in texts:
if "\n" in t:
return 0, "literal newline in a run"
if ctx == "memo":
sec = d.sections[0]
if abs(int(sec.page_width) - 7772400) > 20000 or abs(int(sec.page_height) - 10058400) > 20000:
return 0, f"page size {sec.page_width}x{sec.page_height} not US Letter"
heads = [p for p in body_paras if _is_heading1(p)]
if not heads or _ptext(heads[0]) != spec["title"]:
return 0, f"heading {[_ptext(h) for h in heads][:1]}"
for s in spec["paras"]:
if s not in texts:
return 0, f"missing paragraph: {s[:30]}"
found = 0
for p in body_paras:
t = _ptext(p)
if t in spec["bullets"]:
if not _has_numpr(p):
return 0, f"bullet item without numbering: {t}"
found += 1
if t and t[0] in BULLET_CHARS and t[1:].strip() in spec["bullets"]:
return 0, f"literal bullet character: {t[:12]!r}"
if found != 3:
return 0, f"found {found} of 3 list items"
return 1, "ok"
if ctx == "table":
heads = [p for p in body_paras if _is_heading1(p)]
if not heads or _ptext(heads[0]) != spec["title"]:
return 0, "heading"
if not d.tables:
return 0, "no table"
t = d.tables[0]
cells = [[c.text.strip() for c in r.cells] for r in t.rows]
if cells[0] != ["Region", "Units", "Incidents"]:
return 0, f"header {cells[0]}"
want = [[r, str(u), str(c)] for r, u, c in spec["rows"]]
if cells[1:1 + len(want)] != want:
return 0, f"rows {cells[1:1 + len(want)]}"
tot = cells[1 + len(want)]
if tot != ["Total", str(sum(u for _, u, _ in spec["rows"])), str(sum(c for _, _, c in spec["rows"]))]:
return 0, f"total row {tot}"
last = t.rows[1 + len(want)]
for c in last.cells:
for p in c.paragraphs:
for r in p.runs:
if r.text.strip() and not r.bold:
return 0, "total row run not bold"
for c in t.rows[0].cells:
val, fill = _cell_shading(c)
if val is None or fill in (None, "auto", "FFFFFF", "ffffff"):
return 0, f"header shading missing ({val},{fill})"
if val.lower() == "solid":
return 0, "header shading SOLID"
ok, why = _table_widths_dxa(t)
if not ok:
return 0, why
return 1, "ok"
if ctx == "landscape":
if len(d.sections) < 2:
return 0, f"{len(d.sections)} section(s)"
s2 = d.sections[1]
if int(s2.page_width) <= int(s2.page_height):
return 0, "section 2 not landscape by dimensions"
sectpr = s2._sectPr
pgsz = sectpr.find(f"{W}pgSz")
if pgsz is None or pgsz.get(f"{W}orient") != "landscape":
return 0, "section 2 orient attribute not landscape"
htxt = " ".join(_ptext(p) for p in s2.header.paragraphs).strip()
if htxt != spec["header"]:
return 0, f"header {htxt!r}"
heads = [p for p in body_paras if _is_heading1(p)]
if not heads or _ptext(heads[0]) != spec["title"]:
return 0, "heading"
if spec["para"] not in texts:
return 0, "paragraph"
if not d.tables:
return 0, "no table"
t = d.tables[0]
cells = [[c.text.strip() for c in r.cells] for r in t.rows]
want = [["Item", "Count"]] + [[a, str(b)] for a, b in spec["rows"]]
if cells[:len(want)] != want:
return 0, f"table {cells[:len(want)]}"
ok, why = _table_widths_dxa(t)
if not ok:
return 0, why
return 1, "ok"
except Exception as e: # noqa: BLE001
return 0, f"grader exception: {e}"
return 0, "unknown context"
def user_prompt(spec: dict) -> str:
return (spec["spec"] + "\n\nWrite ONE complete Node.js script using the docx package (require('docx')) that builds "
"this document and writes it to the path in process.env.OUTPUT using Packer.toBuffer and fs.writeFileSync. "
"Output only the script in a single ```javascript fence. Do not convert to PDF. Do not add anything the spec did not ask for.")
# ---- end of docx_tasks.py ----
import shutil, zipfile, re, os
def build_reference(spec, out_path):
rows = spec['rows']
n_cols_1 = 2
header_widths = [4000, 2000]
table_width = sum(header_widths)
def esc(s):
return (s.replace('&', '&').replace('<', '<').replace('>', '>')
.replace('"', '"').replace("'", '''))
def para(text, style=None):
pPr = f'' if style else ''
return f'{pPr}{esc(text)}'
def cell(text, width, bold=False, shade=None):
rpr = '' if bold else ''
shd = f'' if shade else ''
return (f'{shd}'
f'{rpr}{esc(text)}')
def row(cells_texts, widths, bold=False, shade_first_row=False):
cells_xml = ''
for txt, w in zip(cells_texts, widths):
cells_xml += cell(txt, w, bold=bold, shade=('D9D9D9' if shade_first_row else None))
return f'{cells_xml}'
grid_xml = ''.join(f'' for w in header_widths)
table_rows = row(['Item', 'Count'], header_widths, shade_first_row=True)
for a, b in rows:
table_rows += row([a, str(b)], header_widths)
table_xml = (f''
f'{grid_xml}{table_rows}')
# Section 1: portrait, US letter-ish dims not required here but keep sane
sec1_width, sec1_height = 12240, 15840 # portrait letter
sec2_width, sec2_height = 15840, 12240 # landscape letter
header_text = spec['header']
header_rid = 'rId100'
header_part_xml = (
'\n'
''
f'{esc(header_text)}'
''
)
body_p1 = para(spec['title'], style='Heading1')
body_p2 = para(spec['para'])
sectPr1 = (f''
f'')
sectPr2 = (f''
f''
f'')
# first "section" is embedded as a paragraph with sectPr in its pPr to end section 1
p1_with_sect = (f''
f'')
document_xml = (
'\n'
''
''
+ body_p1
+ body_p2
# end of section 1: a paragraph carrying sectPr for section 1 (required OOXML pattern)
+ '' + sectPr1 + ''
+ table_xml
+ ''
+ f''
+ f''
+ ''
+ ''
+ ''
)
content_types = (
'\n'
''
''
''
''
''
''
''
)
rels = (
'\n'
''
''
''
)
doc_rels = (
'\n'
''
f''
''
''
)
styles_xml = (
'\n'
''
''
''
''
''
)
tmp_dir = out_path + '_build_tmp'
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.makedirs(os.path.join(tmp_dir, '_rels'))
os.makedirs(os.path.join(tmp_dir, 'word', '_rels'))
with open(os.path.join(tmp_dir, '[Content_Types].xml'), 'w', encoding='utf-8') as f:
f.write(content_types)
with open(os.path.join(tmp_dir, '_rels', '.rels'), 'w', encoding='utf-8') as f:
f.write(rels)
with open(os.path.join(tmp_dir, 'word', 'document.xml'), 'w', encoding='utf-8') as f:
f.write(document_xml)
with open(os.path.join(tmp_dir, 'word', 'header1.xml'), 'w', encoding='utf-8') as f:
f.write(header_part_xml)
with open(os.path.join(tmp_dir, 'word', 'styles.xml'), 'w', encoding='utf-8') as f:
f.write(styles_xml)
with open(os.path.join(tmp_dir, 'word', '_rels', 'document.xml.rels'), 'w', encoding='utf-8') as f:
f.write(doc_rels)
if os.path.exists(out_path):
os.remove(out_path)
with zipfile.ZipFile(out_path, 'w', zipfile.ZIP_DEFLATED) as zf:
for root, dirs, files in os.walk(tmp_dir):
for fn in files:
full = os.path.join(root, fn)
arc = os.path.relpath(full, tmp_dir)
zf.write(full, arc)
shutil.rmtree(tmp_dir)
return out_path
def _extract_zip(path, dest):
with zipfile.ZipFile(path, 'r') as zf:
zf.extractall(dest)
def _rezip(src_dir, out_path):
if os.path.exists(out_path):
os.remove(out_path)
with zipfile.ZipFile(out_path, 'w', zipfile.ZIP_DEFLATED) as zf:
for root, dirs, files in os.walk(src_dir):
for fn in files:
full = os.path.join(root, fn)
arc = os.path.relpath(full, src_dir)
zf.write(full, arc)
def spoil_portrait_second_section(path):
out = path + '.spoil_portrait_second_section'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
doc_path = os.path.join(tmp, 'word', 'document.xml')
with open(doc_path, 'r', encoding='utf-8') as f:
xml = f.read()
# make the final (section 2) sectPr's pgSz portrait dims and drop orient attr
xml = re.sub(r'',
'', xml)
with open(doc_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
def spoil_percentage_table_width(path):
out = path + '.spoil_percentage_table_width'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
doc_path = os.path.join(tmp, 'word', 'document.xml')
with open(doc_path, 'r', encoding='utf-8') as f:
xml = f.read()
xml = xml.replace('', '')
with open(doc_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
def spoil_header_text_wrong(path):
out = path + '.spoil_header_text_wrong'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
hdr_path = os.path.join(tmp, 'word', 'header1.xml')
with open(hdr_path, 'r', encoding='utf-8') as f:
xml = f.read()
xml = xml.replace('Training Attendance Report - Appendix', 'Training Attendance Report')
with open(hdr_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
def spoil_missing_cell_width(path):
out = path + '.spoil_missing_cell_width'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
doc_path = os.path.join(tmp, 'word', 'document.xml')
with open(doc_path, 'r', encoding='utf-8') as f:
xml = f.read()
# remove the tcW on exactly one cell (the first Item cell in header row)
xml = xml.replace(
''
'Item',
''
'Item',
1
)
with open(doc_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
def spoil_wrong_body_paragraph(path):
out = path + '.spoil_wrong_body_paragraph'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
doc_path = os.path.join(tmp, 'word', 'document.xml')
with open(doc_path, 'r', encoding='utf-8') as f:
xml = f.read()
xml = xml.replace(
'Radio channel 4 is reserved for crane operations during lifts.',
'Radio channel 4 is reserved for crane operations during lifts!'
)
with open(doc_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
def spoil_only_one_section(path):
out = path + '.spoil_only_one_section'
tmp = out + '_dir'
if os.path.exists(tmp):
shutil.rmtree(tmp)
_extract_zip(path, tmp)
doc_path = os.path.join(tmp, 'word', 'document.xml')
with open(doc_path, 'r', encoding='utf-8') as f:
xml = f.read()
# remove the paragraph that carries the section-1 sectPr, merging into a single section
xml = re.sub(r'.*?', '', xml, count=1, flags=re.S)
with open(doc_path, 'w', encoding='utf-8') as f:
f.write(xml)
_rezip(tmp, out)
shutil.rmtree(tmp)
return out
SPOILERS = [
("portrait_second_section", spoil_portrait_second_section),
("percentage_table_width", spoil_percentage_table_width),
("header_text_wrong", spoil_header_text_wrong),
("missing_cell_width", spoil_missing_cell_width),
("wrong_body_paragraph", spoil_wrong_body_paragraph),
("only_one_section", spoil_only_one_section),
]