from __future__ import annotations """Mutation gate for docx_tasks, forged and verified 6/6 near-misses rejected. The checker in docx_tasks.py could grade an output but nothing showed it would reject a broken one. These are the deliberate near-misses it refuses, each one verified to change the file and to be rejected. """ # ---- inlined from docx_tasks.py so this file runs on its own ---- """Real-skill task bank for the anthropics `docx` skill, graded MECHANICALLY. The skill creates documents with docx-js (Node). The model writes a JS script; the runner executes it with `docx` resolvable from external/node; Python reads the produced .docx (python-docx + raw OOXML) and grades it. No model grades anything. The skill states footguns explicitly, and the grader holds the file to them: bullets must be numbering (no literal bullet characters), table widths in DXA (not percentage), shading type CLEAR (not SOLID), US Letter when asked, landscape via section orientation. A tier that ignores the skill's own rules fails. That is the point of this bank: on `xlsx` every thinking tier was at the ceiling; `docx` has rules a careless model breaks. Three contexts: memo title heading, exact paragraphs, a real bullet list, US Letter table heading + a DXA-width table, CLEAR-shaded header row, bold total row landscape two sections, second landscape with a header and a table """ # (moved to the top of this file) import os import random import re import subprocess import sys import tempfile CONTEXTS = ["memo", "table", "landscape"] NODE_DIR = os.path.join(os.path.dirname(__file__), "..", "external", "node") TOPICS = ["Quarterly Safety Review", "Vendor Onboarding Notes", "Site Access Policy", "Fleet Maintenance Summary", "Training Attendance Report", "Incident Follow-up"] SENTENCES = ["All contractors must sign in at the front desk before entering the yard.", "The east gate will be closed for resurfacing from Monday to Wednesday.", "Please submit expense forms by the fifth business day of each month.", "Radio channel 4 is reserved for crane operations during lifts.", "The new visitor badges expire at the end of the calendar year.", "Report any damaged signage to facilities using the online form.", "Overtime requires written approval from a supervisor in advance.", "First aid kits are inspected on the first Friday of every month."] ITEMS = ["Hard hats", "Safety boots", "Hi-vis vests", "Eye protection", "Gloves", "Hearing protection", "Fall arrest harness", "Face shield"] REGIONS = ["North", "South", "East", "West", "Central"] def build(n_per: int = 8, seed: int = 31) -> list[dict]: rng = random.Random(seed) specs: list[dict] = [] for i in range(n_per): title = rng.choice(TOPICS) paras = rng.sample(SENTENCES, 3) bullets = rng.sample(ITEMS, 3) specs.append({"context": "memo", "id": f"mm{i}", "title": title, "paras": paras, "bullets": bullets, "spec": (f"Create a US Letter Word document. First a Heading 1 with exactly the text \"{title}\". " f"Then these three paragraphs, each its own paragraph, exact text:\n" + "\n".join(paras) + f"\nThen a bullet list with exactly these three items, in this order: {', '.join(bullets)}. " f"Use a real bullet list (numbering), not bullet characters typed into the text.")}) for i in range(n_per): rows = [(rng.choice(REGIONS), rng.randint(10, 400), rng.randint(1, 60)) for _ in range(rng.randint(3, 5))] title = rng.choice(TOPICS) specs.append({"context": "table", "id": f"tb{i}", "title": title, "rows": rows, "spec": (f"Create a Word document with a Heading 1 with exactly the text \"{title}\", then a table. " f"Header row, exactly: Region, Units, Incidents. Then these rows:\n" + "\n".join(f"{r}, {u}, {c}" for r, u, c in rows) + f"\nThen a final row with the text Total in the first cell, {sum(u for _, u, _ in rows)} in the second " f"and {sum(c for _, _, c in rows)} in the third; every run of text in that final row must be bold. " f"The header row cells must have a light grey shading. Table and cell widths must be set in DXA " f"with columnWidths on the table and width on every cell, and the column widths must sum to the table width.")}) for i in range(n_per): title = rng.choice(TOPICS) para = rng.choice(SENTENCES) header = f"{title} - Appendix" rows = [(rng.choice(ITEMS), rng.randint(1, 99)) for _ in range(3)] specs.append({"context": "landscape", "id": f"ls{i}", "title": title, "para": para, "header": header, "rows": rows, "spec": (f"Create a Word document with two sections. Section one, portrait: a Heading 1 with exactly the text " f"\"{title}\" and one paragraph with exactly the text \"{para}\". Section two, LANDSCAPE orientation, " f"with a page header containing exactly the text \"{header}\", and a table with header row exactly " f"Item, Count and these rows:\n" + "\n".join(f"{a}, {b}" for a, b in rows) + "\nUse DXA widths on the table and every cell.")}) rng.shuffle(specs) return specs # ---------------------------------------------------------------- run the model's script FENCE = re.compile(r"```(?:javascript|js|typescript|ts)?\s*\n(.*?)```", re.S) def extract_script(reply: str) -> str | None: m = FENCE.findall(reply or "") if m: return max(m, key=len) if "require(" in (reply or "") and "docx" in reply: return reply return None def run_script(script: str, timeout_s: int = 60) -> tuple[str | None, str]: d = tempfile.mkdtemp(prefix="sg_docx_") out = os.path.join(d, "out.docx") path = os.path.join(d, "build.js") with open(path, "w", encoding="utf-8") as fh: fh.write(script) env = {**os.environ, "OUTPUT": out, "NODE_PATH": os.path.abspath(os.path.join(NODE_DIR, "node_modules"))} try: p = subprocess.run(["node", path], cwd=d, env=env, capture_output=True, text=True, timeout=timeout_s) log = (p.stdout + p.stderr)[-2000:] if p.returncode != 0: # the error line is near the TOP of node's stderr; the tail is only the stack err = next((ln for ln in p.stderr.splitlines() if re.search(r"Error|error", ln) and "at " not in ln[:6]), "") return None, f"exit {p.returncode}: {err[:300] or log[:300]}" except subprocess.TimeoutExpired: return None, "timeout" except FileNotFoundError: return None, "node not found" return (out if os.path.exists(out) else None), log # ---------------------------------------------------------------- grading W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" BULLET_CHARS = "•●◦‣⁃·-*" def _ptext(p) -> str: return "".join(r.text for r in p.runs).strip() def _is_heading1(p) -> bool: name = (p.style.name if p.style is not None else "") or "" if name.lower().startswith("heading 1"): return True ppr = p._p.pPr if ppr is not None and ppr.pStyle is not None: return str(ppr.pStyle.val).lower() in ("heading1", "heading 1") return False def _has_numpr(p) -> bool: ppr = p._p.pPr return ppr is not None and ppr.numPr is not None def _table_widths_dxa(t) -> tuple[bool, str]: """columnWidths on the table (tblGrid gridCol) AND width on every cell, all DXA.""" tbl = t._tbl tblw = tbl.find(f"{W}tblPr/{W}tblW") if tblw is not None and tblw.get(f"{W}type") not in (None, "dxa", "auto"): return False, f"tblW type {tblw.get(f'{W}type')}" grid = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol") if g.get(f"{W}w")] if not grid: return False, "no tblGrid columnWidths" for tr in tbl.findall(f"{W}tr"): for tc in tr.findall(f"{W}tc"): tcw = tc.find(f"{W}tcPr/{W}tcW") if tcw is None: return False, "cell without width" if tcw.get(f"{W}type") not in (None, "dxa"): return False, f"cell width type {tcw.get(f'{W}type')}" if tblw is not None and tblw.get(f"{W}type") == "dxa" and tblw.get(f"{W}w"): if abs(sum(grid) - int(tblw.get(f"{W}w"))) > 2: return False, f"columns sum {sum(grid)} != table width {tblw.get(f'{W}w')}" return True, "ok" def _cell_shading(tc) -> tuple[str | None, str | None]: shd = tc._tc.find(f"{W}tcPr/{W}shd") if shd is None: return None, None return shd.get(f"{W}val"), shd.get(f"{W}fill") def grade(spec: dict, path: str | None) -> tuple[int, str]: if not path: return 0, "no file" try: import docx as pydocx d = pydocx.Document(path) except Exception as e: # noqa: BLE001 return 0, f"unreadable: {e}" ctx = spec["context"] try: body_paras = [p for p in d.paragraphs] texts = [_ptext(p) for p in body_paras] for t in texts: if "\n" in t: return 0, "literal newline in a run" if ctx == "memo": sec = d.sections[0] if abs(int(sec.page_width) - 7772400) > 20000 or abs(int(sec.page_height) - 10058400) > 20000: return 0, f"page size {sec.page_width}x{sec.page_height} not US Letter" heads = [p for p in body_paras if _is_heading1(p)] if not heads or _ptext(heads[0]) != spec["title"]: return 0, f"heading {[_ptext(h) for h in heads][:1]}" for s in spec["paras"]: if s not in texts: return 0, f"missing paragraph: {s[:30]}" found = 0 for p in body_paras: t = _ptext(p) if t in spec["bullets"]: if not _has_numpr(p): return 0, f"bullet item without numbering: {t}" found += 1 if t and t[0] in BULLET_CHARS and t[1:].strip() in spec["bullets"]: return 0, f"literal bullet character: {t[:12]!r}" if found != 3: return 0, f"found {found} of 3 list items" return 1, "ok" if ctx == "table": heads = [p for p in body_paras if _is_heading1(p)] if not heads or _ptext(heads[0]) != spec["title"]: return 0, "heading" if not d.tables: return 0, "no table" t = d.tables[0] cells = [[c.text.strip() for c in r.cells] for r in t.rows] if cells[0] != ["Region", "Units", "Incidents"]: return 0, f"header {cells[0]}" want = [[r, str(u), str(c)] for r, u, c in spec["rows"]] if cells[1:1 + len(want)] != want: return 0, f"rows {cells[1:1 + len(want)]}" tot = cells[1 + len(want)] if tot != ["Total", str(sum(u for _, u, _ in spec["rows"])), str(sum(c for _, _, c in spec["rows"]))]: return 0, f"total row {tot}" last = t.rows[1 + len(want)] for c in last.cells: for p in c.paragraphs: for r in p.runs: if r.text.strip() and not r.bold: return 0, "total row run not bold" for c in t.rows[0].cells: val, fill = _cell_shading(c) if val is None or fill in (None, "auto", "FFFFFF", "ffffff"): return 0, f"header shading missing ({val},{fill})" if val.lower() == "solid": return 0, "header shading SOLID" ok, why = _table_widths_dxa(t) if not ok: return 0, why return 1, "ok" if ctx == "landscape": if len(d.sections) < 2: return 0, f"{len(d.sections)} section(s)" s2 = d.sections[1] if int(s2.page_width) <= int(s2.page_height): return 0, "section 2 not landscape by dimensions" sectpr = s2._sectPr pgsz = sectpr.find(f"{W}pgSz") if pgsz is None or pgsz.get(f"{W}orient") != "landscape": return 0, "section 2 orient attribute not landscape" htxt = " ".join(_ptext(p) for p in s2.header.paragraphs).strip() if htxt != spec["header"]: return 0, f"header {htxt!r}" heads = [p for p in body_paras if _is_heading1(p)] if not heads or _ptext(heads[0]) != spec["title"]: return 0, "heading" if spec["para"] not in texts: return 0, "paragraph" if not d.tables: return 0, "no table" t = d.tables[0] cells = [[c.text.strip() for c in r.cells] for r in t.rows] want = [["Item", "Count"]] + [[a, str(b)] for a, b in spec["rows"]] if cells[:len(want)] != want: return 0, f"table {cells[:len(want)]}" ok, why = _table_widths_dxa(t) if not ok: return 0, why return 1, "ok" except Exception as e: # noqa: BLE001 return 0, f"grader exception: {e}" return 0, "unknown context" def user_prompt(spec: dict) -> str: return (spec["spec"] + "\n\nWrite ONE complete Node.js script using the docx package (require('docx')) that builds " "this document and writes it to the path in process.env.OUTPUT using Packer.toBuffer and fs.writeFileSync. " "Output only the script in a single ```javascript fence. Do not convert to PDF. Do not add anything the spec did not ask for.") # ---- end of docx_tasks.py ---- import shutil, zipfile, re, os def build_reference(spec, out_path): rows = spec['rows'] n_cols_1 = 2 header_widths = [4000, 2000] table_width = sum(header_widths) def esc(s): return (s.replace('&', '&').replace('<', '<').replace('>', '>') .replace('"', '"').replace("'", ''')) def para(text, style=None): pPr = f'' if style else '' return f'{pPr}{esc(text)}' def cell(text, width, bold=False, shade=None): rpr = '' if bold else '' shd = f'' if shade else '' return (f'{shd}' f'{rpr}{esc(text)}') def row(cells_texts, widths, bold=False, shade_first_row=False): cells_xml = '' for txt, w in zip(cells_texts, widths): cells_xml += cell(txt, w, bold=bold, shade=('D9D9D9' if shade_first_row else None)) return f'{cells_xml}' grid_xml = ''.join(f'' for w in header_widths) table_rows = row(['Item', 'Count'], header_widths, shade_first_row=True) for a, b in rows: table_rows += row([a, str(b)], header_widths) table_xml = (f'' f'{grid_xml}{table_rows}') # Section 1: portrait, US letter-ish dims not required here but keep sane sec1_width, sec1_height = 12240, 15840 # portrait letter sec2_width, sec2_height = 15840, 12240 # landscape letter header_text = spec['header'] header_rid = 'rId100' header_part_xml = ( '\n' '' f'{esc(header_text)}' '' ) body_p1 = para(spec['title'], style='Heading1') body_p2 = para(spec['para']) sectPr1 = (f'' f'') sectPr2 = (f'' f'' f'') # first "section" is embedded as a paragraph with sectPr in its pPr to end section 1 p1_with_sect = (f'' f'') document_xml = ( '\n' '' '' + body_p1 + body_p2 # end of section 1: a paragraph carrying sectPr for section 1 (required OOXML pattern) + '' + sectPr1 + '' + table_xml + '' + f'' + f'' + '' + '' + '' ) content_types = ( '\n' '' '' '' '' '' '' '' ) rels = ( '\n' '' '' '' ) doc_rels = ( '\n' '' f'' '' '' ) styles_xml = ( '\n' '' '' '' '' '' ) tmp_dir = out_path + '_build_tmp' if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.makedirs(os.path.join(tmp_dir, '_rels')) os.makedirs(os.path.join(tmp_dir, 'word', '_rels')) with open(os.path.join(tmp_dir, '[Content_Types].xml'), 'w', encoding='utf-8') as f: f.write(content_types) with open(os.path.join(tmp_dir, '_rels', '.rels'), 'w', encoding='utf-8') as f: f.write(rels) with open(os.path.join(tmp_dir, 'word', 'document.xml'), 'w', encoding='utf-8') as f: f.write(document_xml) with open(os.path.join(tmp_dir, 'word', 'header1.xml'), 'w', encoding='utf-8') as f: f.write(header_part_xml) with open(os.path.join(tmp_dir, 'word', 'styles.xml'), 'w', encoding='utf-8') as f: f.write(styles_xml) with open(os.path.join(tmp_dir, 'word', '_rels', 'document.xml.rels'), 'w', encoding='utf-8') as f: f.write(doc_rels) if os.path.exists(out_path): os.remove(out_path) with zipfile.ZipFile(out_path, 'w', zipfile.ZIP_DEFLATED) as zf: for root, dirs, files in os.walk(tmp_dir): for fn in files: full = os.path.join(root, fn) arc = os.path.relpath(full, tmp_dir) zf.write(full, arc) shutil.rmtree(tmp_dir) return out_path def _extract_zip(path, dest): with zipfile.ZipFile(path, 'r') as zf: zf.extractall(dest) def _rezip(src_dir, out_path): if os.path.exists(out_path): os.remove(out_path) with zipfile.ZipFile(out_path, 'w', zipfile.ZIP_DEFLATED) as zf: for root, dirs, files in os.walk(src_dir): for fn in files: full = os.path.join(root, fn) arc = os.path.relpath(full, src_dir) zf.write(full, arc) def spoil_portrait_second_section(path): out = path + '.spoil_portrait_second_section' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) doc_path = os.path.join(tmp, 'word', 'document.xml') with open(doc_path, 'r', encoding='utf-8') as f: xml = f.read() # make the final (section 2) sectPr's pgSz portrait dims and drop orient attr xml = re.sub(r'', '', xml) with open(doc_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out def spoil_percentage_table_width(path): out = path + '.spoil_percentage_table_width' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) doc_path = os.path.join(tmp, 'word', 'document.xml') with open(doc_path, 'r', encoding='utf-8') as f: xml = f.read() xml = xml.replace('', '') with open(doc_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out def spoil_header_text_wrong(path): out = path + '.spoil_header_text_wrong' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) hdr_path = os.path.join(tmp, 'word', 'header1.xml') with open(hdr_path, 'r', encoding='utf-8') as f: xml = f.read() xml = xml.replace('Training Attendance Report - Appendix', 'Training Attendance Report') with open(hdr_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out def spoil_missing_cell_width(path): out = path + '.spoil_missing_cell_width' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) doc_path = os.path.join(tmp, 'word', 'document.xml') with open(doc_path, 'r', encoding='utf-8') as f: xml = f.read() # remove the tcW on exactly one cell (the first Item cell in header row) xml = xml.replace( '' 'Item', '' 'Item', 1 ) with open(doc_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out def spoil_wrong_body_paragraph(path): out = path + '.spoil_wrong_body_paragraph' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) doc_path = os.path.join(tmp, 'word', 'document.xml') with open(doc_path, 'r', encoding='utf-8') as f: xml = f.read() xml = xml.replace( 'Radio channel 4 is reserved for crane operations during lifts.', 'Radio channel 4 is reserved for crane operations during lifts!' ) with open(doc_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out def spoil_only_one_section(path): out = path + '.spoil_only_one_section' tmp = out + '_dir' if os.path.exists(tmp): shutil.rmtree(tmp) _extract_zip(path, tmp) doc_path = os.path.join(tmp, 'word', 'document.xml') with open(doc_path, 'r', encoding='utf-8') as f: xml = f.read() # remove the paragraph that carries the section-1 sectPr, merging into a single section xml = re.sub(r'.*?', '', xml, count=1, flags=re.S) with open(doc_path, 'w', encoding='utf-8') as f: f.write(xml) _rezip(tmp, out) shutil.rmtree(tmp) return out SPOILERS = [ ("portrait_second_section", spoil_portrait_second_section), ("percentage_table_width", spoil_percentage_table_width), ("header_text_wrong", spoil_header_text_wrong), ("missing_cell_width", spoil_missing_cell_width), ("wrong_body_paragraph", spoil_wrong_body_paragraph), ("only_one_section", spoil_only_one_section), ]