#!/usr/bin/env python3 """Build Anki deck packages from an extracted BNetzA question catalog. Input is the directory produced by `amateurfunk_fetch.py`: data/ manifest-latest.json 2024-03-20-3-auflage/ manifest.json fragenkatalog3b.json svgs/ Output is one `.apkg` per top-level Prüfungsteil: anki/amateurfunk-technische-kenntnisse.apkg anki/amateurfunk-betriebliche-kenntnisse.apkg anki/amateurfunk-kenntnisse-von-vorschriften.apkg The script intentionally uses only the Python standard library. A modern Anki `.apkg` is a ZIP containing a SQLite `collection.anki2` database plus a `media` JSON map and the referenced media files. """ import argparse import dataclasses import hashlib import html import json import os import random import re import shutil import sqlite3 import sys import unicodedata import zipfile from datetime import datetime, timezone from pathlib import Path DEFAULT_DATA_DIR = Path("data") DEFAULT_OUT_DIR = Path("anki") EXIT_OK = 0 EXIT_ERROR = 1 FIELD_SEP = "\x1f" TOP_LEVEL_PREFIX = "Prüfungsfragen im Prüfungsteil: " CLASS_TAGS = {"1": "N", "2": "E", "3": "A"} DEFAULT_BUILD_EPOCH = 0 SAFE_INLINE_TAG_RE = re.compile(r"()", flags=re.IGNORECASE) @dataclasses.dataclass class QuestionItem: """A catalog question plus its path through the section tree.""" question: dict path: tuple[str, ...] @dataclasses.dataclass class Category: """One top-level exam part, emitted as one Anki package.""" title: str short_title: str slug: str questions: list[QuestionItem] class AnkiBuildError(Exception): """Raised when the local catalog cannot be converted into decks.""" def load_latest_catalog(data_dir: Path): """Return `(edition_dir, manifest, catalog)` from a fetch output dir.""" latest_path = data_dir / "manifest-latest.json" try: latest = json.loads(latest_path.read_text("utf-8")) slug = latest["slug"] edition_dir = data_dir / slug manifest = json.loads((edition_dir / "manifest.json").read_text("utf-8")) catalog = json.loads((edition_dir / manifest["json_filename"]).read_text("utf-8")) except (OSError, KeyError, json.JSONDecodeError) as e: raise AnkiBuildError( f"could not read fetched catalog under {data_dir}: {e}" ) from e return edition_dir, manifest, catalog def collect_categories(catalog: dict) -> list[Category]: """Split the catalog into one category per top-level section.""" sections = catalog.get("sections") if not isinstance(sections, list): raise AnkiBuildError("catalog has no top-level sections list") categories = [] for section in sections: title = section.get("title", "Unbenannte Kategorie") questions = [] _collect_questions(section, (title,), questions) short_title = _short_category_title(title) categories.append( Category( title=title, short_title=short_title, slug=slugify(short_title), questions=questions, ) ) return categories def _collect_questions(section: dict, path: tuple[str, ...], out: list[QuestionItem]): for question in section.get("questions") or []: out.append(QuestionItem(question=question, path=path)) for child in section.get("sections") or []: title = child.get("title", "Unbenannt") _collect_questions(child, path + (title,), out) def _short_category_title(title: str) -> str: if title.startswith(TOP_LEVEL_PREFIX): return title[len(TOP_LEVEL_PREFIX):] return title def slugify(value: str) -> str: """Make a readable ASCII filename slug.""" replacements = { "ä": "ae", "ö": "oe", "ü": "ue", "Ä": "Ae", "Ö": "Oe", "Ü": "Ue", "ß": "ss", } for src, dst in replacements.items(): value = value.replace(src, dst) value = unicodedata.normalize("NFKD", value) value = value.encode("ascii", "ignore").decode("ascii") value = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-").lower() return value or "unbenannt" def stable_id(namespace: str, text: str, *, digits: int = 13) -> int: """Return a deterministic positive integer ID for Anki rows.""" digest = hashlib.sha1(f"{namespace}:{text}".encode("utf-8")).hexdigest() modulus = 10 ** digits floor = 10 ** (digits - 1) return floor + (int(digest[:15], 16) % (modulus - floor)) def stable_guid(text: str) -> str: """Return a deterministic note GUID.""" return hashlib.sha1(text.encode("utf-8")).hexdigest()[:20] def checksum_sort_field(sort_field: str) -> int: """Anki stores the first 32 bits of SHA1(sort field) as `csum`.""" digest = hashlib.sha1(sort_field.encode("utf-8")).hexdigest() return int(digest[:8], 16) def randomized_answers(question: dict, seed: str): """Return shuffled answer choices and the displayed correct label.""" choices = [ {"source": "answer_a", "text": question["answer_a"], "picture": question.get("picture_a"), "correct": True}, {"source": "answer_b", "text": question["answer_b"], "picture": question.get("picture_b"), "correct": False}, {"source": "answer_c", "text": question["answer_c"], "picture": question.get("picture_c"), "correct": False}, {"source": "answer_d", "text": question["answer_d"], "picture": question.get("picture_d"), "correct": False}, ] rnd_seed = hashlib.sha256( f"{seed}:{question.get('number', '')}".encode("utf-8") ).hexdigest() rnd = random.Random(int(rnd_seed[:16], 16)) rnd.shuffle(choices) for index, choice in enumerate(choices): choice["label"] = "ABCD"[index] correct = next(choice for choice in choices if choice["correct"]) return choices, correct["label"], correct def render_question(item: QuestionItem, media: "MediaRegistry", seed: str): """Render one question as `(front_html, back_html, correct_label)`.""" question = item.question choices, correct_label, correct = randomized_answers(question, seed) number = html.escape(str(question.get("number", ""))) path = html.escape(display_path(item.path)) front_parts = [ '
', f'
{number}' f'{path}
', f'
{text_html(question.get("question", ""))}
', ] q_image = media.image_html(question.get("picture_question")) if q_image: front_parts.append(f'
{q_image}
') front_parts.append('
    ') for choice in choices: choice_image = media.image_html(choice.get("picture")) front_parts.append( '
  1. ' f'
    {text_html(choice["text"])}
    ' f'{f"
    {choice_image}
    " if choice_image else ""}' '
  2. ' ) front_parts.append("
") correct_image = media.image_html(correct.get("picture")) back_parts = [ '
', f'
Richtige Antwort: {correct_label}
', f'
{text_html(correct["text"])}
', ] if correct_image: back_parts.append(f'
{correct_image}
') back_parts.append("
") return "".join(front_parts), "".join(back_parts), correct_label def display_path(path: tuple[str, ...]) -> str: """Return the user-facing section path with the top prefix stripped.""" if not path: return "" return " / ".join([_short_category_title(path[0]), *path[1:]]) def text_html(value) -> str: """Escape text for HTML, preserve line breaks, and enable MathJax. Upstream uses bare `$...$` for inline LaTeX. Anki's built-in MathJax recognizes `\\(...\\)`, so tokenize math spans and rewrite them while still escaping HTML-sensitive characters. """ parts = [] for is_math, chunk in tokenize_dollar_math(str(value)): if is_math: escaped = html.escape(chunk) parts.append(r"\(" + escaped + r"\)") else: parts.append("
".join(escape_text_preserving_safe_tags(chunk).splitlines())) return "".join(parts) def escape_text_preserving_safe_tags(text: str) -> str: """Escape text while preserving the catalog's safe inline markup. The live catalog uses `...` to emphasize negation in question stems. Preserve only exact underline tags; escape everything else so ordinary comparisons such as `2 < 3` and unexpected HTML stay safe. """ escaped_parts = [] for part in SAFE_INLINE_TAG_RE.split(text): if SAFE_INLINE_TAG_RE.fullmatch(part): escaped_parts.append(part.lower()) else: escaped_parts.append(html.escape(part)) return "".join(escaped_parts) def tokenize_dollar_math(text: str): """Yield `(is_math, text)` chunks, converting unescaped `$...$` pairs. This intentionally handles the simple inline convention used by the BNetzA catalog. Unmatched dollars are treated as normal text. """ index = 0 while index < len(text): start = find_unescaped_dollar(text, index) if start is None: yield False, text[index:] return end = find_unescaped_dollar(text, start + 1) if end is None: yield False, text[index:] return if start > index: yield False, text[index:start] yield True, text[start + 1:end] index = end + 1 def find_unescaped_dollar(text: str, start: int): while True: pos = text.find("$", start) if pos == -1: return None backslashes = 0 cursor = pos - 1 while cursor >= 0 and text[cursor] == "\\": backslashes += 1 cursor -= 1 if backslashes % 2 == 0: return pos start = pos + 1 class MediaRegistry: """Resolve catalog image stems and collect media files for an apkg.""" def __init__(self, media_dir: Path): self.media_dir = media_dir self.by_filename: dict[str, Path] = {} self.used_paths: set[Path] = set() self.missing: list[str] = [] self._index_media_dir() def _index_media_dir(self): if not self.media_dir.exists(): return for path in self.media_dir.iterdir(): if path.is_file(): self.by_filename[path.name] = path def resolve(self, reference) -> Path | None: if not reference: return None ref = str(reference) if "/" in ref or "\\" in ref: self.missing.append(ref) return None candidates = [] if Path(ref).suffix: candidates.append(ref) else: candidates.extend([f"{ref}.svg", f"{ref}.png"]) for candidate in candidates: path = self.by_filename.get(candidate) if path is not None: self.used_paths.add(path) return path self.missing.append(ref) return None def image_html(self, reference) -> str: path = self.resolve(reference) if path is None: return "" filename = html.escape(path.name, quote=True) return f'' def build_apkg_for_category( category: Category, edition_dir: Path, out_path: Path, *, seed: str, build_epoch: int, ): """Write one category as an Anki `.apkg` file.""" tmp_dir = out_path.parent / f".{out_path.name}.tmp" db_path = tmp_dir / "collection.anki2" # Unlike the fetcher, this builder has no operator-recoverable state # in its package temp file. A stale `.apkg.tmp` is just an incomplete # output artifact, so we overwrite it on the next deterministic build. if tmp_dir.exists(): shutil.rmtree(tmp_dir) tmp_dir.mkdir(parents=True) media = MediaRegistry(edition_dir / "svgs") deck_name = f"Amateurfunk::{category.short_title}" deck_id = stable_id("deck", deck_name) model_id = stable_id("model", "Amateurfunk Multiple Choice") now = build_epoch try: notes = [] for ordinal, item in enumerate(category.questions): front, back, correct_label = render_question(item, media, seed) question = item.question number = str(question.get("number", f"q{ordinal}")) note_id = stable_id("note", f"{category.slug}:{number}") card_id = stable_id("card", f"{category.slug}:{number}") fields = [ number, category.short_title, display_path(item.path), front, back, correct_label, ] notes.append( { "note_id": note_id, "card_id": card_id, "guid": stable_guid(f"{category.slug}:{number}"), "fields": FIELD_SEP.join(fields), "sort": number, "tags": tags_for_item(item), } ) media_paths = {path.name: path for path in media.used_paths} create_collection_db( db_path, deck_id=deck_id, deck_name=deck_name, model_id=model_id, notes=notes, now=now, ) write_apkg(out_path, db_path, media_paths, build_epoch=build_epoch) finally: shutil.rmtree(tmp_dir, ignore_errors=True) return { "path": out_path, "deck": deck_name, "questions": len(category.questions), "media": len(media_paths), "missing_media": sorted(set(media.missing)), } def tags_for_item(item: QuestionItem) -> str: """Build Anki tag field with path metadata.""" question = item.question class_tag = CLASS_TAGS.get(str(question.get("class")), "unknown") tags = [ f"klasse-{class_tag}", ] tags.extend(f"pfad-{slugify(part)}" for part in item.path[1:]) return " " + " ".join(tags) + " " def create_collection_db( db_path: Path, *, deck_id: int, deck_name: str, model_id: int, notes: list[dict], now: int, ): """Create the SQLite collection.anki2 database used inside an apkg.""" conn = sqlite3.connect(db_path) try: create_schema(conn) insert_collection_metadata( conn, deck_id=deck_id, deck_name=deck_name, model_id=model_id, now=now, ) for due, note in enumerate(notes, start=1): conn.execute( """ INSERT INTO notes (id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( note["note_id"], note["guid"], model_id, now, -1, note["tags"], note["fields"], note["sort"], checksum_sort_field(note["sort"]), 0, "", ), ) conn.execute( """ INSERT INTO cards (id, nid, did, ord, mod, usn, type, queue, due, ivl, factor, reps, lapses, left, odue, odid, flags, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( note["card_id"], note["note_id"], deck_id, 0, now, -1, 0, 0, due, 0, 0, 0, 0, 0, 0, 0, 0, "", ), ) conn.commit() finally: conn.close() def create_schema(conn: sqlite3.Connection): conn.executescript( """ CREATE TABLE col ( id integer primary key, crt integer not null, mod integer not null, scm integer not null, ver integer not null, dty integer not null, usn integer not null, ls integer not null, conf text not null, models text not null, decks text not null, dconf text not null, tags text not null ); CREATE TABLE notes ( id integer primary key, guid text not null, mid integer not null, mod integer not null, usn integer not null, tags text not null, flds text not null, sfld integer not null, csum integer not null, flags integer not null, data text not null ); CREATE TABLE cards ( id integer primary key, nid integer not null, did integer not null, ord integer not null, mod integer not null, usn integer not null, type integer not null, queue integer not null, due integer not null, ivl integer not null, factor integer not null, reps integer not null, lapses integer not null, left integer not null, odue integer not null, odid integer not null, flags integer not null, data text not null ); CREATE TABLE revlog ( id integer primary key, cid integer not null, usn integer not null, ease integer not null, ivl integer not null, lastIvl integer not null, factor integer not null, time integer not null, type integer not null ); CREATE TABLE graves ( usn integer not null, oid integer not null, type integer not null ); CREATE INDEX ix_notes_usn on notes (usn); CREATE INDEX ix_cards_usn on cards (usn); CREATE INDEX ix_cards_nid on cards (nid); CREATE INDEX ix_cards_sched on cards (did, queue, due); CREATE INDEX ix_revlog_usn on revlog (usn); CREATE INDEX ix_revlog_cid on revlog (cid); """ ) def insert_collection_metadata( conn: sqlite3.Connection, *, deck_id: int, deck_name: str, model_id: int, now: int, ): conn.execute( """ INSERT INTO col (id, crt, mod, scm, ver, dty, usn, ls, conf, models, decks, dconf, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( 1, now, now * 1000, now * 1000, 11, 0, 0, 0, json.dumps(collection_conf(deck_id), separators=(",", ":")), json.dumps({str(model_id): model_json(model_id, now)}, separators=(",", ":")), json.dumps({str(deck_id): deck_json(deck_id, deck_name, now)}, separators=(",", ":")), json.dumps(default_deck_conf(now), separators=(",", ":")), "{}", ), ) def collection_conf(deck_id: int): return { "nextPos": 1, "estTimes": True, "activeDecks": [deck_id], "sortType": "noteFld", "timeLim": 0, "sortBackwards": False, "addToCur": True, "curDeck": deck_id, "newBury": True, "newSpread": 0, "dueCounts": True, "curModel": None, "collapseTime": 1200, } def deck_json(deck_id: int, deck_name: str, now: int): return { "id": deck_id, "name": deck_name, "mod": now, "usn": -1, "lrnToday": [0, 0], "revToday": [0, 0], "newToday": [0, 0], "timeToday": [0, 0], "collapsed": False, "browserCollapsed": False, "dyn": 0, "conf": 1, "desc": "", "extendNew": 0, "extendRev": 0, } def default_deck_conf(now: int): return { "1": { "id": 1, "name": "Default", "mod": now, "usn": -1, "maxTaken": 60, "autoplay": True, "timer": 0, "replayq": True, "new": { "delays": [1, 10], "ints": [1, 4, 0], "initialFactor": 2500, "separate": True, "order": 1, "perDay": 20, "bury": True, }, "rev": { "perDay": 200, "ease4": 1.3, "fuzz": 0.05, "minSpace": 1, "ivlFct": 1, "maxIvl": 36500, "bury": True, }, "lapse": { "delays": [10], "mult": 0, "minInt": 1, "leechFails": 8, "leechAction": 0, }, } } def model_json(model_id: int, now: int): fields = [ field_json("Number", 0), field_json("Category", 1), field_json("Path", 2), field_json("Front", 3), field_json("Back", 4), field_json("CorrectLabel", 5), ] return { "id": model_id, "name": "Amateurfunk Multiple Choice", "type": 0, "mod": now, "usn": -1, "sortf": 0, "did": None, "tmpls": [ { "name": "Card 1", "ord": 0, "qfmt": "{{Front}}", "afmt": "{{FrontSide}}\n
\n{{Back}}", "did": None, "bqfmt": "", "bafmt": "", } ], "flds": fields, "css": CARD_CSS, "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage[utf8]{inputenc}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\begin{document}", "latexPost": "\\end{document}", "req": [[0, "any", [3]]], "vers": [], } def field_json(name: str, ord_: int): return { "name": name, "ord": ord_, "sticky": False, "rtl": False, "font": "Arial", "size": 20, "description": "", "plainText": False, "collapsed": False, "excludeFromSearch": False, "tag": None, "preventDeletion": False, } CARD_CSS = """ .card { font-family: Arial, sans-serif; font-size: 18px; line-height: 1.45; color: #111; background: #fff; text-align: left; } .af-meta { color: #555; font-size: 13px; margin-bottom: 0.75rem; } .af-number { font-weight: bold; margin-right: 0.75rem; } .af-question { font-size: 20px; margin-bottom: 1rem; } .af-answers { padding-left: 1.7rem; } .af-answers li { margin: 0.7rem 0; } .af-media { max-width: 100%; height: auto; margin: 0.4rem 0; } .af-correct-label { font-weight: bold; color: #0b6b3a; margin-bottom: 0.5rem; } """ def write_apkg( out_path: Path, db_path: Path, media_paths: dict[str, Path], *, build_epoch: int, ): """Write Anki package ZIP with collection DB and media files.""" out_path.parent.mkdir(parents=True, exist_ok=True) tmp_path = out_path.with_suffix(out_path.suffix + ".tmp") media_map = {} try: with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: zip_writestr_fixed( zf, "collection.anki2", db_path.read_bytes(), build_epoch=build_epoch, ) for index, filename in enumerate(sorted(media_paths)): archive_name = str(index) media_map[archive_name] = filename zip_writestr_fixed( zf, archive_name, media_bytes_for_package(media_paths[filename]), build_epoch=build_epoch, ) zip_writestr_fixed( zf, "media", json.dumps(media_map, ensure_ascii=False).encode("utf-8"), build_epoch=build_epoch, ) os.replace(tmp_path, out_path) except Exception: tmp_path.unlink(missing_ok=True) raise def zip_writestr_fixed( zf: zipfile.ZipFile, name: str, payload: bytes, *, build_epoch: int, ): """Write a ZIP member with deterministic timestamp and attributes.""" info = zipfile.ZipInfo(name, zip_datetime(build_epoch)) info.compress_type = zipfile.ZIP_DEFLATED info.external_attr = 0o644 << 16 zf.writestr(info, payload) def zip_datetime(epoch: int): """Return a ZIP-compatible UTC timestamp tuple. ZIP timestamps cannot represent dates before 1980. """ safe_epoch = max(int(epoch), 315532800) value = datetime.fromtimestamp(safe_epoch, timezone.utc) return ( value.year, value.month, value.day, value.hour, value.minute, value.second, ) def media_bytes_for_package(path: Path) -> bytes: """Return media bytes as they should be stored in the Anki package. BNetzA SVGs are transparent. In Anki dark mode that makes black line drawings hard to read because the card background can be black. When packaging SVGs, add an explicit white background rectangle as the first painted element. The extracted source SVGs are left untouched. """ raw = path.read_bytes() if path.suffix.lower() != ".svg": return raw text = raw.decode("utf-8-sig", errors="replace") return svg_with_white_background(text).encode("utf-8") def svg_with_white_background(svg_text: str) -> str: """Inject a white background rect immediately after the opening svg tag.""" if 'data-af-white-background="1"' in svg_text: return svg_text match = re.search(r"]*>", svg_text, flags=re.IGNORECASE) if not match: return svg_text background = ( '' ) return svg_text[:match.end()] + background + svg_text[match.end():] def build_epoch_from_manifest(manifest: dict, override_epoch: int | None = None) -> int: """Return deterministic build timestamp from manifest or CLI override.""" if override_epoch is not None: return int(override_epoch) fetched_at = manifest.get("fetched_at") if not fetched_at: return DEFAULT_BUILD_EPOCH try: normalized = str(fetched_at).replace("Z", "+00:00") return int(datetime.fromisoformat(normalized).timestamp()) except ValueError as e: raise AnkiBuildError(f"invalid manifest fetched_at value: {fetched_at!r}") from e def build_all( data_dir: Path, out_dir: Path, *, seed: str, override_epoch: int | None = None, ): edition_dir, manifest, catalog = load_latest_catalog(data_dir) build_epoch = build_epoch_from_manifest(manifest, override_epoch) categories = collect_categories(catalog) if not categories: raise AnkiBuildError("catalog has no categories") results = [] for category in categories: out_path = out_dir / f"amateurfunk-{category.slug}.apkg" results.append( build_apkg_for_category( category, edition_dir, out_path, seed=seed, build_epoch=build_epoch, ) ) return results def _parse_args(argv): parser = argparse.ArgumentParser( description="Build Anki .apkg decks from an extracted BNetzA catalog.", ) parser.add_argument( "--data", type=Path, default=DEFAULT_DATA_DIR, help="fetch output directory containing manifest-latest.json (default: ./data)", ) parser.add_argument( "--out", type=Path, default=DEFAULT_OUT_DIR, help="output directory for .apkg files (default: ./anki)", ) parser.add_argument( "--seed", default="amateurfunk-anki-v1", help="deterministic seed for answer shuffling", ) parser.add_argument( "--epoch", type=int, default=None, help=( "override the package timestamp epoch; by default this is " "derived from manifest.json fetched_at" ), ) return parser.parse_args(argv) def main(argv=None): args = _parse_args(argv) try: results = build_all( args.data, args.out, seed=args.seed, override_epoch=args.epoch, ) except AnkiBuildError as e: print(f"error: {e}", file=sys.stderr) return EXIT_ERROR except Exception as e: # noqa: BLE001 print(f"error: failed to build Anki packages: {e}", file=sys.stderr) return EXIT_ERROR for result in results: print( f"wrote {result['path']} " f"({result['questions']} cards, {result['media']} media files)" ) if result["missing_media"]: print( f"warning: {len(result['missing_media'])} missing media reference(s) " f"in {result['deck']}", file=sys.stderr, ) return EXIT_OK if __name__ == "__main__": sys.exit(main())