diff --git a/.gitignore b/.gitignore index 5077ca6..357c6cf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .pytest_cache/ __pycache__/ data/ +anki/ diff --git a/amateurfunk_anki.py b/amateurfunk_anki.py new file mode 100644 index 0000000..b14d114 --- /dev/null +++ b/amateurfunk_anki.py @@ -0,0 +1,995 @@ +#!/usr/bin/env python3 +"""Build Anki deck packages from an extracted BNetzA question catalog. + +Input is the directory produced by `amateurfunk_fetch.py`: + + data/ + manifest-latest.json + 2024-03-20-3-auflage/ + manifest.json + fragenkatalog3b.json + svgs/ + +Output is one `.apkg` per top-level Prüfungsteil: + + anki/amateurfunk-technische-kenntnisse.apkg + anki/amateurfunk-betriebliche-kenntnisse.apkg + anki/amateurfunk-kenntnisse-von-vorschriften.apkg + +The script intentionally uses only the Python standard library. A modern +Anki `.apkg` is a ZIP containing a SQLite `collection.anki2` database +plus a `media` JSON map and the referenced media files. +""" + +import argparse +import dataclasses +import hashlib +import html +import json +import os +import random +import re +import shutil +import sqlite3 +import sys +import unicodedata +import zipfile +from datetime import datetime, timezone +from pathlib import Path + + +DEFAULT_DATA_DIR = Path("data") +DEFAULT_OUT_DIR = Path("anki") +EXIT_OK = 0 +EXIT_ERROR = 1 + +FIELD_SEP = "\x1f" +TOP_LEVEL_PREFIX = "Prüfungsfragen im Prüfungsteil: " +CLASS_TAGS = {"1": "N", "2": "E", "3": "A"} +DEFAULT_BUILD_EPOCH = 0 +SAFE_INLINE_TAG_RE = re.compile(r"()", flags=re.IGNORECASE) + + +@dataclasses.dataclass +class QuestionItem: + """A catalog question plus its path through the section tree.""" + + question: dict + path: tuple[str, ...] + + +@dataclasses.dataclass +class Category: + """One top-level exam part, emitted as one Anki package.""" + + title: str + short_title: str + slug: str + questions: list[QuestionItem] + + +class AnkiBuildError(Exception): + """Raised when the local catalog cannot be converted into decks.""" + + +def load_latest_catalog(data_dir: Path): + """Return `(edition_dir, manifest, catalog)` from a fetch output dir.""" + latest_path = data_dir / "manifest-latest.json" + try: + latest = json.loads(latest_path.read_text("utf-8")) + slug = latest["slug"] + edition_dir = data_dir / slug + manifest = json.loads((edition_dir / "manifest.json").read_text("utf-8")) + catalog = json.loads((edition_dir / manifest["json_filename"]).read_text("utf-8")) + except (OSError, KeyError, json.JSONDecodeError) as e: + raise AnkiBuildError( + f"could not read fetched catalog under {data_dir}: {e}" + ) from e + return edition_dir, manifest, catalog + + +def collect_categories(catalog: dict) -> list[Category]: + """Split the catalog into one category per top-level section.""" + sections = catalog.get("sections") + if not isinstance(sections, list): + raise AnkiBuildError("catalog has no top-level sections list") + + categories = [] + for section in sections: + title = section.get("title", "Unbenannte Kategorie") + questions = [] + _collect_questions(section, (title,), questions) + short_title = _short_category_title(title) + categories.append( + Category( + title=title, + short_title=short_title, + slug=slugify(short_title), + questions=questions, + ) + ) + return categories + + +def _collect_questions(section: dict, path: tuple[str, ...], out: list[QuestionItem]): + for question in section.get("questions") or []: + out.append(QuestionItem(question=question, path=path)) + for child in section.get("sections") or []: + title = child.get("title", "Unbenannt") + _collect_questions(child, path + (title,), out) + + +def _short_category_title(title: str) -> str: + if title.startswith(TOP_LEVEL_PREFIX): + return title[len(TOP_LEVEL_PREFIX):] + return title + + +def slugify(value: str) -> str: + """Make a readable ASCII filename slug.""" + replacements = { + "ä": "ae", + "ö": "oe", + "ü": "ue", + "Ä": "Ae", + "Ö": "Oe", + "Ü": "Ue", + "ß": "ss", + } + for src, dst in replacements.items(): + value = value.replace(src, dst) + value = unicodedata.normalize("NFKD", value) + value = value.encode("ascii", "ignore").decode("ascii") + value = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-").lower() + return value or "unbenannt" + + +def stable_id(namespace: str, text: str, *, digits: int = 13) -> int: + """Return a deterministic positive integer ID for Anki rows.""" + digest = hashlib.sha1(f"{namespace}:{text}".encode("utf-8")).hexdigest() + modulus = 10 ** digits + floor = 10 ** (digits - 1) + return floor + (int(digest[:15], 16) % (modulus - floor)) + + +def stable_guid(text: str) -> str: + """Return a deterministic note GUID.""" + return hashlib.sha1(text.encode("utf-8")).hexdigest()[:20] + + +def checksum_sort_field(sort_field: str) -> int: + """Anki stores the first 32 bits of SHA1(sort field) as `csum`.""" + digest = hashlib.sha1(sort_field.encode("utf-8")).hexdigest() + return int(digest[:8], 16) + + +def randomized_answers(question: dict, seed: str): + """Return shuffled answer choices and the displayed correct label.""" + choices = [ + {"source": "answer_a", "text": question["answer_a"], "picture": question.get("picture_a"), "correct": True}, + {"source": "answer_b", "text": question["answer_b"], "picture": question.get("picture_b"), "correct": False}, + {"source": "answer_c", "text": question["answer_c"], "picture": question.get("picture_c"), "correct": False}, + {"source": "answer_d", "text": question["answer_d"], "picture": question.get("picture_d"), "correct": False}, + ] + rnd_seed = hashlib.sha256( + f"{seed}:{question.get('number', '')}".encode("utf-8") + ).hexdigest() + rnd = random.Random(int(rnd_seed[:16], 16)) + rnd.shuffle(choices) + for index, choice in enumerate(choices): + choice["label"] = "ABCD"[index] + correct = next(choice for choice in choices if choice["correct"]) + return choices, correct["label"], correct + + +def render_question(item: QuestionItem, media: "MediaRegistry", seed: str): + """Render one question as `(front_html, back_html, correct_label)`.""" + question = item.question + choices, correct_label, correct = randomized_answers(question, seed) + number = html.escape(str(question.get("number", ""))) + path = html.escape(display_path(item.path)) + + front_parts = [ + '
', + f'
{number}' + f'{path}
', + f'
{text_html(question.get("question", ""))}
', + ] + q_image = media.image_html(question.get("picture_question")) + if q_image: + front_parts.append(f'
{q_image}
') + + front_parts.append('
    ') + for choice in choices: + choice_image = media.image_html(choice.get("picture")) + front_parts.append( + '
  1. ' + f'
    {text_html(choice["text"])}
    ' + f'{f"
    {choice_image}
    " if choice_image else ""}' + '
  2. ' + ) + front_parts.append("
") + + correct_image = media.image_html(correct.get("picture")) + back_parts = [ + '
', + f'
Richtige Antwort: {correct_label}
', + f'
{text_html(correct["text"])}
', + ] + if correct_image: + back_parts.append(f'
{correct_image}
') + back_parts.append("
") + return "".join(front_parts), "".join(back_parts), correct_label + + +def display_path(path: tuple[str, ...]) -> str: + """Return the user-facing section path with the top prefix stripped.""" + if not path: + return "" + return " / ".join([_short_category_title(path[0]), *path[1:]]) + + +def text_html(value) -> str: + """Escape text for HTML, preserve line breaks, and enable MathJax. + + Upstream uses bare `$...$` for inline LaTeX. Anki's built-in MathJax + recognizes `\\(...\\)`, so tokenize math spans and rewrite them while + still escaping HTML-sensitive characters. + """ + parts = [] + for is_math, chunk in tokenize_dollar_math(str(value)): + if is_math: + escaped = html.escape(chunk) + parts.append(r"\(" + escaped + r"\)") + else: + parts.append("
".join(escape_text_preserving_safe_tags(chunk).splitlines())) + return "".join(parts) + + +def escape_text_preserving_safe_tags(text: str) -> str: + """Escape text while preserving the catalog's safe inline markup. + + The live catalog uses `...` to emphasize negation in question + stems. Preserve only exact underline tags; escape everything else so + ordinary comparisons such as `2 < 3` and unexpected HTML stay safe. + """ + escaped_parts = [] + for part in SAFE_INLINE_TAG_RE.split(text): + if SAFE_INLINE_TAG_RE.fullmatch(part): + escaped_parts.append(part.lower()) + else: + escaped_parts.append(html.escape(part)) + return "".join(escaped_parts) + + +def tokenize_dollar_math(text: str): + """Yield `(is_math, text)` chunks, converting unescaped `$...$` pairs. + + This intentionally handles the simple inline convention used by the + BNetzA catalog. Unmatched dollars are treated as normal text. + """ + index = 0 + while index < len(text): + start = find_unescaped_dollar(text, index) + if start is None: + yield False, text[index:] + return + end = find_unescaped_dollar(text, start + 1) + if end is None: + yield False, text[index:] + return + if start > index: + yield False, text[index:start] + yield True, text[start + 1:end] + index = end + 1 + + +def find_unescaped_dollar(text: str, start: int): + while True: + pos = text.find("$", start) + if pos == -1: + return None + backslashes = 0 + cursor = pos - 1 + while cursor >= 0 and text[cursor] == "\\": + backslashes += 1 + cursor -= 1 + if backslashes % 2 == 0: + return pos + start = pos + 1 + + +class MediaRegistry: + """Resolve catalog image stems and collect media files for an apkg.""" + + def __init__(self, media_dir: Path): + self.media_dir = media_dir + self.by_filename: dict[str, Path] = {} + self.used_paths: set[Path] = set() + self.missing: list[str] = [] + self._index_media_dir() + + def _index_media_dir(self): + if not self.media_dir.exists(): + return + for path in self.media_dir.iterdir(): + if path.is_file(): + self.by_filename[path.name] = path + + def resolve(self, reference) -> Path | None: + if not reference: + return None + ref = str(reference) + if "/" in ref or "\\" in ref: + self.missing.append(ref) + return None + + candidates = [] + if Path(ref).suffix: + candidates.append(ref) + else: + candidates.extend([f"{ref}.svg", f"{ref}.png"]) + for candidate in candidates: + path = self.by_filename.get(candidate) + if path is not None: + self.used_paths.add(path) + return path + self.missing.append(ref) + return None + + def image_html(self, reference) -> str: + path = self.resolve(reference) + if path is None: + return "" + filename = html.escape(path.name, quote=True) + return f'' + + +def build_apkg_for_category( + category: Category, + edition_dir: Path, + out_path: Path, + *, + seed: str, + build_epoch: int, +): + """Write one category as an Anki `.apkg` file.""" + tmp_dir = out_path.parent / f".{out_path.name}.tmp" + db_path = tmp_dir / "collection.anki2" + # Unlike the fetcher, this builder has no operator-recoverable state + # in its package temp file. A stale `.apkg.tmp` is just an incomplete + # output artifact, so we overwrite it on the next deterministic build. + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + media = MediaRegistry(edition_dir / "svgs") + + deck_name = f"Amateurfunk::{category.short_title}" + deck_id = stable_id("deck", deck_name) + model_id = stable_id("model", "Amateurfunk Multiple Choice") + now = build_epoch + + try: + notes = [] + for ordinal, item in enumerate(category.questions): + front, back, correct_label = render_question(item, media, seed) + question = item.question + number = str(question.get("number", f"q{ordinal}")) + note_id = stable_id("note", f"{category.slug}:{number}") + card_id = stable_id("card", f"{category.slug}:{number}") + fields = [ + number, + category.short_title, + display_path(item.path), + front, + back, + correct_label, + ] + notes.append( + { + "note_id": note_id, + "card_id": card_id, + "guid": stable_guid(f"{category.slug}:{number}"), + "fields": FIELD_SEP.join(fields), + "sort": number, + "tags": tags_for_item(item), + } + ) + + media_paths = {path.name: path for path in media.used_paths} + + create_collection_db( + db_path, + deck_id=deck_id, + deck_name=deck_name, + model_id=model_id, + notes=notes, + now=now, + ) + write_apkg(out_path, db_path, media_paths, build_epoch=build_epoch) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + return { + "path": out_path, + "deck": deck_name, + "questions": len(category.questions), + "media": len(media_paths), + "missing_media": sorted(set(media.missing)), + } + + +def tags_for_item(item: QuestionItem) -> str: + """Build Anki tag field with path metadata.""" + question = item.question + class_tag = CLASS_TAGS.get(str(question.get("class")), "unknown") + tags = [ + f"klasse-{class_tag}", + ] + tags.extend(f"pfad-{slugify(part)}" for part in item.path[1:]) + return " " + " ".join(tags) + " " + + +def create_collection_db( + db_path: Path, + *, + deck_id: int, + deck_name: str, + model_id: int, + notes: list[dict], + now: int, +): + """Create the SQLite collection.anki2 database used inside an apkg.""" + conn = sqlite3.connect(db_path) + try: + create_schema(conn) + insert_collection_metadata( + conn, + deck_id=deck_id, + deck_name=deck_name, + model_id=model_id, + now=now, + ) + for due, note in enumerate(notes, start=1): + conn.execute( + """ + INSERT INTO notes + (id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + note["note_id"], + note["guid"], + model_id, + now, + -1, + note["tags"], + note["fields"], + note["sort"], + checksum_sort_field(note["sort"]), + 0, + "", + ), + ) + conn.execute( + """ + INSERT INTO cards + (id, nid, did, ord, mod, usn, type, queue, due, ivl, factor, + reps, lapses, left, odue, odid, flags, data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + note["card_id"], + note["note_id"], + deck_id, + 0, + now, + -1, + 0, + 0, + due, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + "", + ), + ) + conn.commit() + finally: + conn.close() + + +def create_schema(conn: sqlite3.Connection): + conn.executescript( + """ + CREATE TABLE col ( + id integer primary key, + crt integer not null, + mod integer not null, + scm integer not null, + ver integer not null, + dty integer not null, + usn integer not null, + ls integer not null, + conf text not null, + models text not null, + decks text not null, + dconf text not null, + tags text not null + ); + CREATE TABLE notes ( + id integer primary key, + guid text not null, + mid integer not null, + mod integer not null, + usn integer not null, + tags text not null, + flds text not null, + sfld integer not null, + csum integer not null, + flags integer not null, + data text not null + ); + CREATE TABLE cards ( + id integer primary key, + nid integer not null, + did integer not null, + ord integer not null, + mod integer not null, + usn integer not null, + type integer not null, + queue integer not null, + due integer not null, + ivl integer not null, + factor integer not null, + reps integer not null, + lapses integer not null, + left integer not null, + odue integer not null, + odid integer not null, + flags integer not null, + data text not null + ); + CREATE TABLE revlog ( + id integer primary key, + cid integer not null, + usn integer not null, + ease integer not null, + ivl integer not null, + lastIvl integer not null, + factor integer not null, + time integer not null, + type integer not null + ); + CREATE TABLE graves ( + usn integer not null, + oid integer not null, + type integer not null + ); + CREATE INDEX ix_notes_usn on notes (usn); + CREATE INDEX ix_cards_usn on cards (usn); + CREATE INDEX ix_cards_nid on cards (nid); + CREATE INDEX ix_cards_sched on cards (did, queue, due); + CREATE INDEX ix_revlog_usn on revlog (usn); + CREATE INDEX ix_revlog_cid on revlog (cid); + """ + ) + + +def insert_collection_metadata( + conn: sqlite3.Connection, + *, + deck_id: int, + deck_name: str, + model_id: int, + now: int, +): + conn.execute( + """ + INSERT INTO col + (id, crt, mod, scm, ver, dty, usn, ls, conf, models, decks, dconf, tags) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 1, + now, + now * 1000, + now * 1000, + 11, + 0, + 0, + 0, + json.dumps(collection_conf(deck_id), separators=(",", ":")), + json.dumps({str(model_id): model_json(model_id, now)}, separators=(",", ":")), + json.dumps({str(deck_id): deck_json(deck_id, deck_name, now)}, separators=(",", ":")), + json.dumps(default_deck_conf(now), separators=(",", ":")), + "{}", + ), + ) + + +def collection_conf(deck_id: int): + return { + "nextPos": 1, + "estTimes": True, + "activeDecks": [deck_id], + "sortType": "noteFld", + "timeLim": 0, + "sortBackwards": False, + "addToCur": True, + "curDeck": deck_id, + "newBury": True, + "newSpread": 0, + "dueCounts": True, + "curModel": None, + "collapseTime": 1200, + } + + +def deck_json(deck_id: int, deck_name: str, now: int): + return { + "id": deck_id, + "name": deck_name, + "mod": now, + "usn": -1, + "lrnToday": [0, 0], + "revToday": [0, 0], + "newToday": [0, 0], + "timeToday": [0, 0], + "collapsed": False, + "browserCollapsed": False, + "dyn": 0, + "conf": 1, + "desc": "", + "extendNew": 0, + "extendRev": 0, + } + + +def default_deck_conf(now: int): + return { + "1": { + "id": 1, + "name": "Default", + "mod": now, + "usn": -1, + "maxTaken": 60, + "autoplay": True, + "timer": 0, + "replayq": True, + "new": { + "delays": [1, 10], + "ints": [1, 4, 0], + "initialFactor": 2500, + "separate": True, + "order": 1, + "perDay": 20, + "bury": True, + }, + "rev": { + "perDay": 200, + "ease4": 1.3, + "fuzz": 0.05, + "minSpace": 1, + "ivlFct": 1, + "maxIvl": 36500, + "bury": True, + }, + "lapse": { + "delays": [10], + "mult": 0, + "minInt": 1, + "leechFails": 8, + "leechAction": 0, + }, + } + } + + +def model_json(model_id: int, now: int): + fields = [ + field_json("Number", 0), + field_json("Category", 1), + field_json("Path", 2), + field_json("Front", 3), + field_json("Back", 4), + field_json("CorrectLabel", 5), + ] + return { + "id": model_id, + "name": "Amateurfunk Multiple Choice", + "type": 0, + "mod": now, + "usn": -1, + "sortf": 0, + "did": None, + "tmpls": [ + { + "name": "Card 1", + "ord": 0, + "qfmt": "{{Front}}", + "afmt": "{{FrontSide}}\n
\n{{Back}}", + "did": None, + "bqfmt": "", + "bafmt": "", + } + ], + "flds": fields, + "css": CARD_CSS, + "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage[utf8]{inputenc}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\begin{document}", + "latexPost": "\\end{document}", + "req": [[0, "any", [3]]], + "vers": [], + } + + +def field_json(name: str, ord_: int): + return { + "name": name, + "ord": ord_, + "sticky": False, + "rtl": False, + "font": "Arial", + "size": 20, + "description": "", + "plainText": False, + "collapsed": False, + "excludeFromSearch": False, + "tag": None, + "preventDeletion": False, + } + + +CARD_CSS = """ +.card { + font-family: Arial, sans-serif; + font-size: 18px; + line-height: 1.45; + color: #111; + background: #fff; + text-align: left; +} +.af-meta { + color: #555; + font-size: 13px; + margin-bottom: 0.75rem; +} +.af-number { + font-weight: bold; + margin-right: 0.75rem; +} +.af-question { + font-size: 20px; + margin-bottom: 1rem; +} +.af-answers { + padding-left: 1.7rem; +} +.af-answers li { + margin: 0.7rem 0; +} +.af-media { + max-width: 100%; + height: auto; + margin: 0.4rem 0; +} +.af-correct-label { + font-weight: bold; + color: #0b6b3a; + margin-bottom: 0.5rem; +} +""" + + +def write_apkg( + out_path: Path, + db_path: Path, + media_paths: dict[str, Path], + *, + build_epoch: int, +): + """Write Anki package ZIP with collection DB and media files.""" + out_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = out_path.with_suffix(out_path.suffix + ".tmp") + media_map = {} + try: + with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zip_writestr_fixed( + zf, + "collection.anki2", + db_path.read_bytes(), + build_epoch=build_epoch, + ) + for index, filename in enumerate(sorted(media_paths)): + archive_name = str(index) + media_map[archive_name] = filename + zip_writestr_fixed( + zf, + archive_name, + media_bytes_for_package(media_paths[filename]), + build_epoch=build_epoch, + ) + zip_writestr_fixed( + zf, + "media", + json.dumps(media_map, ensure_ascii=False).encode("utf-8"), + build_epoch=build_epoch, + ) + os.replace(tmp_path, out_path) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + +def zip_writestr_fixed( + zf: zipfile.ZipFile, + name: str, + payload: bytes, + *, + build_epoch: int, +): + """Write a ZIP member with deterministic timestamp and attributes.""" + info = zipfile.ZipInfo(name, zip_datetime(build_epoch)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + zf.writestr(info, payload) + + +def zip_datetime(epoch: int): + """Return a ZIP-compatible UTC timestamp tuple. + + ZIP timestamps cannot represent dates before 1980. + """ + safe_epoch = max(int(epoch), 315532800) + value = datetime.fromtimestamp(safe_epoch, timezone.utc) + return ( + value.year, + value.month, + value.day, + value.hour, + value.minute, + value.second, + ) + + +def media_bytes_for_package(path: Path) -> bytes: + """Return media bytes as they should be stored in the Anki package. + + BNetzA SVGs are transparent. In Anki dark mode that makes black line + drawings hard to read because the card background can be black. When + packaging SVGs, add an explicit white background rectangle as the + first painted element. The extracted source SVGs are left untouched. + """ + raw = path.read_bytes() + if path.suffix.lower() != ".svg": + return raw + text = raw.decode("utf-8-sig", errors="replace") + return svg_with_white_background(text).encode("utf-8") + + +def svg_with_white_background(svg_text: str) -> str: + """Inject a white background rect immediately after the opening svg tag.""" + if 'data-af-white-background="1"' in svg_text: + return svg_text + match = re.search(r"]*>", svg_text, flags=re.IGNORECASE) + if not match: + return svg_text + background = ( + '' + ) + return svg_text[:match.end()] + background + svg_text[match.end():] + + +def build_epoch_from_manifest(manifest: dict, override_epoch: int | None = None) -> int: + """Return deterministic build timestamp from manifest or CLI override.""" + if override_epoch is not None: + return int(override_epoch) + fetched_at = manifest.get("fetched_at") + if not fetched_at: + return DEFAULT_BUILD_EPOCH + try: + normalized = str(fetched_at).replace("Z", "+00:00") + return int(datetime.fromisoformat(normalized).timestamp()) + except ValueError as e: + raise AnkiBuildError(f"invalid manifest fetched_at value: {fetched_at!r}") from e + + +def build_all( + data_dir: Path, + out_dir: Path, + *, + seed: str, + override_epoch: int | None = None, +): + edition_dir, manifest, catalog = load_latest_catalog(data_dir) + build_epoch = build_epoch_from_manifest(manifest, override_epoch) + categories = collect_categories(catalog) + if not categories: + raise AnkiBuildError("catalog has no categories") + + results = [] + for category in categories: + out_path = out_dir / f"amateurfunk-{category.slug}.apkg" + results.append( + build_apkg_for_category( + category, + edition_dir, + out_path, + seed=seed, + build_epoch=build_epoch, + ) + ) + return results + + +def _parse_args(argv): + parser = argparse.ArgumentParser( + description="Build Anki .apkg decks from an extracted BNetzA catalog.", + ) + parser.add_argument( + "--data", + type=Path, + default=DEFAULT_DATA_DIR, + help="fetch output directory containing manifest-latest.json (default: ./data)", + ) + parser.add_argument( + "--out", + type=Path, + default=DEFAULT_OUT_DIR, + help="output directory for .apkg files (default: ./anki)", + ) + parser.add_argument( + "--seed", + default="amateurfunk-anki-v1", + help="deterministic seed for answer shuffling", + ) + parser.add_argument( + "--epoch", + type=int, + default=None, + help=( + "override the package timestamp epoch; by default this is " + "derived from manifest.json fetched_at" + ), + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = _parse_args(argv) + try: + results = build_all( + args.data, + args.out, + seed=args.seed, + override_epoch=args.epoch, + ) + except AnkiBuildError as e: + print(f"error: {e}", file=sys.stderr) + return EXIT_ERROR + except Exception as e: # noqa: BLE001 + print(f"error: failed to build Anki packages: {e}", file=sys.stderr) + return EXIT_ERROR + + for result in results: + print( + f"wrote {result['path']} " + f"({result['questions']} cards, {result['media']} media files)" + ) + if result["missing_media"]: + print( + f"warning: {len(result['missing_media'])} missing media reference(s) " + f"in {result['deck']}", + file=sys.stderr, + ) + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_amateurfunk_anki.py b/test_amateurfunk_anki.py new file mode 100644 index 0000000..7960ceb --- /dev/null +++ b/test_amateurfunk_anki.py @@ -0,0 +1,274 @@ +"""Tests for amateurfunk_anki. + +These tests inspect the generated `.apkg` package structure directly: +ZIP entries, media map, and the SQLite collection database. They do not +require Anki itself to be installed. +""" +from __future__ import annotations + +import json +import sqlite3 +import tempfile +import zipfile +from pathlib import Path + +import unittest + +import amateurfunk_anki as aa + + +def make_catalog(): + return { + "metadata": { + "edition": "3. Auflage, März 2024", + "issued_on": "2024-03-20", + "valid_from": "2024-06-24", + "license": "DL-DE->BY-2.0", + }, + "sections": [ + { + "title": "Prüfungsfragen im Prüfungsteil: Technische Kenntnisse", + "sections": [ + { + "title": "Grundlagen", + "questions": [ + question("NA101", "1", "Was ist A?", "NA101_q"), + question("EA202", "2", "Was ist E?", None), + ], + } + ], + }, + { + "title": "Prüfungsfragen im Prüfungsteil: Betriebliche Kenntnisse", + "questions": [ + question("BA101", "1", "Betrieb?", "BA101_q"), + ], + }, + { + "title": "Prüfungsfragen im Prüfungsteil: Kenntnisse von Vorschriften", + "questions": [ + question("VA101", "1", "Vorschrift?", None), + ], + }, + ], + } + + +def question(number, klass, prompt, picture): + data = { + "number": number, + "class": klass, + "question": prompt, + "answer_a": f"{number} richtig", + "answer_b": f"{number} falsch B", + "answer_c": f"{number} falsch C", + "answer_d": f"{number} falsch D", + } + if picture: + data["picture_question"] = picture + return data + + +def make_fetched_data(root: Path): + data_dir = root / "data" + edition_dir = data_dir / "2024-03-20-3-auflage" + media_dir = edition_dir / "svgs" + media_dir.mkdir(parents=True) + (media_dir / "NA101_q.svg").write_text("na", encoding="utf-8") + (media_dir / "BA101_q.svg").write_text("ba", encoding="utf-8") + (media_dir / "NA101_a.svg").write_text("answer-a", encoding="utf-8") + + catalog = make_catalog() + (edition_dir / "fragenkatalog3b.json").write_text( + json.dumps(catalog, ensure_ascii=False), + encoding="utf-8", + ) + (edition_dir / "manifest.json").write_text( + json.dumps( + { + "slug": "2024-03-20-3-auflage", + "json_filename": "fragenkatalog3b.json", + "fetched_at": "2026-05-20T12:00:00+00:00", + "metadata": catalog["metadata"], + } + ), + encoding="utf-8", + ) + (data_dir / "manifest-latest.json").write_text( + json.dumps({"slug": "2024-03-20-3-auflage"}), + encoding="utf-8", + ) + return data_dir + + +def extract_collection(apkg_path: Path, tmp: Path): + with zipfile.ZipFile(apkg_path) as zf: + zf.extract("collection.anki2", tmp) + media = json.loads(zf.read("media").decode("utf-8")) + names = set(zf.namelist()) + return tmp / "collection.anki2", media, names + + +class TestAnkiBuild(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.data_dir = make_fetched_data(self.root) + self.out_dir = self.root / "anki" + + def test_builds_one_apkg_per_top_level_category(self): + results = aa.build_all(self.data_dir, self.out_dir, seed="test-seed") + paths = sorted(path.name for path in self.out_dir.glob("*.apkg")) + self.assertEqual( + paths, + [ + "amateurfunk-betriebliche-kenntnisse.apkg", + "amateurfunk-kenntnisse-von-vorschriften.apkg", + "amateurfunk-technische-kenntnisse.apkg", + ], + ) + self.assertEqual([r["questions"] for r in results], [2, 1, 1]) + + def test_apkg_contains_notes_cards_and_media(self): + aa.build_all(self.data_dir, self.out_dir, seed="test-seed") + apkg = self.out_dir / "amateurfunk-technische-kenntnisse.apkg" + db_path, media, names = extract_collection(apkg, self.root) + + self.assertIn("collection.anki2", names) + self.assertIn("media", names) + self.assertEqual(set(media.values()), {"NA101_q.svg"}) + self.assertIn("0", names) + with zipfile.ZipFile(apkg) as zf: + packaged_svg = zf.read("0").decode("utf-8") + self.assertIn('data-af-white-background="1"', packaged_svg) + self.assertIn('fill="#fff"', packaged_svg) + self.assertLess( + packaged_svg.index('data-af-white-background="1"'), + packaged_svg.index("na"), + ) + + conn = sqlite3.connect(db_path) + try: + note_count = conn.execute("select count(*) from notes").fetchone()[0] + card_count = conn.execute("select count(*) from cards").fetchone()[0] + self.assertEqual(note_count, 2) + self.assertEqual(card_count, 2) + fields = [row[0] for row in conn.execute("select flds from notes")] + finally: + conn.close() + + joined = "\n".join(fields) + self.assertIn('src="NA101_q.svg"', joined) + self.assertIn("Richtige Antwort:", joined) + self.assertIn("NA101 richtig", joined) + self.assertIn("Technische Kenntnisse / Grundlagen", joined) + self.assertNotIn("Prüfungsfragen im Prüfungsteil", joined) + + def test_answers_are_shuffled_and_back_reveals_display_label(self): + item = aa.QuestionItem( + question=question("TEST123", "1", "Prompt?", None), + path=("Prüfungsfragen im Prüfungsteil: Technische Kenntnisse", "Leaf"), + ) + media = aa.MediaRegistry(self.root / "missing") + front, back, correct_label = aa.render_question(item, media, "seed") + + self.assertIn("Richtige Antwort:", back) + self.assertIn(f"Richtige Antwort: {correct_label}", back) + self.assertIn("TEST123 richtig", back) + + # The answer texts should not be in their upstream A/B/C/D order. + positions = [ + front.index("TEST123 richtig"), + front.index("TEST123 falsch B"), + front.index("TEST123 falsch C"), + front.index("TEST123 falsch D"), + ] + self.assertNotEqual(positions, sorted(positions)) + + def test_latex_is_converted_to_mathjax_and_text_is_escaped(self): + rendered = aa.text_html("Leistung < $P = U \\cdot I$ & $a < b$") + self.assertIn("Leistung < ", rendered) + self.assertIn(r"\(P = U \cdot I\)", rendered) + self.assertIn(r"\(a < b\)", rendered) + self.assertIn(" & ", rendered) + self.assertNotIn("$P =", rendered) + + def test_catalog_underline_markup_is_preserved_safely(self): + rendered = aa.text_html("foo nicht 2 < 3 ") + self.assertIn("nicht", rendered) + self.assertIn("2 < 3", rendered) + self.assertIn("<script>x</script>", rendered) + + def test_underline_inside_math_is_escaped_not_rendered_as_html(self): + rendered = aa.text_html("$x$") + self.assertEqual(rendered, r"\(<u>x</u>\)") + + def test_tags_use_license_letters_and_skip_singleton_number_tag(self): + item = aa.QuestionItem( + question=question("NA101", "1", "Prompt?", None), + path=("Prüfungsfragen im Prüfungsteil: Technische Kenntnisse", "Leaf"), + ) + tags = aa.tags_for_item(item) + self.assertIn(" klasse-N ", tags) + self.assertNotIn(" klasse-1 ", tags) + self.assertNotIn("nummer-NA101", tags) + + def test_breadcrumb_strips_verbose_pruefungsteil_prefix(self): + item = aa.QuestionItem( + question=question("TEST123", "1", "Prompt?", None), + path=("Prüfungsfragen im Prüfungsteil: Technische Kenntnisse", "Leaf"), + ) + front, _back, _label = aa.render_question( + item, + aa.MediaRegistry(self.root / "missing"), + "seed", + ) + self.assertIn("Technische Kenntnisse / Leaf", front) + self.assertNotIn("Prüfungsfragen im Prüfungsteil", front) + + def test_answer_choice_pictures_follow_shuffle(self): + media_dir = self.root / "media" + media_dir.mkdir() + (media_dir / "TEST123_a.svg").write_text("a", encoding="utf-8") + (media_dir / "TEST123_b.svg").write_text("b", encoding="utf-8") + q = question("TEST123", "1", "Prompt?", None) + q["picture_a"] = "TEST123_a" + q["picture_b"] = "TEST123_b" + item = aa.QuestionItem( + question=q, + path=("Prüfungsfragen im Prüfungsteil: Technische Kenntnisse", "Leaf"), + ) + media = aa.MediaRegistry(media_dir) + front, back, _label = aa.render_question(item, media, "seed") + self.assertIn('src="TEST123_a.svg"', front) + self.assertIn('src="TEST123_b.svg"', front) + self.assertIn('src="TEST123_a.svg"', back) + self.assertEqual({p.name for p in media.used_paths}, {"TEST123_a.svg", "TEST123_b.svg"}) + + def test_svg_white_background_injection_is_idempotent(self): + svg = '' + once = aa.svg_with_white_background(svg) + twice = aa.svg_with_white_background(once) + + self.assertEqual(once, twice) + self.assertIn('