#!/usr/bin/env python3
"""Build Anki deck packages from an extracted BNetzA question catalog.
Input is the directory produced by `amateurfunk_fetch.py`:
data/
manifest-latest.json
2024-03-20-3-auflage/
manifest.json
fragenkatalog3b.json
svgs/
Output is one `.apkg` per top-level Prüfungsteil:
anki/amateurfunk-technische-kenntnisse.apkg
anki/amateurfunk-betriebliche-kenntnisse.apkg
anki/amateurfunk-kenntnisse-von-vorschriften.apkg
The script intentionally uses only the Python standard library. A modern
Anki `.apkg` is a ZIP containing a SQLite `collection.anki2` database
plus a `media` JSON map and the referenced media files.
"""
import argparse
import dataclasses
import hashlib
import html
import json
import os
import random
import re
import shutil
import sqlite3
import sys
import unicodedata
import zipfile
from datetime import datetime, timezone
from pathlib import Path
DEFAULT_DATA_DIR = Path("data")
DEFAULT_OUT_DIR = Path("anki")
EXIT_OK = 0
EXIT_ERROR = 1
FIELD_SEP = "\x1f"
TOP_LEVEL_PREFIX = "Prüfungsfragen im Prüfungsteil: "
CLASS_TAGS = {"1": "N", "2": "E", "3": "A"}
DEFAULT_BUILD_EPOCH = 0
SAFE_INLINE_TAG_RE = re.compile(r"(?u>)", flags=re.IGNORECASE)
@dataclasses.dataclass
class QuestionItem:
"""A catalog question plus its path through the section tree."""
question: dict
path: tuple[str, ...]
@dataclasses.dataclass
class Category:
"""One top-level exam part, emitted as one Anki package."""
title: str
short_title: str
slug: str
questions: list[QuestionItem]
class AnkiBuildError(Exception):
"""Raised when the local catalog cannot be converted into decks."""
def load_latest_catalog(data_dir: Path):
"""Return `(edition_dir, manifest, catalog)` from a fetch output dir."""
latest_path = data_dir / "manifest-latest.json"
try:
latest = json.loads(latest_path.read_text("utf-8"))
slug = latest["slug"]
edition_dir = data_dir / slug
manifest = json.loads((edition_dir / "manifest.json").read_text("utf-8"))
catalog = json.loads((edition_dir / manifest["json_filename"]).read_text("utf-8"))
except (OSError, KeyError, json.JSONDecodeError) as e:
raise AnkiBuildError(
f"could not read fetched catalog under {data_dir}: {e}"
) from e
return edition_dir, manifest, catalog
def collect_categories(catalog: dict) -> list[Category]:
"""Split the catalog into one category per top-level section."""
sections = catalog.get("sections")
if not isinstance(sections, list):
raise AnkiBuildError("catalog has no top-level sections list")
categories = []
for section in sections:
title = section.get("title", "Unbenannte Kategorie")
questions = []
_collect_questions(section, (title,), questions)
short_title = _short_category_title(title)
categories.append(
Category(
title=title,
short_title=short_title,
slug=slugify(short_title),
questions=questions,
)
)
return categories
def _collect_questions(section: dict, path: tuple[str, ...], out: list[QuestionItem]):
for question in section.get("questions") or []:
out.append(QuestionItem(question=question, path=path))
for child in section.get("sections") or []:
title = child.get("title", "Unbenannt")
_collect_questions(child, path + (title,), out)
def _short_category_title(title: str) -> str:
if title.startswith(TOP_LEVEL_PREFIX):
return title[len(TOP_LEVEL_PREFIX):]
return title
def slugify(value: str) -> str:
"""Make a readable ASCII filename slug."""
replacements = {
"ä": "ae",
"ö": "oe",
"ü": "ue",
"Ä": "Ae",
"Ö": "Oe",
"Ü": "Ue",
"ß": "ss",
}
for src, dst in replacements.items():
value = value.replace(src, dst)
value = unicodedata.normalize("NFKD", value)
value = value.encode("ascii", "ignore").decode("ascii")
value = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-").lower()
return value or "unbenannt"
def stable_id(namespace: str, text: str, *, digits: int = 13) -> int:
"""Return a deterministic positive integer ID for Anki rows."""
digest = hashlib.sha1(f"{namespace}:{text}".encode("utf-8")).hexdigest()
modulus = 10 ** digits
floor = 10 ** (digits - 1)
return floor + (int(digest[:15], 16) % (modulus - floor))
def stable_guid(text: str) -> str:
"""Return a deterministic note GUID."""
return hashlib.sha1(text.encode("utf-8")).hexdigest()[:20]
def checksum_sort_field(sort_field: str) -> int:
"""Anki stores the first 32 bits of SHA1(sort field) as `csum`."""
digest = hashlib.sha1(sort_field.encode("utf-8")).hexdigest()
return int(digest[:8], 16)
def randomized_answers(question: dict, seed: str):
"""Return shuffled answer choices and the displayed correct label."""
choices = [
{"source": "answer_a", "text": question["answer_a"], "picture": question.get("picture_a"), "correct": True},
{"source": "answer_b", "text": question["answer_b"], "picture": question.get("picture_b"), "correct": False},
{"source": "answer_c", "text": question["answer_c"], "picture": question.get("picture_c"), "correct": False},
{"source": "answer_d", "text": question["answer_d"], "picture": question.get("picture_d"), "correct": False},
]
rnd_seed = hashlib.sha256(
f"{seed}:{question.get('number', '')}".encode("utf-8")
).hexdigest()
rnd = random.Random(int(rnd_seed[:16], 16))
rnd.shuffle(choices)
for index, choice in enumerate(choices):
choice["label"] = "ABCD"[index]
correct = next(choice for choice in choices if choice["correct"])
return choices, correct["label"], correct
def render_question(item: QuestionItem, media: "MediaRegistry", seed: str):
"""Render one question as `(front_html, back_html, correct_label)`."""
question = item.question
choices, correct_label, correct = randomized_answers(question, seed)
number = html.escape(str(question.get("number", "")))
path = html.escape(display_path(item.path))
front_parts = [
'
',
f'
{number}'
f'{path}
',
f'
{text_html(question.get("question", ""))}
',
]
q_image = media.image_html(question.get("picture_question"))
if q_image:
front_parts.append(f'
{q_image}
')
front_parts.append('
')
for choice in choices:
choice_image = media.image_html(choice.get("picture"))
front_parts.append(
'- '
f'
{text_html(choice["text"])}
'
f'{f"{choice_image}
" if choice_image else ""}'
' '
)
front_parts.append("
")
correct_image = media.image_html(correct.get("picture"))
back_parts = [
'',
f'
Richtige Antwort: {correct_label}
',
f'
{text_html(correct["text"])}
',
]
if correct_image:
back_parts.append(f'
{correct_image}
')
back_parts.append("
")
return "".join(front_parts), "".join(back_parts), correct_label
def display_path(path: tuple[str, ...]) -> str:
"""Return the user-facing section path with the top prefix stripped."""
if not path:
return ""
return " / ".join([_short_category_title(path[0]), *path[1:]])
def text_html(value) -> str:
"""Escape text for HTML, preserve line breaks, and enable MathJax.
Upstream uses bare `$...$` for inline LaTeX. Anki's built-in MathJax
recognizes `\\(...\\)`, so tokenize math spans and rewrite them while
still escaping HTML-sensitive characters.
"""
parts = []
for is_math, chunk in tokenize_dollar_math(str(value)):
if is_math:
escaped = html.escape(chunk)
parts.append(r"\(" + escaped + r"\)")
else:
parts.append("
".join(escape_text_preserving_safe_tags(chunk).splitlines()))
return "".join(parts)
def escape_text_preserving_safe_tags(text: str) -> str:
"""Escape text while preserving the catalog's safe inline markup.
The live catalog uses `...` to emphasize negation in question
stems. Preserve only exact underline tags; escape everything else so
ordinary comparisons such as `2 < 3` and unexpected HTML stay safe.
"""
escaped_parts = []
for part in SAFE_INLINE_TAG_RE.split(text):
if SAFE_INLINE_TAG_RE.fullmatch(part):
escaped_parts.append(part.lower())
else:
escaped_parts.append(html.escape(part))
return "".join(escaped_parts)
def tokenize_dollar_math(text: str):
"""Yield `(is_math, text)` chunks, converting unescaped `$...$` pairs.
This intentionally handles the simple inline convention used by the
BNetzA catalog. Unmatched dollars are treated as normal text.
"""
index = 0
while index < len(text):
start = find_unescaped_dollar(text, index)
if start is None:
yield False, text[index:]
return
end = find_unescaped_dollar(text, start + 1)
if end is None:
yield False, text[index:]
return
if start > index:
yield False, text[index:start]
yield True, text[start + 1:end]
index = end + 1
def find_unescaped_dollar(text: str, start: int):
while True:
pos = text.find("$", start)
if pos == -1:
return None
backslashes = 0
cursor = pos - 1
while cursor >= 0 and text[cursor] == "\\":
backslashes += 1
cursor -= 1
if backslashes % 2 == 0:
return pos
start = pos + 1
class MediaRegistry:
"""Resolve catalog image stems and collect media files for an apkg."""
def __init__(self, media_dir: Path):
self.media_dir = media_dir
self.by_filename: dict[str, Path] = {}
self.used_paths: set[Path] = set()
self.missing: list[str] = []
self._index_media_dir()
def _index_media_dir(self):
if not self.media_dir.exists():
return
for path in self.media_dir.iterdir():
if path.is_file():
self.by_filename[path.name] = path
def resolve(self, reference) -> Path | None:
if not reference:
return None
ref = str(reference)
if "/" in ref or "\\" in ref:
self.missing.append(ref)
return None
candidates = []
if Path(ref).suffix:
candidates.append(ref)
else:
candidates.extend([f"{ref}.svg", f"{ref}.png"])
for candidate in candidates:
path = self.by_filename.get(candidate)
if path is not None:
self.used_paths.add(path)
return path
self.missing.append(ref)
return None
def image_html(self, reference) -> str:
path = self.resolve(reference)
if path is None:
return ""
filename = html.escape(path.name, quote=True)
return f'
'
def build_apkg_for_category(
category: Category,
edition_dir: Path,
out_path: Path,
*,
seed: str,
build_epoch: int,
):
"""Write one category as an Anki `.apkg` file."""
tmp_dir = out_path.parent / f".{out_path.name}.tmp"
db_path = tmp_dir / "collection.anki2"
# Unlike the fetcher, this builder has no operator-recoverable state
# in its package temp file. A stale `.apkg.tmp` is just an incomplete
# output artifact, so we overwrite it on the next deterministic build.
if tmp_dir.exists():
shutil.rmtree(tmp_dir)
tmp_dir.mkdir(parents=True)
media = MediaRegistry(edition_dir / "svgs")
deck_name = f"Amateurfunk::{category.short_title}"
deck_id = stable_id("deck", deck_name)
model_id = stable_id("model", "Amateurfunk Multiple Choice")
now = build_epoch
try:
notes = []
for ordinal, item in enumerate(category.questions):
front, back, correct_label = render_question(item, media, seed)
question = item.question
number = str(question.get("number", f"q{ordinal}"))
note_id = stable_id("note", f"{category.slug}:{number}")
card_id = stable_id("card", f"{category.slug}:{number}")
fields = [
number,
category.short_title,
display_path(item.path),
front,
back,
correct_label,
]
notes.append(
{
"note_id": note_id,
"card_id": card_id,
"guid": stable_guid(f"{category.slug}:{number}"),
"fields": FIELD_SEP.join(fields),
"sort": number,
"tags": tags_for_item(item),
}
)
media_paths = {path.name: path for path in media.used_paths}
create_collection_db(
db_path,
deck_id=deck_id,
deck_name=deck_name,
model_id=model_id,
notes=notes,
now=now,
)
write_apkg(out_path, db_path, media_paths, build_epoch=build_epoch)
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
return {
"path": out_path,
"deck": deck_name,
"questions": len(category.questions),
"media": len(media_paths),
"missing_media": sorted(set(media.missing)),
}
def tags_for_item(item: QuestionItem) -> str:
"""Build Anki tag field with path metadata."""
question = item.question
class_tag = CLASS_TAGS.get(str(question.get("class")), "unknown")
tags = [
f"klasse-{class_tag}",
]
tags.extend(f"pfad-{slugify(part)}" for part in item.path[1:])
return " " + " ".join(tags) + " "
def create_collection_db(
db_path: Path,
*,
deck_id: int,
deck_name: str,
model_id: int,
notes: list[dict],
now: int,
):
"""Create the SQLite collection.anki2 database used inside an apkg."""
conn = sqlite3.connect(db_path)
try:
create_schema(conn)
insert_collection_metadata(
conn,
deck_id=deck_id,
deck_name=deck_name,
model_id=model_id,
now=now,
)
for due, note in enumerate(notes, start=1):
conn.execute(
"""
INSERT INTO notes
(id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
note["note_id"],
note["guid"],
model_id,
now,
-1,
note["tags"],
note["fields"],
note["sort"],
checksum_sort_field(note["sort"]),
0,
"",
),
)
conn.execute(
"""
INSERT INTO cards
(id, nid, did, ord, mod, usn, type, queue, due, ivl, factor,
reps, lapses, left, odue, odid, flags, data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
note["card_id"],
note["note_id"],
deck_id,
0,
now,
-1,
0,
0,
due,
0,
0,
0,
0,
0,
0,
0,
0,
"",
),
)
conn.commit()
finally:
conn.close()
def create_schema(conn: sqlite3.Connection):
conn.executescript(
"""
CREATE TABLE col (
id integer primary key,
crt integer not null,
mod integer not null,
scm integer not null,
ver integer not null,
dty integer not null,
usn integer not null,
ls integer not null,
conf text not null,
models text not null,
decks text not null,
dconf text not null,
tags text not null
);
CREATE TABLE notes (
id integer primary key,
guid text not null,
mid integer not null,
mod integer not null,
usn integer not null,
tags text not null,
flds text not null,
sfld integer not null,
csum integer not null,
flags integer not null,
data text not null
);
CREATE TABLE cards (
id integer primary key,
nid integer not null,
did integer not null,
ord integer not null,
mod integer not null,
usn integer not null,
type integer not null,
queue integer not null,
due integer not null,
ivl integer not null,
factor integer not null,
reps integer not null,
lapses integer not null,
left integer not null,
odue integer not null,
odid integer not null,
flags integer not null,
data text not null
);
CREATE TABLE revlog (
id integer primary key,
cid integer not null,
usn integer not null,
ease integer not null,
ivl integer not null,
lastIvl integer not null,
factor integer not null,
time integer not null,
type integer not null
);
CREATE TABLE graves (
usn integer not null,
oid integer not null,
type integer not null
);
CREATE INDEX ix_notes_usn on notes (usn);
CREATE INDEX ix_cards_usn on cards (usn);
CREATE INDEX ix_cards_nid on cards (nid);
CREATE INDEX ix_cards_sched on cards (did, queue, due);
CREATE INDEX ix_revlog_usn on revlog (usn);
CREATE INDEX ix_revlog_cid on revlog (cid);
"""
)
def insert_collection_metadata(
conn: sqlite3.Connection,
*,
deck_id: int,
deck_name: str,
model_id: int,
now: int,
):
conn.execute(
"""
INSERT INTO col
(id, crt, mod, scm, ver, dty, usn, ls, conf, models, decks, dconf, tags)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
1,
now,
now * 1000,
now * 1000,
11,
0,
0,
0,
json.dumps(collection_conf(deck_id), separators=(",", ":")),
json.dumps({str(model_id): model_json(model_id, now)}, separators=(",", ":")),
json.dumps({str(deck_id): deck_json(deck_id, deck_name, now)}, separators=(",", ":")),
json.dumps(default_deck_conf(now), separators=(",", ":")),
"{}",
),
)
def collection_conf(deck_id: int):
return {
"nextPos": 1,
"estTimes": True,
"activeDecks": [deck_id],
"sortType": "noteFld",
"timeLim": 0,
"sortBackwards": False,
"addToCur": True,
"curDeck": deck_id,
"newBury": True,
"newSpread": 0,
"dueCounts": True,
"curModel": None,
"collapseTime": 1200,
}
def deck_json(deck_id: int, deck_name: str, now: int):
return {
"id": deck_id,
"name": deck_name,
"mod": now,
"usn": -1,
"lrnToday": [0, 0],
"revToday": [0, 0],
"newToday": [0, 0],
"timeToday": [0, 0],
"collapsed": False,
"browserCollapsed": False,
"dyn": 0,
"conf": 1,
"desc": "",
"extendNew": 0,
"extendRev": 0,
}
def default_deck_conf(now: int):
return {
"1": {
"id": 1,
"name": "Default",
"mod": now,
"usn": -1,
"maxTaken": 60,
"autoplay": True,
"timer": 0,
"replayq": True,
"new": {
"delays": [1, 10],
"ints": [1, 4, 0],
"initialFactor": 2500,
"separate": True,
"order": 1,
"perDay": 20,
"bury": True,
},
"rev": {
"perDay": 200,
"ease4": 1.3,
"fuzz": 0.05,
"minSpace": 1,
"ivlFct": 1,
"maxIvl": 36500,
"bury": True,
},
"lapse": {
"delays": [10],
"mult": 0,
"minInt": 1,
"leechFails": 8,
"leechAction": 0,
},
}
}
def model_json(model_id: int, now: int):
fields = [
field_json("Number", 0),
field_json("Category", 1),
field_json("Path", 2),
field_json("Front", 3),
field_json("Back", 4),
field_json("CorrectLabel", 5),
]
return {
"id": model_id,
"name": "Amateurfunk Multiple Choice",
"type": 0,
"mod": now,
"usn": -1,
"sortf": 0,
"did": None,
"tmpls": [
{
"name": "Card 1",
"ord": 0,
"qfmt": "{{Front}}",
"afmt": "{{FrontSide}}\n
\n{{Back}}",
"did": None,
"bqfmt": "",
"bafmt": "",
}
],
"flds": fields,
"css": CARD_CSS,
"latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage[utf8]{inputenc}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\begin{document}",
"latexPost": "\\end{document}",
"req": [[0, "any", [3]]],
"vers": [],
}
def field_json(name: str, ord_: int):
return {
"name": name,
"ord": ord_,
"sticky": False,
"rtl": False,
"font": "Arial",
"size": 20,
"description": "",
"plainText": False,
"collapsed": False,
"excludeFromSearch": False,
"tag": None,
"preventDeletion": False,
}
CARD_CSS = """
.card {
font-family: Arial, sans-serif;
font-size: 18px;
line-height: 1.45;
color: #111;
background: #fff;
text-align: left;
}
.af-meta {
color: #555;
font-size: 13px;
margin-bottom: 0.75rem;
}
.af-number {
font-weight: bold;
margin-right: 0.75rem;
}
.af-question {
font-size: 20px;
margin-bottom: 1rem;
}
.af-answers {
padding-left: 1.7rem;
}
.af-answers li {
margin: 0.7rem 0;
}
.af-media {
max-width: 100%;
height: auto;
margin: 0.4rem 0;
}
.af-correct-label {
font-weight: bold;
color: #0b6b3a;
margin-bottom: 0.5rem;
}
"""
def write_apkg(
out_path: Path,
db_path: Path,
media_paths: dict[str, Path],
*,
build_epoch: int,
):
"""Write Anki package ZIP with collection DB and media files."""
out_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = out_path.with_suffix(out_path.suffix + ".tmp")
media_map = {}
try:
with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
zip_writestr_fixed(
zf,
"collection.anki2",
db_path.read_bytes(),
build_epoch=build_epoch,
)
for index, filename in enumerate(sorted(media_paths)):
archive_name = str(index)
media_map[archive_name] = filename
zip_writestr_fixed(
zf,
archive_name,
media_bytes_for_package(media_paths[filename]),
build_epoch=build_epoch,
)
zip_writestr_fixed(
zf,
"media",
json.dumps(media_map, ensure_ascii=False).encode("utf-8"),
build_epoch=build_epoch,
)
os.replace(tmp_path, out_path)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
def zip_writestr_fixed(
zf: zipfile.ZipFile,
name: str,
payload: bytes,
*,
build_epoch: int,
):
"""Write a ZIP member with deterministic timestamp and attributes."""
info = zipfile.ZipInfo(name, zip_datetime(build_epoch))
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o644 << 16
zf.writestr(info, payload)
def zip_datetime(epoch: int):
"""Return a ZIP-compatible UTC timestamp tuple.
ZIP timestamps cannot represent dates before 1980.
"""
safe_epoch = max(int(epoch), 315532800)
value = datetime.fromtimestamp(safe_epoch, timezone.utc)
return (
value.year,
value.month,
value.day,
value.hour,
value.minute,
value.second,
)
def media_bytes_for_package(path: Path) -> bytes:
"""Return media bytes as they should be stored in the Anki package.
BNetzA SVGs are transparent. In Anki dark mode that makes black line
drawings hard to read because the card background can be black. When
packaging SVGs, add an explicit white background rectangle as the
first painted element. The extracted source SVGs are left untouched.
"""
raw = path.read_bytes()
if path.suffix.lower() != ".svg":
return raw
text = raw.decode("utf-8-sig", errors="replace")
return svg_with_white_background(text).encode("utf-8")
def svg_with_white_background(svg_text: str) -> str:
"""Inject a white background rect immediately after the opening svg tag."""
if 'data-af-white-background="1"' in svg_text:
return svg_text
match = re.search(r"