From dd61be73f3c5317d5469af5484c4d19aaf28dd46 Mon Sep 17 00:00:00 2001 From: Renat Nurgaliyev Date: Wed, 20 May 2026 18:18:53 +0200 Subject: [PATCH] Cleanup --- CLAUDE.md | 88 ++-- DESIGN.md | 235 ++++++++++- amateurfunk_anki.py | 844 ++++++++++++++++++++++++++------------ test_amateurfunk_anki.py | 4 +- test_amateurfunk_fetch.py | 1 - 5 files changed, 862 insertions(+), 310 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4731f9e..df05e1e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,13 +1,12 @@ -# Amateurfunk — BNetzA Question Catalog Downloader +# Amateurfunk — BNetzA Question Catalog → Anki Decks -A small Python tool that downloads the latest German amateur-radio exam -question catalog ("Fragenkatalog") published by the Bundesnetzagentur -(BNetzA) and extracts the structured JSON + SVG image assets for further -use (study apps, flashcards, training tools, etc.). +A small two-stage Python pipeline that downloads the German amateur-radio +exam question catalog ("Fragenkatalog") published by the Bundesnetzagentur +(BNetzA) and turns it into Anki decks. -The full source-discovery notes, JSON schema, and design decisions live -in `DESIGN.md`. This file is a short orientation for anyone (human or -agent) opening the project. +The full source-discovery notes, JSON schema, exam-structure details, and +per-stage design decisions live in `DESIGN.md`. This file is a short +orientation for anyone (human or agent) opening the project. ## What the catalog is @@ -25,7 +24,7 @@ agent) opening the project. - ZIP (machine-readable): `https://www.bundesnetzagentur.de/SharedDocs/Downloads/DE/Sachgebiete/Telekommunikation/Unternehmen_Institutionen/Frequenzen/Amateurfunk/Fragenkatalog/PruefungsfragenZIP.zip?__blob=publicationFile` - Landing page (short link): `https://www.bnetza.de/amateurfunk-fragenkatalog` -- PDF (human-readable, not used by this tool): same path with +- PDF (human-readable, not used by this pipeline): same path with `Pruefungsfragen.pdf` instead of `PruefungsfragenZIP.zip`. The ZIP URL is stable across editions — BNetzA replaces the file @@ -34,32 +33,65 @@ detection. The filename inside the ZIP (`fragenkatalog3b.json`) encodes the edition (`3b` = 3rd edition, revision b) and will change on new editions, so we discover it from the archive rather than hard-coding. -## Scope of the tool (initial) +## Pipeline overview + +``` +BNetzA ZIP ──[Stage 1: amateurfunk_fetch.py]──► data// + ├── fragenkatalog*.json + ├── svgs/ + ├── README.txt + └── manifest.json + +data/ ──[Stage 2: amateurfunk_anki.py]──► anki/ + ├── amateurfunk-technische-kenntnisse.apkg + ├── amateurfunk-betriebliche-kenntnisse.apkg + └── amateurfunk-kenntnisse-von-vorschriften.apkg +``` + +### Stage 1 — `amateurfunk_fetch.py` 1. Download the ZIP from the canonical URL. 2. Verify it is a valid ZIP and contains the expected JSON + SVG files. 3. Extract to a target directory (default: `./data//`). -4. Emit a small `manifest.json` next to the data: source URL, fetched-at - timestamp, `Last-Modified` from the server, JSON edition metadata, - sha256 of the ZIP. +4. Emit a small `manifest.json` next to the data: source URL, + fetched-at timestamp, `Last-Modified` from the server, JSON edition + metadata, sha256 of the ZIP. 5. Be idempotent — re-running without an upstream change is a no-op. The skip key is the HTTP `Last-Modified` header recorded on the previous manifest; the ZIP is deleted by default after extraction, so the recorded sha256 is provenance, not a re-verification target. - See `DESIGN.md` for the full idempotency contract. + See `DESIGN.md` §4 for the full idempotency contract. -Out of scope for v1 (kept for later): rendering LaTeX/SVG, building a -study app, multi-edition diffing, mirroring the PDF. +### Stage 2 — `amateurfunk_anki.py` + +1. Read the latest edition from `data/` (following + `manifest-latest.json` to a per-edition directory). +2. Split the catalog into three categories — one per top-level + Prüfungsteil (Technische / Betriebliche / Vorschriften). The + license-class axis is mapped into tags, not separate packages. +3. Render every question as an Anki note: shuffled A/B/C/D choices on + the front, the displayed position of the correct answer on the + back. Inline `$...$` LaTeX is converted to MathJax `\(...\)` + delimiters; the catalog's safe inline markup (`...`) is + preserved. +4. Hand-roll the v11 Anki collection (SQLite + JSON config) and + package it as a `.apkg` ZIP with deterministic timestamps. Same + input → byte-identical output across runs. + +The Anki design decisions (shuffle seeding, deterministic build epoch, +SVG dark-mode handling, schema choices) live in `DESIGN.md` §7. ## Repo conventions -- Python 3.11+, standard library only where reasonable (`urllib`, - `zipfile`, `hashlib`, `json`, `pathlib`). Add deps only if they pay - for themselves — none expected for v1. -- Single-purpose script: `amateurfunk_fetch.py` (or a tiny package if - it grows). No framework, no CLI library beyond `argparse`. -- Downloaded data is treated as a build artifact: kept under `data/` - and gitignored. +- Python 3.11+, standard library only. No third-party dependencies + in either stage. +- Single-file scripts: `amateurfunk_fetch.py`, `amateurfunk_anki.py`. + No frameworks, no CLI library beyond `argparse`. +- Style: section banners, commented constants, docstrings on every + function, inline comments at decision points. The two scripts + intentionally read the same way. +- Outputs are build artifacts: kept under `data/` and `anki/`, both + gitignored. - License attribution string (required by DL-DE→BY-2.0) is preserved verbatim from the upstream `README.txt` whenever we redistribute the data. @@ -67,10 +99,14 @@ study app, multi-edition diffing, mirroring the PDF. ## Working on this repo - Start from `DESIGN.md` — it has the JSON schema, the question/answer - conventions (answer A is always correct, B/C/D are distractors), the - LaTeX-in-questions caveat, and the SVG naming convention. + conventions (answer A is always correct upstream → consumers shuffle + before display), the LaTeX-in-questions caveat, the exam-structure + rationale for the three Anki packages, and per-stage design notes. - Do not invent new download URLs; the ones in `DESIGN.md` were verified against the live BNetzA site. - When BNetzA publishes a new edition, expect a new - `fragenkatalog.json` filename inside the ZIP. The downloader + `fragenkatalog.json` filename inside the ZIP. The fetcher must not hard-code the current name. +- Both stages have a fixture-driven test suite. Run with + `python3 -m unittest test_amateurfunk_fetch test_amateurfunk_anki`. + Network access is only needed for the manual smoke test of Stage 1. diff --git a/DESIGN.md b/DESIGN.md index 153a123..d7712b3 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,8 +1,11 @@ -# Design Notes — BNetzA Amateurfunk Question Catalog Downloader +# Design Notes — BNetzA Amateurfunk Question Catalog → Anki -Status: **design only, no code yet.** This document captures the -source-discovery work and the proposed shape of the tool. Everything -below was verified against the live BNetzA site in May 2026. +Status: **both stages implemented.** Stage 1 (`amateurfunk_fetch.py`) +downloads and validates the upstream ZIP; Stage 2 +(`amateurfunk_anki.py`) turns the extracted data into Anki decks. This +document captures the source-discovery work, the JSON schema, and the +design contracts that both scripts implement. Everything below was +verified against the live BNetzA site in May 2026. --- @@ -234,7 +237,7 @@ from the data. --- -## 4. Tool design (v1) +## 4. Stage 1 — Fetcher (`amateurfunk_fetch.py`) ### Goal @@ -430,21 +433,223 @@ These were open in the first draft and have been settled: `missing_pictures`, do not fail). Rationale: avoid bricking the tool on an upstream typo. -## 6. Open questions +## 6. Open questions (Stage 1) -These do not block writing v1: +These do not block the Stage 1 implementation: 1. **PNG fallbacks** — the archive ships PNGs for ~a handful of - figures alongside the SVGs. v1 just extracts everything as-is. A - future renderer can prefer SVG and fall back to PNG transparently. + figures alongside the SVGs. The fetcher just extracts everything + as-is. Stage 2 prefers SVG and falls back to PNG by stem: + `MediaRegistry.resolve()` tries `.svg` first, then + `.png`. So an upstream entry that ships only a PNG (or one + whose SVG fails to render in some future consumer) still resolves. 2. **Schema drift safety** — if a future edition adds or renames fields, the validator should warn but not fail. v1 fails closed on missing required top-level/metadata/question keys but tolerates extra keys. We can soften this further if BNetzA evolves the format. -3. **Packaging** — v1 stays a single-file script with `argparse` - (`amateurfunk_fetch.py`). Considered and declined: a - `pyproject.toml` package with a console-script entry point. For a - tool that runs at most quarterly, the packaging ceremony does not - pay for itself. Revisit if the scope grows (e.g. importable from - another project, distributed on PyPI). +3. **Packaging** — both scripts stay single-file with `argparse` + (`amateurfunk_fetch.py`, `amateurfunk_anki.py`). Considered and + declined: a `pyproject.toml` package with console-script entry + points. For tools that run at most quarterly, the packaging + ceremony does not pay for itself. Revisit if the scope grows + (e.g. importable from another project, distributed on PyPI). + +--- + +## 7. Stage 2 — Anki deck builder (`amateurfunk_anki.py`) + +### Goal + +Given the per-edition directory produced by Stage 1, build a set of +Anki `.apkg` files that turn every catalog question into a flash card. +Same input must produce byte-identical output across runs — this is +important so generated decks can be checksummed and cached cleanly. + +### CLI shape + +``` +amateurfunk-anki [--data DIR] [--out DIR] [--seed STR] [--epoch INT] +``` + +- `--data DIR` — fetch output root (default `./data`). Must contain + `manifest-latest.json` pointing at a per-edition directory. +- `--out DIR` — destination for `.apkg` files (default `./anki`). +- `--seed STR` — deterministic seed for answer shuffling. The default + is fixed; changing it produces a different (but still deterministic) + shuffle. +- `--epoch INT` — override the package timestamp epoch. By default we + derive it from the manifest's `fetched_at`; this flag is mainly for + tests and explicit rebuilds. + +Exit codes: `0` success, `1` configuration / catalog / build error. +There is no Stage-2 equivalent of the fetcher's `EXIT_BAD_STATE` — +the builder has no operator-recoverable local state, just generated +output artifacts. + +### Output layout + +``` +anki/ + amateurfunk-technische-kenntnisse.apkg (~2.5 MB, 1374 cards) + amateurfunk-betriebliche-kenntnisse.apkg (~130 KB, 172 cards) + amateurfunk-kenntnisse-von-vorschriften.apkg (~75 KB, 204 cards) +``` + +One `.apkg` per top-level Prüfungsteil. The card counts match the +study pools from §3 axis 1: Technische = 1374 (cumulative N+E+A); +Betriebliche = 172 (shared across all candidates); Vorschriften = 204 +(also shared). License class is conveyed via tags +(`klasse-N` / `klasse-E` / `klasse-A`), not separate packages — +Anki users browse decks by topic and filter by class. + +### Steps + +1. **Load** the latest catalog: follow `manifest-latest.json` to a + per-edition directory, read the catalog JSON, the per-edition + `manifest.json`, and index the `svgs/` folder. +2. **Categorize** the question tree into three `Category` objects, + one per top-level Prüfungsteil. Each category carries the flat + list of every question that lives anywhere under it, along with + each question's path through the section tree (used for the card + breadcrumb and the path tags). +3. **For each category, render every question** as an Anki note: + - Compute a stable per-question seed from `--seed` + question + number; shuffle the A/B/C/D choices using it. The displayed + label (A/B/C/D) is assigned post-shuffle; the back of the card + names the *displayed* position of the correct answer. + - Render the question stem and answer texts to HTML via + `text_html()`, which tokenizes inline `$...$` math and rewrites + it to MathJax `\(...\)` delimiters (Anki 2.1+ ships MathJax + built in and recognizes those delimiters; bare `$...$` would + show as source). Escapes the rest for HTML safety, preserves + line breaks as `
`, and preserves `...` tags that + the catalog uses to emphasize negation in question stems. + - Resolve picture references through a per-category + `MediaRegistry`, which records every file actually used so the + packager can include only those. +4. **Build a v11 Anki collection** as an in-memory SQLite database: + one `col` row carrying JSON config blobs (deck, model, dconf), + one `notes` row per question, one `cards` row per question. + Modern Anki understands v11 and upgrades the collection on + first open. +5. **Package** the collection plus the referenced media into a + `.apkg` ZIP. Media is addressed by sequential integer keys + (Anki's convention); a `media` JSON map at the archive root + translates those keys back to filenames. +6. **Post-process SVGs** at packaging time: inject a white + `` immediately after the opening `` tag so dark-mode + Anki users can still read the black-line BNetzA figures. The + extracted source SVGs on disk are never modified. The injection + is idempotent (marked with `data-af-white-background="1"`). +7. **Write `.apkg` atomically**: build into `.tmp`, then + `os.replace` into place. Stale `.tmp` directories are + overwritten on the next deterministic build — unlike the + fetcher's `.bak`/`.tmp` siblings, the Stage-2 temp file holds + no operator-recoverable state. + +### Determinism + +The contract is: same catalog in → same `.apkg` bytes out. +Determinism rests on three things: + +1. **Stable IDs.** `stable_id(namespace, text)` hashes a + namespaced key with SHA-1 and squashes into the standard + 13-digit Anki ID range. Different namespaces + (`"deck"` / `"model"` / `"note"` / `"card"`) keep IDs from + colliding across kinds; the namespaced inputs are stable + (`f"{category.slug}:{number}"` for notes and cards). + `stable_guid(text)` produces the 20-character note GUID used + for re-import deduplication. +2. **Stable shuffle.** `randomized_answers()` builds a per-question + `random.Random` seeded from SHA-256 of + `f"{cli_seed}:{question_number}"`. +3. **Stable timestamps.** Every `now` value in the collection (the + `mod` columns, the JSON config blob timestamps) is fixed to + `build_epoch` — derived from the manifest's `fetched_at`, or + overridden via `--epoch`. ZIP member timestamps are also fixed + via `ZipInfo(name, zip_datetime(build_epoch))`. Without this + last step, the inner SQLite would be identical but the + archive's per-entry mtimes would still vary between runs. + +The combined effect: two runs with the same `data/` produce +byte-identical sha256 on each `.apkg`. Verified during review. + +### Rendering decisions worth knowing + +- **A is always correct upstream, then shuffled.** This is the + consumer-side caveat from §3 made concrete: we set the + `correct` flag on `answer_a` before shuffling, then carry it + through. The back of the card reveals the *displayed* position + ("Richtige Antwort: B"), not the source position. +- **LaTeX → MathJax.** The catalog ships ~430 fragments + containing inline `$...$` (DESIGN §3 quotes the upstream README + on this). Anki 2.1+ MathJax recognizes `\(...\)` natively; + `$...$` does not render. The rewrite happens in `text_html()`. +- **Safe inline tags.** The catalog uses `...` for + emphasis. Naive HTML escaping would convert those to + `<u>...</u>` and lose the emphasis. We split on a + regex that matches exactly `` and `` and escape only the + non-tag pieces. +- **Slug for filenames.** `slugify()` maps German umlauts and ß + to ASCII digraphs before NFKD-normalizing the rest, then keeps + only `[a-z0-9-]`. Yields readable, sortable filenames + (`amateurfunk-technische-kenntnisse.apkg`). +- **Tags.** Each note carries `klasse-N|E|A` plus + `pfad-` for every section level below the + top-level Prüfungsteil. Per-question number tags are + deliberately *not* emitted — they would create ~1750 singletons + in Anki's tag tree, and the number is already in the dedicated + `Number` field for search. +- **Breadcrumb consistency.** The visible card breadcrumb and the + stored `Path` field both go through one `display_path()` + helper, so they never drift. The boilerplate + `Prüfungsfragen im Prüfungsteil:` prefix is stripped at this + layer. +- **SVG white background.** BNetzA SVGs are transparent with + black line work. In Anki's dark mode the lines disappear into + the card background. We inject a white `` as the first + painted element when packaging the SVG into the `.apkg`. The + on-disk extracted files are left untouched. + +### Testing focus + +Network-free, fixture-driven. Real Anki is NOT required to +verify the output — tests open the `.apkg` ZIP directly, parse +the embedded SQLite, and check structural invariants. + +- Top-level structure: one `.apkg` per Prüfungsteil; expected + filename slugs. +- Note count and field shape per category. +- Tag mapping: a class-1 question carries `klasse-N`, no + `nummer-*` tags. +- Breadcrumb rendering: both the visible HTML breadcrumb and the + stored `Path` field contain the short form; neither contains + the Prüfungsteil prefix. +- LaTeX rewrite: a fixture question with `$...$` produces + `\(...\)` in the rendered HTML; a non-math `<` is still + escaped to `<`; safe inline `...` tags survive + escaping. +- Answer shuffle: the front HTML contains the answer texts in a + non-A-first order; the back names the displayed letter of the + correct answer. +- Picture handling on shuffled answers: a fixture question whose + `picture_a`/`picture_b`/... fields point at different files + carries the right image with the right shuffled choice. +- Media inclusion: `MediaRegistry.used_paths` is populated by + resolution and only those files end up in the `.apkg`. +- SVG white background injection: idempotent; the marker rect is + emitted exactly once. +- Deterministic build: two consecutive builds against the same + catalog with a fixed `--epoch` produce byte-identical `.apkg` + files. + +### What we deliberately do NOT do in Stage 2 + +- No editable cloze cards or "type the answer" templates — the + exam is multiple choice, and the deck mirrors that. +- No per-class packages. Class is a tag axis; package split is + by Prüfungsteil. +- No automatic edition diffs / "new questions since last build" + decks. Useful but out of scope. +- No upload to AnkiWeb. Decks are local artifacts. diff --git a/amateurfunk_anki.py b/amateurfunk_anki.py index b14d114..8698abf 100644 --- a/amateurfunk_anki.py +++ b/amateurfunk_anki.py @@ -1,28 +1,32 @@ #!/usr/bin/env python3 """Build Anki deck packages from an extracted BNetzA question catalog. -Input is the directory produced by `amateurfunk_fetch.py`: +The full design and contract live in DESIGN.md. As an overview the +script does: - data/ - manifest-latest.json - 2024-03-20-3-auflage/ - manifest.json - fragenkatalog3b.json - svgs/ + 1. Read the per-edition directory written by `amateurfunk_fetch.py` + (the JSON catalog, the `svgs/` folder, and the per-edition + `manifest.json`). + 2. Split the catalog into three categories — one per top-level + Prüfungsteil: Technische / Betriebliche / Vorschriften. (The + license-class axis is mapped into tags, not into separate + packages.) + 3. For each category, render every question as an Anki note: front + shows the (shuffled) A/B/C/D choices, back names the displayed + position of the correct answer. + 4. Hand-roll a v11 Anki collection — SQLite database plus JSON + config blobs — and package it into a `.apkg` ZIP with fixed + timestamps. Same input → byte-identical output. -Output is one `.apkg` per top-level Prüfungsteil: - - anki/amateurfunk-technische-kenntnisse.apkg - anki/amateurfunk-betriebliche-kenntnisse.apkg - anki/amateurfunk-kenntnisse-von-vorschriften.apkg - -The script intentionally uses only the Python standard library. A modern -Anki `.apkg` is a ZIP containing a SQLite `collection.anki2` database -plus a `media` JSON map and the referenced media files. +The script is intentionally a single file with stdlib only. Readability +beats cleverness here — most of the bytes below are docstrings and +comments. Performance is a non-goal (we build three small decks once +per upstream edition). """ import argparse import dataclasses +import datetime as dt import hashlib import html import json @@ -34,53 +38,119 @@ import sqlite3 import sys import unicodedata import zipfile -from datetime import datetime, timezone from pathlib import Path +# ============================================================================ +# Constants +# ============================================================================ + +# Default location of the fetcher's output (one directory up from a +# per-edition slug, containing `manifest-latest.json`). DEFAULT_DATA_DIR = Path("data") + +# Default destination for the generated `.apkg` files. DEFAULT_OUT_DIR = Path("anki") + +# Exit codes. The builder is much simpler than the fetcher — there is +# no "operator must resolve local state" case here, so two codes are +# enough. EXIT_OK = 0 EXIT_ERROR = 1 +# Anki stores all of a note's field values in a single string column +# (`flds`), separated by the Unit Separator control character. Never +# changes; spelled out as a constant so the intent is obvious. FIELD_SEP = "\x1f" + +# The German top-level section titles all share this prefix +# (e.g. "Prüfungsfragen im Prüfungsteil: Technische Kenntnisse"). +# We strip it for human-facing display so cards and tags aren't +# dominated by boilerplate. TOP_LEVEL_PREFIX = "Prüfungsfragen im Prüfungsteil: " + +# Map from the catalog's class digits to the user-facing license +# letters documented in DESIGN.md §3 ("Important consumer-side +# conventions"). Tags use the letters because that's what learners +# search for. CLASS_TAGS = {"1": "N", "2": "E", "3": "A"} + +# Fallback build epoch if neither the manifest nor `--epoch` supplies +# one. Picked as 0 so missing-metadata builds are still deterministic +# and obviously wrong (timestamps would all show 1970). DEFAULT_BUILD_EPOCH = 0 + +# The catalog uses inline `...` for emphasis (typically marking +# negation in question stems — "_NICHT_"). We preserve those tags and +# escape everything else. Anything not in this list goes through +# `html.escape` so plain `<` and `>` remain safe. SAFE_INLINE_TAG_RE = re.compile(r"()", flags=re.IGNORECASE) -@dataclasses.dataclass -class QuestionItem: - """A catalog question plus its path through the section tree.""" - - question: dict - path: tuple[str, ...] - - -@dataclasses.dataclass -class Category: - """One top-level exam part, emitted as one Anki package.""" - - title: str - short_title: str - slug: str - questions: list[QuestionItem] +# ============================================================================ +# Exception types and data classes +# ============================================================================ class AnkiBuildError(Exception): """Raised when the local catalog cannot be converted into decks.""" -def load_latest_catalog(data_dir: Path): - """Return `(edition_dir, manifest, catalog)` from a fetch output dir.""" +@dataclasses.dataclass +class QuestionItem: + """A catalog question together with its path through the section tree. + + `question` is the raw question dict (with `number`, `class`, + `question`, `answer_a` .. `answer_d`, optional `picture_*`). + `path` is a tuple of section titles from root to leaf — used to + build the card breadcrumb and the path tags. + """ + + question: dict + path: tuple + + +@dataclasses.dataclass +class Category: + """One top-level exam part, emitted as one Anki `.apkg`. + + `title` is the original German title (with the Prüfungsteil + prefix); `short_title` has the prefix stripped (used in the deck + name and the slug); `questions` is the flat list of every question + that lives anywhere under this category. + """ + + title: str + short_title: str + slug: str + questions: list + + +# ============================================================================ +# Loading the fetched catalog +# ============================================================================ + + +def load_latest_catalog(data_dir): + """Return `(edition_dir, manifest, catalog)` from a fetch output dir. + + Reads `data_dir/manifest-latest.json` for the slug pointer, then + `data_dir//manifest.json` for the per-edition manifest, then + the catalog JSON file the manifest names. Any missing file or + malformed JSON is surfaced as an `AnkiBuildError` so `main()` can + report it cleanly. + """ latest_path = data_dir / "manifest-latest.json" try: latest = json.loads(latest_path.read_text("utf-8")) slug = latest["slug"] edition_dir = data_dir / slug - manifest = json.loads((edition_dir / "manifest.json").read_text("utf-8")) - catalog = json.loads((edition_dir / manifest["json_filename"]).read_text("utf-8")) + manifest = json.loads( + (edition_dir / "manifest.json").read_text("utf-8") + ) + catalog = json.loads( + (edition_dir / manifest["json_filename"]).read_text("utf-8") + ) except (OSError, KeyError, json.JSONDecodeError) as e: raise AnkiBuildError( f"could not read fetched catalog under {data_dir}: {e}" @@ -88,8 +158,19 @@ def load_latest_catalog(data_dir: Path): return edition_dir, manifest, catalog -def collect_categories(catalog: dict) -> list[Category]: - """Split the catalog into one category per top-level section.""" +# ============================================================================ +# Category collection +# ============================================================================ + + +def collect_categories(catalog): + """Split the catalog into one `Category` per top-level section. + + Per DESIGN.md §3 axis 1, the catalog's three top-level sections + are exactly the three Prüfungsteile. We walk each subtree and + flatten every question it contains, recording the section path so + cards can show a breadcrumb and we can emit path-shaped tags. + """ sections = catalog.get("sections") if not isinstance(sections, list): raise AnkiBuildError("catalog has no top-level sections list") @@ -111,7 +192,13 @@ def collect_categories(catalog: dict) -> list[Category]: return categories -def _collect_questions(section: dict, path: tuple[str, ...], out: list[QuestionItem]): +def _collect_questions(section, path, out): + """Recurse into a section node and append every question to `out`. + + `path` is the tuple of section titles from the top of the catalog + down to the current node — extended by one element on each + recursive call. + """ for question in section.get("questions") or []: out.append(QuestionItem(question=question, path=path)) for child in section.get("sections") or []: @@ -119,14 +206,27 @@ def _collect_questions(section: dict, path: tuple[str, ...], out: list[QuestionI _collect_questions(child, path + (title,), out) -def _short_category_title(title: str) -> str: +def _short_category_title(title): + """Strip the boilerplate `Prüfungsfragen im Prüfungsteil:` prefix.""" if title.startswith(TOP_LEVEL_PREFIX): return title[len(TOP_LEVEL_PREFIX):] return title -def slugify(value: str) -> str: - """Make a readable ASCII filename slug.""" +# ============================================================================ +# Slugs and stable IDs +# ============================================================================ + + +def slugify(value): + """Reduce a German title to a readable lowercase ASCII filename slug. + + Umlauts and ß map to digraphs (`ä → ae`, `ß → ss`) before NFKD + normalization strips any remaining diacritics; everything that + isn't `[A-Za-z0-9]` becomes a single hyphen, and the result is + lowercased. Returns `"unbenannt"` for inputs that collapse to the + empty string. + """ replacements = { "ä": "ae", "ö": "oe", @@ -144,32 +244,82 @@ def slugify(value: str) -> str: return value or "unbenannt" -def stable_id(namespace: str, text: str, *, digits: int = 13) -> int: - """Return a deterministic positive integer ID for Anki rows.""" +def stable_id(namespace, text, digits=13): + """Return a deterministic positive integer ID for Anki row IDs. + + Anki uses 13-digit integers (ms-since-epoch-shaped) for deck, + model, note, and card IDs. We hash a namespaced text key with + SHA-1, take the first 60 bits, and squash them into the + `[10**(digits-1), 10**digits)` range. Different namespaces (e.g. + `"deck"` vs `"note"`) keep IDs from colliding across kinds. + """ digest = hashlib.sha1(f"{namespace}:{text}".encode("utf-8")).hexdigest() modulus = 10 ** digits floor = 10 ** (digits - 1) return floor + (int(digest[:15], 16) % (modulus - floor)) -def stable_guid(text: str) -> str: - """Return a deterministic note GUID.""" +def stable_guid(text): + """Return a deterministic 20-character note GUID. + + Anki uses the GUID to deduplicate notes when an `.apkg` is + re-imported. Deriving it from a stable per-question key (the + category slug plus the question number) means importing the same + deck twice updates rather than duplicates. + """ return hashlib.sha1(text.encode("utf-8")).hexdigest()[:20] -def checksum_sort_field(sort_field: str) -> int: - """Anki stores the first 32 bits of SHA1(sort field) as `csum`.""" +def checksum_sort_field(sort_field): + """Compute Anki's `notes.csum`: first 32 bits of SHA1(sort field). + + Anki uses this for fast duplicate-detection in the note browser. + """ digest = hashlib.sha1(sort_field.encode("utf-8")).hexdigest() return int(digest[:8], 16) -def randomized_answers(question: dict, seed: str): - """Return shuffled answer choices and the displayed correct label.""" +# ============================================================================ +# Answer shuffling +# ============================================================================ + + +def randomized_answers(question, seed): + """Return `(choices, correct_label, correct_choice)` for one question. + + Upstream always stores the correct answer in `answer_a` and the + three distractors in `answer_b/c/d` (see DESIGN.md §3 "Important + consumer-side conventions"). We shuffle the four choices using a + seed derived from the CLI seed plus the question number, so the + same input always produces the same display order. After shuffle, + each choice gets a displayed label A/B/C/D, and we return which + label happened to land on the correct answer. + """ choices = [ - {"source": "answer_a", "text": question["answer_a"], "picture": question.get("picture_a"), "correct": True}, - {"source": "answer_b", "text": question["answer_b"], "picture": question.get("picture_b"), "correct": False}, - {"source": "answer_c", "text": question["answer_c"], "picture": question.get("picture_c"), "correct": False}, - {"source": "answer_d", "text": question["answer_d"], "picture": question.get("picture_d"), "correct": False}, + { + "source": "answer_a", + "text": question["answer_a"], + "picture": question.get("picture_a"), + "correct": True, + }, + { + "source": "answer_b", + "text": question["answer_b"], + "picture": question.get("picture_b"), + "correct": False, + }, + { + "source": "answer_c", + "text": question["answer_c"], + "picture": question.get("picture_c"), + "correct": False, + }, + { + "source": "answer_d", + "text": question["answer_d"], + "picture": question.get("picture_d"), + "correct": False, + }, ] rnd_seed = hashlib.sha256( f"{seed}:{question.get('number', '')}".encode("utf-8") @@ -182,8 +332,23 @@ def randomized_answers(question: dict, seed: str): return choices, correct["label"], correct -def render_question(item: QuestionItem, media: "MediaRegistry", seed: str): - """Render one question as `(front_html, back_html, correct_label)`.""" +# ============================================================================ +# HTML rendering +# ============================================================================ + + +def render_question(item, media, seed): + """Render one question as `(front_html, back_html, correct_label)`. + + The front shows the question stem, an optional figure, and the + four shuffled answer choices as an ordered list with `type="A"`. + The back names the displayed position of the correct answer and + repeats its text/figure. + + `media` is the `MediaRegistry` for this category; it records + which figure files are actually used so the packager can include + only those. + """ question = item.question choices, correct_label, correct = randomized_answers(question, seed) number = html.escape(str(question.get("number", ""))) @@ -197,16 +362,21 @@ def render_question(item: QuestionItem, media: "MediaRegistry", seed: str): ] q_image = media.image_html(question.get("picture_question")) if q_image: - front_parts.append(f'
{q_image}
') + front_parts.append( + f'
{q_image}
' + ) front_parts.append('
    ') for choice in choices: choice_image = media.image_html(choice.get("picture")) + image_html_part = ( + f'
    {choice_image}
    ' if choice_image else "" + ) front_parts.append( - '
  1. ' + "
  2. " f'
    {text_html(choice["text"])}
    ' - f'{f"
    {choice_image}
    " if choice_image else ""}' - '
    3. ' + f"{image_html_part}" + "" ) front_parts.append("
") @@ -217,24 +387,39 @@ def render_question(item: QuestionItem, media: "MediaRegistry", seed: str): f'
{text_html(correct["text"])}
', ] if correct_image: - back_parts.append(f'
{correct_image}
') + back_parts.append( + f'
{correct_image}
' + ) back_parts.append("") return "".join(front_parts), "".join(back_parts), correct_label -def display_path(path: tuple[str, ...]) -> str: - """Return the user-facing section path with the top prefix stripped.""" +def display_path(path): + """Return the user-facing section path with the top prefix stripped. + + Used for both the visible card breadcrumb and the stored `Path` + note field so the two never drift. + """ if not path: return "" return " / ".join([_short_category_title(path[0]), *path[1:]]) -def text_html(value) -> str: +# ============================================================================ +# LaTeX and HTML escaping +# ============================================================================ + + +def text_html(value): """Escape text for HTML, preserve line breaks, and enable MathJax. - Upstream uses bare `$...$` for inline LaTeX. Anki's built-in MathJax - recognizes `\\(...\\)`, so tokenize math spans and rewrite them while - still escaping HTML-sensitive characters. + The catalog uses bare `$...$` for inline LaTeX (DESIGN.md §3 + quotes the upstream README on this). Anki 2.1+ ships built-in + MathJax that recognizes `\\(...\\)` but not `$...$`, so we + tokenize the input into alternating math / non-math runs, escape + each run for HTML, and wrap math runs in `\\(...\\)` afterwards. + Non-math newlines become `
` and the catalog's safe inline + markup (`...`) survives escaping. """ parts = [] for is_math, chunk in tokenize_dollar_math(str(value)): @@ -242,16 +427,19 @@ def text_html(value) -> str: escaped = html.escape(chunk) parts.append(r"\(" + escaped + r"\)") else: - parts.append("
".join(escape_text_preserving_safe_tags(chunk).splitlines())) + escaped = escape_text_preserving_safe_tags(chunk) + parts.append("
".join(escaped.splitlines())) return "".join(parts) -def escape_text_preserving_safe_tags(text: str) -> str: - """Escape text while preserving the catalog's safe inline markup. +def escape_text_preserving_safe_tags(text): + """HTML-escape `text` while preserving exact `` / `` tags. - The live catalog uses `...` to emphasize negation in question - stems. Preserve only exact underline tags; escape everything else so - ordinary comparisons such as `2 < 3` and unexpected HTML stay safe. + The live catalog uses `...` to emphasize negation in + question stems (e.g. "Welche der folgenden Aussagen ist nicht + korrekt"). Preserving those tags keeps that emphasis visible on + the rendered card. Anything else — including ordinary mathematical + comparisons like `2 < 3` — still goes through `html.escape`. """ escaped_parts = [] for part in SAFE_INLINE_TAG_RE.split(text): @@ -262,11 +450,12 @@ def escape_text_preserving_safe_tags(text: str) -> str: return "".join(escaped_parts) -def tokenize_dollar_math(text: str): - """Yield `(is_math, text)` chunks, converting unescaped `$...$` pairs. +def tokenize_dollar_math(text): + """Yield `(is_math, chunk)` pairs, splitting on unescaped `$...$`. - This intentionally handles the simple inline convention used by the - BNetzA catalog. Unmatched dollars are treated as normal text. + A backslash-escaped `\\$` is treated as a literal dollar and does + not open or close a math span. Unmatched dollars (an opening `$` + with no closing partner) are treated as ordinary text. """ index = 0 while index < len(text): @@ -284,7 +473,8 @@ def tokenize_dollar_math(text: str): index = end + 1 -def find_unescaped_dollar(text: str, start: int): +def find_unescaped_dollar(text, start): + """Return the index of the next `$` not preceded by an odd run of `\\`.""" while True: pos = text.find("$", start) if pos == -1: @@ -299,24 +489,50 @@ def find_unescaped_dollar(text: str, start: int): start = pos + 1 -class MediaRegistry: - """Resolve catalog image stems and collect media files for an apkg.""" +# ============================================================================ +# Media registry +# ============================================================================ - def __init__(self, media_dir: Path): + +class MediaRegistry: + """Resolve catalog image stems and collect media files for one `.apkg`. + + Indexes every file in `media_dir` by filename. Each call to + `resolve(reference)` accepts the catalog's bare stem + (e.g. `"AB109_q"`) or a filename with extension; on a hit, the + `Path` is recorded in `used_paths` so the packager can include + only the files we actually referenced. + """ + + def __init__(self, media_dir): self.media_dir = media_dir - self.by_filename: dict[str, Path] = {} - self.used_paths: set[Path] = set() - self.missing: list[str] = [] + # filename → Path for every file under `media_dir` + self.by_filename = {} + # files referenced by at least one resolved reference + self.used_paths = set() + # references that didn't resolve (recorded for diagnostics) + self.missing = [] self._index_media_dir() def _index_media_dir(self): + """Populate `by_filename` from a single pass over `media_dir`.""" if not self.media_dir.exists(): return for path in self.media_dir.iterdir(): if path.is_file(): self.by_filename[path.name] = path - def resolve(self, reference) -> Path | None: + def resolve(self, reference): + """Return the resolved `Path`, or `None` if the reference is + missing. + + Picture references in the catalog are stems without + extensions (e.g. `"AB109_q"`); we try the stem with `.svg` + first, then `.png`. A reference that already carries an + extension is tried as-is. References containing path + separators are rejected outright — the catalog only ever + names a bare basename. + """ if not reference: return None ref = str(reference) @@ -324,11 +540,10 @@ class MediaRegistry: self.missing.append(ref) return None - candidates = [] if Path(ref).suffix: - candidates.append(ref) + candidates = [ref] else: - candidates.extend([f"{ref}.svg", f"{ref}.png"]) + candidates = [f"{ref}.svg", f"{ref}.png"] for candidate in candidates: path = self.by_filename.get(candidate) if path is not None: @@ -337,7 +552,8 @@ class MediaRegistry: self.missing.append(ref) return None - def image_html(self, reference) -> str: + def image_html(self, reference): + """Return an `` tag for `reference`, or `""` if unresolved.""" path = self.resolve(reference) if path is None: return "" @@ -345,29 +561,58 @@ class MediaRegistry: return f'' -def build_apkg_for_category( - category: Category, - edition_dir: Path, - out_path: Path, - *, - seed: str, - build_epoch: int, -): - """Write one category as an Anki `.apkg` file.""" +# ============================================================================ +# Tagging +# ============================================================================ + + +def tags_for_item(item): + """Return the Anki tags field for one note. + + Tags are space-separated and the field carries a leading and + trailing space — that's the convention Anki itself uses. We emit + a single `klasse-N/E/A` tag plus one `pfad-
` tag per + section level below the top-level Prüfungsteil. The question + number itself is intentionally NOT a tag (it's already in the + `Number` field; per-question tags would clutter Anki's tag tree + with 1750 singletons). + """ + question = item.question + class_tag = CLASS_TAGS.get(str(question.get("class")), "unknown") + tags = [f"klasse-{class_tag}"] + tags.extend(f"pfad-{slugify(part)}" for part in item.path[1:]) + return " " + " ".join(tags) + " " + + +# ============================================================================ +# Anki package writer (apkg ZIP + SVG post-processing) +# ============================================================================ + + +def build_apkg_for_category(category, edition_dir, out_path, seed, build_epoch): + """Write one category as an Anki `.apkg` file. + + Renders every question to a note, hand-rolls the v11 SQLite + collection, and writes the final ZIP with deterministic + timestamps. Returns a small result dict describing what was + written (for the CLI summary). + """ tmp_dir = out_path.parent / f".{out_path.name}.tmp" db_path = tmp_dir / "collection.anki2" - # Unlike the fetcher, this builder has no operator-recoverable state - # in its package temp file. A stale `.apkg.tmp` is just an incomplete - # output artifact, so we overwrite it on the next deterministic build. + + # Unlike the fetcher, this builder has no operator-recoverable + # state in its package temp file. A stale `.apkg.tmp` is just an + # incomplete output artifact, so we overwrite it on the next + # (deterministic) build. if tmp_dir.exists(): shutil.rmtree(tmp_dir) tmp_dir.mkdir(parents=True) + media = MediaRegistry(edition_dir / "svgs") deck_name = f"Amateurfunk::{category.short_title}" deck_id = stable_id("deck", deck_name) model_id = stable_id("model", "Amateurfunk Multiple Choice") - now = build_epoch try: notes = [] @@ -385,17 +630,16 @@ def build_apkg_for_category( back, correct_label, ] - notes.append( - { - "note_id": note_id, - "card_id": card_id, - "guid": stable_guid(f"{category.slug}:{number}"), - "fields": FIELD_SEP.join(fields), - "sort": number, - "tags": tags_for_item(item), - } - ) + notes.append({ + "note_id": note_id, + "card_id": card_id, + "guid": stable_guid(f"{category.slug}:{number}"), + "fields": FIELD_SEP.join(fields), + "sort": number, + "tags": tags_for_item(item), + }) + # Only the files actually referenced make it into the package. media_paths = {path.name: path for path in media.used_paths} create_collection_db( @@ -404,9 +648,11 @@ def build_apkg_for_category( deck_name=deck_name, model_id=model_id, notes=notes, - now=now, + now=build_epoch, + ) + write_apkg( + out_path, db_path, media_paths, build_epoch=build_epoch, ) - write_apkg(out_path, db_path, media_paths, build_epoch=build_epoch) finally: shutil.rmtree(tmp_dir, ignore_errors=True) @@ -419,27 +665,123 @@ def build_apkg_for_category( } -def tags_for_item(item: QuestionItem) -> str: - """Build Anki tag field with path metadata.""" - question = item.question - class_tag = CLASS_TAGS.get(str(question.get("class")), "unknown") - tags = [ - f"klasse-{class_tag}", - ] - tags.extend(f"pfad-{slugify(part)}" for part in item.path[1:]) - return " " + " ".join(tags) + " " +def write_apkg(out_path, db_path, media_paths, build_epoch): + """Write the Anki package ZIP atomically. + + The ZIP contains: + * `collection.anki2` — the SQLite database + * one numbered entry per media file (Anki addresses media by + sequential integer keys, not by filename) + * `media` — a JSON object mapping those keys back to filenames + + Every member is written with a fixed timestamp so the output is + byte-identical across runs of the same input. + """ + out_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = out_path.with_suffix(out_path.suffix + ".tmp") + media_map = {} + try: + with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zip_writestr_fixed( + zf, "collection.anki2", db_path.read_bytes(), + build_epoch=build_epoch, + ) + for index, filename in enumerate(sorted(media_paths)): + archive_name = str(index) + media_map[archive_name] = filename + zip_writestr_fixed( + zf, archive_name, + media_bytes_for_package(media_paths[filename]), + build_epoch=build_epoch, + ) + zip_writestr_fixed( + zf, "media", + json.dumps(media_map, ensure_ascii=False).encode("utf-8"), + build_epoch=build_epoch, + ) + os.replace(tmp_path, out_path) + except Exception: + tmp_path.unlink(missing_ok=True) + raise -def create_collection_db( - db_path: Path, - *, - deck_id: int, - deck_name: str, - model_id: int, - notes: list[dict], - now: int, -): - """Create the SQLite collection.anki2 database used inside an apkg.""" +def zip_writestr_fixed(zf, name, payload, build_epoch): + """Write one ZIP member with a deterministic timestamp + attrs.""" + info = zipfile.ZipInfo(name, zip_datetime(build_epoch)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + zf.writestr(info, payload) + + +def zip_datetime(epoch): + """Return a ZIP-compatible UTC date-time tuple for `epoch`. + + ZIP timestamps can't represent dates before 1980, so we clamp the + lower bound to 1980-01-01 UTC. This keeps the + `DEFAULT_BUILD_EPOCH = 0` fallback from blowing up. + """ + safe_epoch = max(int(epoch), 315532800) + value = dt.datetime.fromtimestamp(safe_epoch, dt.timezone.utc) + return ( + value.year, + value.month, + value.day, + value.hour, + value.minute, + value.second, + ) + + +def media_bytes_for_package(path): + """Return media bytes as they should be stored in the package. + + BNetzA SVGs are transparent. In Anki dark mode that makes the + black line drawings nearly invisible against the dark card + background. When packaging an SVG we inject an explicit white + background rectangle as the first painted element. Non-SVG files + (PNGs, etc.) are passed through untouched. The extracted source + files on disk are likewise never modified. + """ + raw = path.read_bytes() + if path.suffix.lower() != ".svg": + return raw + text = raw.decode("utf-8-sig", errors="replace") + return svg_with_white_background(text).encode("utf-8") + + +def svg_with_white_background(svg_text): + """Inject a white background `` after the opening ``. + + Idempotent: subsequent calls notice the `data-af-white-background` + marker and leave the SVG alone. If we can't find an opening + `` tag we return the input unchanged (better to ship the + catalog's SVG as-is than to refuse the whole package). + """ + if 'data-af-white-background="1"' in svg_text: + return svg_text + match = re.search(r"]*>", svg_text, flags=re.IGNORECASE) + if not match: + return svg_text + background = ( + '' + ) + return svg_text[:match.end()] + background + svg_text[match.end():] + + +# ============================================================================ +# SQLite collection schema +# ============================================================================ + + +def create_collection_db(db_path, deck_id, deck_name, model_id, notes, now): + """Create the `collection.anki2` SQLite database for one package. + + Uses the v11 schema, which modern Anki still understands (it + upgrades the collection on first open). We pre-write a single + `col` row with the JSON config blobs, then insert one `notes` + row and one `cards` row per question. + """ conn = sqlite3.connect(db_path) try: create_schema(conn) @@ -504,7 +846,8 @@ def create_collection_db( conn.close() -def create_schema(conn: sqlite3.Connection): +def create_schema(conn): + """Create the v11 Anki collection schema (tables + indices).""" conn.executescript( """ CREATE TABLE col ( @@ -581,14 +924,12 @@ def create_schema(conn: sqlite3.Connection): ) -def insert_collection_metadata( - conn: sqlite3.Connection, - *, - deck_id: int, - deck_name: str, - model_id: int, - now: int, -): +def insert_collection_metadata(conn, deck_id, deck_name, model_id, now): + """Write the single `col` row that carries the JSON config blobs. + + `crt` is a seconds-epoch creation time; `mod` and `scm` are in + milliseconds (Anki's mixed convention, not ours to fix). + """ conn.execute( """ INSERT INTO col @@ -605,15 +946,32 @@ def insert_collection_metadata( 0, 0, json.dumps(collection_conf(deck_id), separators=(",", ":")), - json.dumps({str(model_id): model_json(model_id, now)}, separators=(",", ":")), - json.dumps({str(deck_id): deck_json(deck_id, deck_name, now)}, separators=(",", ":")), + json.dumps( + {str(model_id): model_json(model_id, now)}, + separators=(",", ":"), + ), + json.dumps( + {str(deck_id): deck_json(deck_id, deck_name, now)}, + separators=(",", ":"), + ), json.dumps(default_deck_conf(now), separators=(",", ":")), "{}", ), ) -def collection_conf(deck_id: int): +# ============================================================================ +# Anki JSON config (col.conf / models / decks / dconf entries) +# ============================================================================ + + +def collection_conf(deck_id): + """Return the JSON written into `col.conf`. + + These knobs control review UI defaults (current deck, sort + column, etc.). They're not deck content — Anki rewrites most of + them on first open. + """ return { "nextPos": 1, "estTimes": True, @@ -631,7 +989,12 @@ def collection_conf(deck_id: int): } -def deck_json(deck_id: int, deck_name: str, now: int): +def deck_json(deck_id, deck_name, now): + """Return one deck entry for `col.decks`. + + The `Amateurfunk::Foo` naming gives the user a single top-level + Amateurfunk node in Anki's deck browser with three child decks. + """ return { "id": deck_id, "name": deck_name, @@ -651,7 +1014,8 @@ def deck_json(deck_id: int, deck_name: str, now: int): } -def default_deck_conf(now: int): +def default_deck_conf(now): + """Return a baseline `dconf` entry referenced by `deck_json`.""" return { "1": { "id": 1, @@ -691,7 +1055,15 @@ def default_deck_conf(now: int): } -def model_json(model_id: int, now: int): +def model_json(model_id, now): + """Return the note-type entry for `col.models`. + + A single template (`Card 1`) renders the `Front` field, then the + horizontal rule, then the `Back` field — the standard Anki + "front / divider / back" layout. The other fields (`Number`, + `Category`, `Path`, `CorrectLabel`) are stored for searchability + and exports. + """ fields = [ field_json("Number", 0), field_json("Category", 1), @@ -721,14 +1093,22 @@ def model_json(model_id: int, now: int): ], "flds": fields, "css": CARD_CSS, - "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage[utf8]{inputenc}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\begin{document}", + "latexPre": ( + "\\documentclass[12pt]{article}\n" + "\\special{papersize=3in,5in}\n" + "\\usepackage[utf8]{inputenc}\n" + "\\usepackage{amssymb,amsmath}\n" + "\\pagestyle{empty}\n" + "\\begin{document}" + ), "latexPost": "\\end{document}", "req": [[0, "any", [3]]], "vers": [], } -def field_json(name: str, ord_: int): +def field_json(name, ord_): + """Return one field-definition entry for a model's `flds` list.""" return { "name": name, "ord": ord_, @@ -745,6 +1125,12 @@ def field_json(name: str, ord_: int): } +# ============================================================================ +# Card styling +# ============================================================================ + +# Inlined into the model's `css` field. Kept readable here rather than +# minified; Anki doesn't care about whitespace. CARD_CSS = """ .card { font-family: Arial, sans-serif; @@ -786,108 +1172,19 @@ CARD_CSS = """ """ -def write_apkg( - out_path: Path, - db_path: Path, - media_paths: dict[str, Path], - *, - build_epoch: int, -): - """Write Anki package ZIP with collection DB and media files.""" - out_path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = out_path.with_suffix(out_path.suffix + ".tmp") - media_map = {} - try: - with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - zip_writestr_fixed( - zf, - "collection.anki2", - db_path.read_bytes(), - build_epoch=build_epoch, - ) - for index, filename in enumerate(sorted(media_paths)): - archive_name = str(index) - media_map[archive_name] = filename - zip_writestr_fixed( - zf, - archive_name, - media_bytes_for_package(media_paths[filename]), - build_epoch=build_epoch, - ) - zip_writestr_fixed( - zf, - "media", - json.dumps(media_map, ensure_ascii=False).encode("utf-8"), - build_epoch=build_epoch, - ) - os.replace(tmp_path, out_path) - except Exception: - tmp_path.unlink(missing_ok=True) - raise +# ============================================================================ +# Build epoch and orchestration +# ============================================================================ -def zip_writestr_fixed( - zf: zipfile.ZipFile, - name: str, - payload: bytes, - *, - build_epoch: int, -): - """Write a ZIP member with deterministic timestamp and attributes.""" - info = zipfile.ZipInfo(name, zip_datetime(build_epoch)) - info.compress_type = zipfile.ZIP_DEFLATED - info.external_attr = 0o644 << 16 - zf.writestr(info, payload) +def build_epoch_from_manifest(manifest, override_epoch=None): + """Decide which integer epoch to stamp into the package. - -def zip_datetime(epoch: int): - """Return a ZIP-compatible UTC timestamp tuple. - - ZIP timestamps cannot represent dates before 1980. + Priority is `--epoch` override → manifest `fetched_at` → + `DEFAULT_BUILD_EPOCH`. A non-empty but unparseable `fetched_at` + is treated as a hard error (the manifest is supposed to carry an + ISO-8601 UTC string). """ - safe_epoch = max(int(epoch), 315532800) - value = datetime.fromtimestamp(safe_epoch, timezone.utc) - return ( - value.year, - value.month, - value.day, - value.hour, - value.minute, - value.second, - ) - - -def media_bytes_for_package(path: Path) -> bytes: - """Return media bytes as they should be stored in the Anki package. - - BNetzA SVGs are transparent. In Anki dark mode that makes black line - drawings hard to read because the card background can be black. When - packaging SVGs, add an explicit white background rectangle as the - first painted element. The extracted source SVGs are left untouched. - """ - raw = path.read_bytes() - if path.suffix.lower() != ".svg": - return raw - text = raw.decode("utf-8-sig", errors="replace") - return svg_with_white_background(text).encode("utf-8") - - -def svg_with_white_background(svg_text: str) -> str: - """Inject a white background rect immediately after the opening svg tag.""" - if 'data-af-white-background="1"' in svg_text: - return svg_text - match = re.search(r"]*>", svg_text, flags=re.IGNORECASE) - if not match: - return svg_text - background = ( - '' - ) - return svg_text[:match.end()] + background + svg_text[match.end():] - - -def build_epoch_from_manifest(manifest: dict, override_epoch: int | None = None) -> int: - """Return deterministic build timestamp from manifest or CLI override.""" if override_epoch is not None: return int(override_epoch) fetched_at = manifest.get("fetched_at") @@ -895,18 +1192,20 @@ def build_epoch_from_manifest(manifest: dict, override_epoch: int | None = None) return DEFAULT_BUILD_EPOCH try: normalized = str(fetched_at).replace("Z", "+00:00") - return int(datetime.fromisoformat(normalized).timestamp()) + return int(dt.datetime.fromisoformat(normalized).timestamp()) except ValueError as e: - raise AnkiBuildError(f"invalid manifest fetched_at value: {fetched_at!r}") from e + raise AnkiBuildError( + f"invalid manifest fetched_at value: {fetched_at!r}" + ) from e -def build_all( - data_dir: Path, - out_dir: Path, - *, - seed: str, - override_epoch: int | None = None, -): +def build_all(data_dir, out_dir, seed, override_epoch=None): + """Build every category's `.apkg` and return their result dicts. + + Loads the latest fetched catalog, picks a build epoch, then walks + the three top-level categories writing one `.apkg` each. Raises + `AnkiBuildError` on configuration / catalog problems. + """ edition_dir, manifest, catalog = load_latest_catalog(data_dir) build_epoch = build_epoch_from_manifest(manifest, override_epoch) categories = collect_categories(catalog) @@ -928,15 +1227,27 @@ def build_all( return results +# ============================================================================ +# Main entry point +# ============================================================================ + + def _parse_args(argv): + """Build the argparse object and parse `argv`.""" parser = argparse.ArgumentParser( - description="Build Anki .apkg decks from an extracted BNetzA catalog.", + description=( + "Build Anki .apkg decks from an extracted BNetzA " + "question catalog." + ), ) parser.add_argument( "--data", type=Path, default=DEFAULT_DATA_DIR, - help="fetch output directory containing manifest-latest.json (default: ./data)", + help=( + "fetch output directory containing manifest-latest.json " + "(default: ./data)" + ), ) parser.add_argument( "--out", @@ -955,13 +1266,14 @@ def _parse_args(argv): default=None, help=( "override the package timestamp epoch; by default this is " - "derived from manifest.json fetched_at" + "derived from manifest.json's fetched_at" ), ) return parser.parse_args(argv) def main(argv=None): + """Top-level entry point. Returns an exit code; never raises.""" args = _parse_args(argv) try: results = build_all( @@ -973,8 +1285,10 @@ def main(argv=None): except AnkiBuildError as e: print(f"error: {e}", file=sys.stderr) return EXIT_ERROR - except Exception as e: # noqa: BLE001 - print(f"error: failed to build Anki packages: {e}", file=sys.stderr) + except Exception as e: # noqa: BLE001 (main() promises not to raise) + print( + f"error: failed to build Anki packages: {e}", file=sys.stderr, + ) return EXIT_ERROR for result in results: @@ -984,8 +1298,8 @@ def main(argv=None): ) if result["missing_media"]: print( - f"warning: {len(result['missing_media'])} missing media reference(s) " - f"in {result['deck']}", + f"warning: {len(result['missing_media'])} missing media " + f"reference(s) in {result['deck']}", file=sys.stderr, ) return EXIT_OK diff --git a/test_amateurfunk_anki.py b/test_amateurfunk_anki.py index 7960ceb..7dc33f2 100644 --- a/test_amateurfunk_anki.py +++ b/test_amateurfunk_anki.py @@ -4,16 +4,14 @@ These tests inspect the generated `.apkg` package structure directly: ZIP entries, media map, and the SQLite collection database. They do not require Anki itself to be installed. """ -from __future__ import annotations import json import sqlite3 import tempfile +import unittest import zipfile from pathlib import Path -import unittest - import amateurfunk_anki as aa diff --git a/test_amateurfunk_fetch.py b/test_amateurfunk_fetch.py index bc13d8c..37541bd 100644 --- a/test_amateurfunk_fetch.py +++ b/test_amateurfunk_fetch.py @@ -4,7 +4,6 @@ Network-bound tests are not included; the real BNetzA fetch lives in the manual smoke-test invocation. Everything here runs against fixture ZIPs built in temp directories. """ -from __future__ import annotations import hashlib import json