From 3fc506c40e493fe1b1b0a14f6b89db3e6d95faf3 Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 16 Apr 2026 13:09:34 +0200 Subject: [PATCH] Add Regression-Test: Render img with pymupdf and compare visually against test fixtures --- .gitignore | 6 + README.md | 10 +- requirements-dev.txt | 3 + tests/test_regression.py | 270 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 tests/test_regression.py diff --git a/.gitignore b/.gitignore index e8eaa97..92e7005 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ earlier_versions/ .github/.DS_Store output __pycache__ + +# Fixture-Testdaten (personenbezogene Daten — nicht versioniert) +tests/fixtures/**/*.xlsx +tests/fixtures/**/*.xlsm +tests/fixtures/**/reference/*.pdf +tests/fixtures/**/diff/ diff --git a/README.md b/README.md index db2d1d7..e6ca24b 100644 --- a/README.md +++ b/README.md @@ -145,11 +145,19 @@ Die laufende Nummer entspricht der Nummerierung in Spalte A der Excel-Datei. ## Tests ausführen +**Alle Tests starten** + ```bash pytest tests/ -v ``` -Die Tests benötigen keine echten Excel- oder PDF-Dateien und laufen vollständig mit synthetischen Daten. +**Nur den Regression-Test.** +*Der Test erwartet, dass im Ordner 2026_Feb die Excel-Datei und in reference/ die Ground-Truth-PDFs liegen. Er generiert PDFs in ein Temp-Verzeichnis und vergleicht sie pixelweise. Bei Abweichungen werden Diff-Bilder nach diff geschrieben.* + +```bash +python -m pytest tests/test_regression.py -v +``` + --- diff --git a/requirements-dev.txt b/requirements-dev.txt index 17d7648..d9b1aa7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,5 @@ -r requirements.txt pytest>=7.0.0 +pymupdf>=1.23.0 +numpy>=1.26.0 +Pillow>=10.0.0 diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000..27491d1 --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,270 @@ +"""Regression tests: pixel-diff comparison of script-generated vs. manually created reference PDFs. + +Fixture layout +-------------- +tests/fixtures/ + YYYY_MMM/ ← directory name encodes year and Excel sheet name + reference/ + (1) Name.pdf ← manually created ground-truth PDFs + diff/ ← diff images written here on failure + +A fixture is discovered automatically when: + - The directory name matches YYYY_MMM (e.g. 2026_Feb) + - At least one *.xlsx / *.xlsm file exists in the directory or in input/ + - At least one *.pdf file exists in reference/ + +Rendering +--------- +Both reference and generated PDFs are rendered to images using pymupdf +(no browser required). The images are then compared pixel-by-pixel. + +Similarity threshold +-------------------- +SIMILARITY_THRESHOLD (default 95 %) — fraction of pixels whose R, G, B channels +each differ by at most PIXEL_TOLERANCE (default 5) between reference and generated PDF. + +Diff images +----------- +When a page fails the threshold, three images are saved to + tests/fixtures/YYYY_MMM/diff//seite_NN/ + - seite_NN_reference.png + - seite_NN_generated.png + - seite_NN_diff.png (heatmap: red = difference) +""" +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pymupdf +import pytest +from PIL import Image + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +FIXTURES_DIR = Path(__file__).parent / "fixtures" +SIMILARITY_THRESHOLD = 0.95 # minimum fraction of matching pixels per page +PIXEL_TOLERANCE = 5 # max per-channel delta still counted as "same" +RENDER_DPI = 150 # rendering resolution for PDF → image + + +# --------------------------------------------------------------------------- +# Fixture discovery (runs at collection time) +# --------------------------------------------------------------------------- + +def _discover_fixture_params() -> list: + """Return pytest.param entries for every ready fixture directory. + + Excel file lookup order: + 1. Any *.xlsx / *.xlsm directly inside the fixture directory. + 2. Any *.xlsx / *.xlsm inside the project-level ``input/`` directory + whose filename contains the fixture year (e.g. "2026"). + """ + if not FIXTURES_DIR.exists(): + return [] + + project_root = FIXTURES_DIR.parent.parent + input_dir = project_root / "input" + + params = [] + for d in sorted(FIXTURES_DIR.iterdir()): + if not d.is_dir(): + continue + + parts = d.name.split("_", 1) + if len(parts) != 2 or not parts[0].isdigit() or len(parts[0]) != 4: + continue + year_str, month = parts + + # 1. Excel file in fixture directory + xlsx_files = list(d.glob("*.xlsx")) + list(d.glob("*.xlsm")) + + # 2. Fallback: input/ directory + if not xlsx_files and input_dir.exists(): + xlsx_files = [ + f for f in (list(input_dir.glob("*.xlsx")) + list(input_dir.glob("*.xlsm"))) + if year_str in f.stem + ] + + if not xlsx_files: + continue + + ref_dir = d / "reference" + if not ref_dir.exists() or not list(ref_dir.glob("*.pdf")): + continue + + params.append( + pytest.param(d, xlsx_files[0], month, ref_dir, id=d.name) + ) + + return params + + +# --------------------------------------------------------------------------- +# PDF rendering via pymupdf +# --------------------------------------------------------------------------- + +def _render_pdf_pages(pdf_path: Path, dpi: int = RENDER_DPI) -> list[np.ndarray]: + """Render all pages of a PDF to RGB numpy arrays using pymupdf.""" + zoom = dpi / 72.0 + mat = pymupdf.Matrix(zoom, zoom) + doc = pymupdf.open(str(pdf_path)) + images = [] + for page in doc: + pix = page.get_pixmap(matrix=mat) + img = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.h, pix.w, pix.n) + # Convert RGBA → RGB if needed + if pix.n == 4: + img = img[:, :, :3] + images.append(img.copy()) + doc.close() + return images + + +# --------------------------------------------------------------------------- +# Comparison helpers +# --------------------------------------------------------------------------- + +def _resize_to_match(img: np.ndarray, target_shape: tuple) -> np.ndarray: + """Resize *img* so its (h, w) matches *target_shape*.""" + h, w = target_shape[:2] + pil = Image.fromarray(img).resize((w, h), Image.LANCZOS) + return np.array(pil) + + +def _compute_similarity(ref: np.ndarray, gen: np.ndarray) -> float: + """Return the fraction of pixels where all RGB channels differ by at most PIXEL_TOLERANCE.""" + if ref.shape != gen.shape: + gen = _resize_to_match(gen, ref.shape) + diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32)) + matching = np.all(diff <= PIXEL_TOLERANCE, axis=2) + return float(matching.mean()) + + +def _save_diff_images(ref: np.ndarray, gen: np.ndarray, out_dir: Path, page_label: str) -> None: + """Save reference, generated, and heatmap diff images for a failing page.""" + if ref.shape != gen.shape: + gen = _resize_to_match(gen, ref.shape) + + out_dir.mkdir(parents=True, exist_ok=True) + + Image.fromarray(ref).save(str(out_dir / f"{page_label}_reference.png")) + Image.fromarray(gen).save(str(out_dir / f"{page_label}_generated.png")) + + # Heatmap: identical areas dimmed, differences highlighted in red + diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32)) + max_diff_per_pixel = diff.max(axis=2) + severity = max_diff_per_pixel / 255.0 + + base = (ref * 0.5).astype(np.uint8) + overlay = np.zeros_like(base) + overlay[:, :, 0] = (severity * 255).astype(np.uint8) + alpha = severity[:, :, np.newaxis] + blended = (base * (1.0 - alpha) + overlay * alpha).astype(np.uint8) + blended[max_diff_per_pixel > 30] = [255, 0, 0] + + Image.fromarray(blended).save(str(out_dir / f"{page_label}_diff.png")) + + +# --------------------------------------------------------------------------- +# Main regression test +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("fixture_dir,excel_file,month,ref_dir", _discover_fixture_params()) +def test_pdf_matches_reference( + fixture_dir: Path, + excel_file: Path, + month: str, + ref_dir: Path, + tmp_path: Path, +) -> None: + """Generate PDFs for one fixture month and compare pixel-by-pixel against reference PDFs.""" + + project_root = Path(__file__).parent.parent + + # ------------------------------------------------------------------ + # 1. Generate PDFs via the CLI into a temporary directory + # ------------------------------------------------------------------ + result = subprocess.run( + [ + sys.executable, "generate.py", "generate", + "--file", str(excel_file), + "--month", month, + "--out", str(tmp_path), + ], + input="j\n", + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + cwd=str(project_root), + ) + + if result.returncode != 0: + pytest.fail( + f"generate.py schlug fehl für Fixture '{fixture_dir.name}':\n" + f"STDOUT:\n{result.stdout}\n" + f"STDERR:\n{result.stderr}" + ) + + # ------------------------------------------------------------------ + # 2. Compare each reference PDF against the generated counterpart + # ------------------------------------------------------------------ + diff_dir = fixture_dir / "diff" + reference_pdfs = sorted(ref_dir.glob("*.pdf")) + failures: list[str] = [] + + for ref_pdf in reference_pdfs: + # Skip PDFs with only a sequence number and no name + if re.fullmatch(r'\(\d+\)', ref_pdf.stem.strip()): + continue + + # Strip manual markers (e.g. " #") from stem to find the generated file + clean_stem = re.sub(r'\s*#\s*$', '', ref_pdf.stem).rstrip() + generated_pdf = tmp_path / (clean_stem + ref_pdf.suffix) + + if not generated_pdf.exists(): + generated_pdf = tmp_path / ref_pdf.name + + if not generated_pdf.exists(): + failures.append(f" FEHLT: '{ref_pdf.name}' wurde nicht generiert.") + continue + + # Render both PDFs to images + ref_images = _render_pdf_pages(ref_pdf) + gen_images = _render_pdf_pages(generated_pdf) + page_count = max(len(ref_images), len(gen_images)) + + for page_idx in range(page_count): + page_label = f"'{ref_pdf.name}' Seite {page_idx + 1}" + + if page_idx >= len(ref_images): + failures.append(f" {page_label}: Seite nur in generierter PDF vorhanden.") + continue + if page_idx >= len(gen_images): + failures.append(f" {page_label}: Seite fehlt in generierter PDF.") + continue + + similarity = _compute_similarity(ref_images[page_idx], gen_images[page_idx]) + + if similarity < SIMILARITY_THRESHOLD: + page_tag = f"seite_{page_idx + 1:02d}" + img_dir = diff_dir / clean_stem / page_tag + _save_diff_images( + ref_images[page_idx], gen_images[page_idx], img_dir, page_tag + ) + rel = img_dir.relative_to(project_root) + failures.append( + f" {page_label}: Ähnlichkeit {similarity:.1%} " + f"(Schwelle {SIMILARITY_THRESHOLD:.0%}) → Bilder: {rel}/" + ) + + if failures: + failure_list = "\n".join(failures) + pytest.fail( + f"\nPixel-Diff-Fehler in Fixture '{fixture_dir.name}':\n{failure_list}" + ) \ No newline at end of file