Add Regression-Test: Render img with pymupdf and compare visually against test fixtures

tests/test_regression.py

@@ -0,0 +1,270 @@
+"""Regression tests: pixel-diff comparison of script-generated vs. manually created reference PDFs.
+
+Fixture layout
+--------------
+tests/fixtures/
+    YYYY_MMM/                    ← directory name encodes year and Excel sheet name
+        reference/
+            (1) Name.pdf         ← manually created ground-truth PDFs
+        diff/                    ← diff images written here on failure
+
+A fixture is discovered automatically when:
+  - The directory name matches  YYYY_MMM  (e.g. 2026_Feb)
+  - At least one *.xlsx / *.xlsm file exists in the directory or in input/
+  - At least one *.pdf file exists in reference/
+
+Rendering
+---------
+Both reference and generated PDFs are rendered to images using pymupdf
+(no browser required). The images are then compared pixel-by-pixel.
+
+Similarity threshold
+--------------------
+SIMILARITY_THRESHOLD (default 95 %) — fraction of pixels whose R, G, B channels
+each differ by at most PIXEL_TOLERANCE (default 5) between reference and generated PDF.
+
+Diff images
+-----------
+When a page fails the threshold, three images are saved to
+  tests/fixtures/YYYY_MMM/diff/<pdf-stem>/seite_NN/
+    - seite_NN_reference.png
+    - seite_NN_generated.png
+    - seite_NN_diff.png  (heatmap: red = difference)
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import pymupdf
+import pytest
+from PIL import Image
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+SIMILARITY_THRESHOLD = 0.95   # minimum fraction of matching pixels per page
+PIXEL_TOLERANCE = 5           # max per-channel delta still counted as "same"
+RENDER_DPI = 150              # rendering resolution for PDF → image
+
+
+# ---------------------------------------------------------------------------
+# Fixture discovery (runs at collection time)
+# ---------------------------------------------------------------------------
+
+def _discover_fixture_params() -> list:
+    """Return pytest.param entries for every ready fixture directory.
+
+    Excel file lookup order:
+      1. Any *.xlsx / *.xlsm directly inside the fixture directory.
+      2. Any *.xlsx / *.xlsm inside the project-level ``input/`` directory
+         whose filename contains the fixture year (e.g. "2026").
+    """
+    if not FIXTURES_DIR.exists():
+        return []
+
+    project_root = FIXTURES_DIR.parent.parent
+    input_dir = project_root / "input"
+
+    params = []
+    for d in sorted(FIXTURES_DIR.iterdir()):
+        if not d.is_dir():
+            continue
+
+        parts = d.name.split("_", 1)
+        if len(parts) != 2 or not parts[0].isdigit() or len(parts[0]) != 4:
+            continue
+        year_str, month = parts
+
+        # 1. Excel file in fixture directory
+        xlsx_files = list(d.glob("*.xlsx")) + list(d.glob("*.xlsm"))
+
+        # 2. Fallback: input/ directory
+        if not xlsx_files and input_dir.exists():
+            xlsx_files = [
+                f for f in (list(input_dir.glob("*.xlsx")) + list(input_dir.glob("*.xlsm")))
+                if year_str in f.stem
+            ]
+
+        if not xlsx_files:
+            continue
+
+        ref_dir = d / "reference"
+        if not ref_dir.exists() or not list(ref_dir.glob("*.pdf")):
+            continue
+
+        params.append(
+            pytest.param(d, xlsx_files[0], month, ref_dir, id=d.name)
+        )
+
+    return params
+
+
+# ---------------------------------------------------------------------------
+# PDF rendering via pymupdf
+# ---------------------------------------------------------------------------
+
+def _render_pdf_pages(pdf_path: Path, dpi: int = RENDER_DPI) -> list[np.ndarray]:
+    """Render all pages of a PDF to RGB numpy arrays using pymupdf."""
+    zoom = dpi / 72.0
+    mat = pymupdf.Matrix(zoom, zoom)
+    doc = pymupdf.open(str(pdf_path))
+    images = []
+    for page in doc:
+        pix = page.get_pixmap(matrix=mat)
+        img = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.h, pix.w, pix.n)
+        # Convert RGBA → RGB if needed
+        if pix.n == 4:
+            img = img[:, :, :3]
+        images.append(img.copy())
+    doc.close()
+    return images
+
+
+# ---------------------------------------------------------------------------
+# Comparison helpers
+# ---------------------------------------------------------------------------
+
+def _resize_to_match(img: np.ndarray, target_shape: tuple) -> np.ndarray:
+    """Resize *img* so its (h, w) matches *target_shape*."""
+    h, w = target_shape[:2]
+    pil = Image.fromarray(img).resize((w, h), Image.LANCZOS)
+    return np.array(pil)
+
+
+def _compute_similarity(ref: np.ndarray, gen: np.ndarray) -> float:
+    """Return the fraction of pixels where all RGB channels differ by at most PIXEL_TOLERANCE."""
+    if ref.shape != gen.shape:
+        gen = _resize_to_match(gen, ref.shape)
+    diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
+    matching = np.all(diff <= PIXEL_TOLERANCE, axis=2)
+    return float(matching.mean())
+
+
+def _save_diff_images(ref: np.ndarray, gen: np.ndarray, out_dir: Path, page_label: str) -> None:
+    """Save reference, generated, and heatmap diff images for a failing page."""
+    if ref.shape != gen.shape:
+        gen = _resize_to_match(gen, ref.shape)
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    Image.fromarray(ref).save(str(out_dir / f"{page_label}_reference.png"))
+    Image.fromarray(gen).save(str(out_dir / f"{page_label}_generated.png"))
+
+    # Heatmap: identical areas dimmed, differences highlighted in red
+    diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
+    max_diff_per_pixel = diff.max(axis=2)
+    severity = max_diff_per_pixel / 255.0
+
+    base = (ref * 0.5).astype(np.uint8)
+    overlay = np.zeros_like(base)
+    overlay[:, :, 0] = (severity * 255).astype(np.uint8)
+    alpha = severity[:, :, np.newaxis]
+    blended = (base * (1.0 - alpha) + overlay * alpha).astype(np.uint8)
+    blended[max_diff_per_pixel > 30] = [255, 0, 0]
+
+    Image.fromarray(blended).save(str(out_dir / f"{page_label}_diff.png"))
+
+
+# ---------------------------------------------------------------------------
+# Main regression test
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("fixture_dir,excel_file,month,ref_dir", _discover_fixture_params())
+def test_pdf_matches_reference(
+    fixture_dir: Path,
+    excel_file: Path,
+    month: str,
+    ref_dir: Path,
+    tmp_path: Path,
+) -> None:
+    """Generate PDFs for one fixture month and compare pixel-by-pixel against reference PDFs."""
+
+    project_root = Path(__file__).parent.parent
+
+    # ------------------------------------------------------------------
+    # 1. Generate PDFs via the CLI into a temporary directory
+    # ------------------------------------------------------------------
+    result = subprocess.run(
+        [
+            sys.executable, "generate.py", "generate",
+            "--file", str(excel_file),
+            "--month", month,
+            "--out", str(tmp_path),
+        ],
+        input="j\n",
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        cwd=str(project_root),
+    )
+
+    if result.returncode != 0:
+        pytest.fail(
+            f"generate.py schlug fehl für Fixture '{fixture_dir.name}':\n"
+            f"STDOUT:\n{result.stdout}\n"
+            f"STDERR:\n{result.stderr}"
+        )
+
+    # ------------------------------------------------------------------
+    # 2. Compare each reference PDF against the generated counterpart
+    # ------------------------------------------------------------------
+    diff_dir = fixture_dir / "diff"
+    reference_pdfs = sorted(ref_dir.glob("*.pdf"))
+    failures: list[str] = []
+
+    for ref_pdf in reference_pdfs:
+        # Skip PDFs with only a sequence number and no name
+        if re.fullmatch(r'\(\d+\)', ref_pdf.stem.strip()):
+            continue
+
+        # Strip manual markers (e.g. " #") from stem to find the generated file
+        clean_stem = re.sub(r'\s*#\s*$', '', ref_pdf.stem).rstrip()
+        generated_pdf = tmp_path / (clean_stem + ref_pdf.suffix)
+
+        if not generated_pdf.exists():
+            generated_pdf = tmp_path / ref_pdf.name
+
+        if not generated_pdf.exists():
+            failures.append(f"  FEHLT: '{ref_pdf.name}' wurde nicht generiert.")
+            continue
+
+        # Render both PDFs to images
+        ref_images = _render_pdf_pages(ref_pdf)
+        gen_images = _render_pdf_pages(generated_pdf)
+        page_count = max(len(ref_images), len(gen_images))
+
+        for page_idx in range(page_count):
+            page_label = f"'{ref_pdf.name}' Seite {page_idx + 1}"
+
+            if page_idx >= len(ref_images):
+                failures.append(f"  {page_label}: Seite nur in generierter PDF vorhanden.")
+                continue
+            if page_idx >= len(gen_images):
+                failures.append(f"  {page_label}: Seite fehlt in generierter PDF.")
+                continue
+
+            similarity = _compute_similarity(ref_images[page_idx], gen_images[page_idx])
+
+            if similarity < SIMILARITY_THRESHOLD:
+                page_tag = f"seite_{page_idx + 1:02d}"
+                img_dir = diff_dir / clean_stem / page_tag
+                _save_diff_images(
+                    ref_images[page_idx], gen_images[page_idx], img_dir, page_tag
+                )
+                rel = img_dir.relative_to(project_root)
+                failures.append(
+                    f"  {page_label}: Ähnlichkeit {similarity:.1%} "
+                    f"(Schwelle {SIMILARITY_THRESHOLD:.0%})  →  Bilder: {rel}/"
+                )
+
+    if failures:
+        failure_list = "\n".join(failures)
+        pytest.fail(
+            f"\nPixel-Diff-Fehler in Fixture '{fixture_dir.name}':\n{failure_list}"
+        )