270 lines
9.7 KiB
Python
270 lines
9.7 KiB
Python
"""Regression tests: pixel-diff comparison of script-generated vs. manually created reference PDFs.
|
|
|
|
Fixture layout
|
|
--------------
|
|
tests/fixtures/
|
|
YYYY_MMM/ ← directory name encodes year and Excel sheet name
|
|
reference/
|
|
(1) Name.pdf ← manually created ground-truth PDFs
|
|
diff/ ← diff images written here on failure
|
|
|
|
A fixture is discovered automatically when:
|
|
- The directory name matches YYYY_MMM (e.g. 2026_Feb)
|
|
- At least one *.xlsx / *.xlsm file exists in the directory or in input/
|
|
- At least one *.pdf file exists in reference/
|
|
|
|
Rendering
|
|
---------
|
|
Both reference and generated PDFs are rendered to images using pymupdf
|
|
(no browser required). The images are then compared pixel-by-pixel.
|
|
|
|
Similarity threshold
|
|
--------------------
|
|
SIMILARITY_THRESHOLD (default 95 %) — fraction of pixels whose R, G, B channels
|
|
each differ by at most PIXEL_TOLERANCE (default 5) between reference and generated PDF.
|
|
|
|
Diff images
|
|
-----------
|
|
When a page fails the threshold, three images are saved to
|
|
tests/fixtures/YYYY_MMM/diff/<pdf-stem>/seite_NN/
|
|
- seite_NN_reference.png
|
|
- seite_NN_generated.png
|
|
- seite_NN_diff.png (heatmap: red = difference)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pymupdf
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
FIXTURES_DIR = Path(__file__).parent / "fixtures"
|
|
SIMILARITY_THRESHOLD = 0.95 # minimum fraction of matching pixels per page
|
|
PIXEL_TOLERANCE = 5 # max per-channel delta still counted as "same"
|
|
RENDER_DPI = 150 # rendering resolution for PDF → image
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture discovery (runs at collection time)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _discover_fixture_params() -> list:
|
|
"""Return pytest.param entries for every ready fixture directory.
|
|
|
|
Excel file lookup order:
|
|
1. Any *.xlsx / *.xlsm directly inside the fixture directory.
|
|
2. Any *.xlsx / *.xlsm inside the project-level ``input/`` directory
|
|
whose filename contains the fixture year (e.g. "2026").
|
|
"""
|
|
if not FIXTURES_DIR.exists():
|
|
return []
|
|
|
|
project_root = FIXTURES_DIR.parent.parent
|
|
input_dir = project_root / "input"
|
|
|
|
params = []
|
|
for d in sorted(FIXTURES_DIR.iterdir()):
|
|
if not d.is_dir():
|
|
continue
|
|
|
|
parts = d.name.split("_", 1)
|
|
if len(parts) != 2 or not parts[0].isdigit() or len(parts[0]) != 4:
|
|
continue
|
|
year_str, month = parts
|
|
|
|
# 1. Excel file in fixture directory
|
|
xlsx_files = list(d.glob("*.xlsx")) + list(d.glob("*.xlsm"))
|
|
|
|
# 2. Fallback: input/ directory
|
|
if not xlsx_files and input_dir.exists():
|
|
xlsx_files = [
|
|
f for f in (list(input_dir.glob("*.xlsx")) + list(input_dir.glob("*.xlsm")))
|
|
if year_str in f.stem
|
|
]
|
|
|
|
if not xlsx_files:
|
|
continue
|
|
|
|
ref_dir = d / "reference"
|
|
if not ref_dir.exists() or not list(ref_dir.glob("*.pdf")):
|
|
continue
|
|
|
|
params.append(
|
|
pytest.param(d, xlsx_files[0], month, ref_dir, id=d.name)
|
|
)
|
|
|
|
return params
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PDF rendering via pymupdf
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _render_pdf_pages(pdf_path: Path, dpi: int = RENDER_DPI) -> list[np.ndarray]:
|
|
"""Render all pages of a PDF to RGB numpy arrays using pymupdf."""
|
|
zoom = dpi / 72.0
|
|
mat = pymupdf.Matrix(zoom, zoom)
|
|
doc = pymupdf.open(str(pdf_path))
|
|
images = []
|
|
for page in doc:
|
|
pix = page.get_pixmap(matrix=mat)
|
|
img = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.h, pix.w, pix.n)
|
|
# Convert RGBA → RGB if needed
|
|
if pix.n == 4:
|
|
img = img[:, :, :3]
|
|
images.append(img.copy())
|
|
doc.close()
|
|
return images
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Comparison helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _resize_to_match(img: np.ndarray, target_shape: tuple) -> np.ndarray:
|
|
"""Resize *img* so its (h, w) matches *target_shape*."""
|
|
h, w = target_shape[:2]
|
|
pil = Image.fromarray(img).resize((w, h), Image.LANCZOS)
|
|
return np.array(pil)
|
|
|
|
|
|
def _compute_similarity(ref: np.ndarray, gen: np.ndarray) -> float:
|
|
"""Return the fraction of pixels where all RGB channels differ by at most PIXEL_TOLERANCE."""
|
|
if ref.shape != gen.shape:
|
|
gen = _resize_to_match(gen, ref.shape)
|
|
diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
|
|
matching = np.all(diff <= PIXEL_TOLERANCE, axis=2)
|
|
return float(matching.mean())
|
|
|
|
|
|
def _save_diff_images(ref: np.ndarray, gen: np.ndarray, out_dir: Path, page_label: str) -> None:
|
|
"""Save reference, generated, and heatmap diff images for a failing page."""
|
|
if ref.shape != gen.shape:
|
|
gen = _resize_to_match(gen, ref.shape)
|
|
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
Image.fromarray(ref).save(str(out_dir / f"{page_label}_reference.png"))
|
|
Image.fromarray(gen).save(str(out_dir / f"{page_label}_generated.png"))
|
|
|
|
# Heatmap: identical areas dimmed, differences highlighted in red
|
|
diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
|
|
max_diff_per_pixel = diff.max(axis=2)
|
|
severity = max_diff_per_pixel / 255.0
|
|
|
|
base = (ref * 0.5).astype(np.uint8)
|
|
overlay = np.zeros_like(base)
|
|
overlay[:, :, 0] = (severity * 255).astype(np.uint8)
|
|
alpha = severity[:, :, np.newaxis]
|
|
blended = (base * (1.0 - alpha) + overlay * alpha).astype(np.uint8)
|
|
blended[max_diff_per_pixel > 30] = [255, 0, 0]
|
|
|
|
Image.fromarray(blended).save(str(out_dir / f"{page_label}_diff.png"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main regression test
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("fixture_dir,excel_file,month,ref_dir", _discover_fixture_params())
|
|
def test_pdf_matches_reference(
|
|
fixture_dir: Path,
|
|
excel_file: Path,
|
|
month: str,
|
|
ref_dir: Path,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Generate PDFs for one fixture month and compare pixel-by-pixel against reference PDFs."""
|
|
|
|
project_root = Path(__file__).parent.parent
|
|
|
|
# ------------------------------------------------------------------
|
|
# 1. Generate PDFs via the CLI into a temporary directory
|
|
# ------------------------------------------------------------------
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable, "generate.py", "generate",
|
|
"--file", str(excel_file),
|
|
"--month", month,
|
|
"--out", str(tmp_path),
|
|
],
|
|
input="j\n",
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=str(project_root),
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
pytest.fail(
|
|
f"generate.py schlug fehl für Fixture '{fixture_dir.name}':\n"
|
|
f"STDOUT:\n{result.stdout}\n"
|
|
f"STDERR:\n{result.stderr}"
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# 2. Compare each reference PDF against the generated counterpart
|
|
# ------------------------------------------------------------------
|
|
diff_dir = fixture_dir / "diff"
|
|
reference_pdfs = sorted(ref_dir.glob("*.pdf"))
|
|
failures: list[str] = []
|
|
|
|
for ref_pdf in reference_pdfs:
|
|
# Skip PDFs with only a sequence number and no name
|
|
if re.fullmatch(r'\(\d+\)', ref_pdf.stem.strip()):
|
|
continue
|
|
|
|
# Strip manual markers (e.g. " #") from stem to find the generated file
|
|
clean_stem = re.sub(r'\s*#\s*$', '', ref_pdf.stem).rstrip()
|
|
generated_pdf = tmp_path / (clean_stem + ref_pdf.suffix)
|
|
|
|
if not generated_pdf.exists():
|
|
generated_pdf = tmp_path / ref_pdf.name
|
|
|
|
if not generated_pdf.exists():
|
|
failures.append(f" FEHLT: '{ref_pdf.name}' wurde nicht generiert.")
|
|
continue
|
|
|
|
# Render both PDFs to images
|
|
ref_images = _render_pdf_pages(ref_pdf)
|
|
gen_images = _render_pdf_pages(generated_pdf)
|
|
page_count = max(len(ref_images), len(gen_images))
|
|
|
|
for page_idx in range(page_count):
|
|
page_label = f"'{ref_pdf.name}' Seite {page_idx + 1}"
|
|
|
|
if page_idx >= len(ref_images):
|
|
failures.append(f" {page_label}: Seite nur in generierter PDF vorhanden.")
|
|
continue
|
|
if page_idx >= len(gen_images):
|
|
failures.append(f" {page_label}: Seite fehlt in generierter PDF.")
|
|
continue
|
|
|
|
similarity = _compute_similarity(ref_images[page_idx], gen_images[page_idx])
|
|
|
|
if similarity < SIMILARITY_THRESHOLD:
|
|
page_tag = f"seite_{page_idx + 1:02d}"
|
|
img_dir = diff_dir / clean_stem / page_tag
|
|
_save_diff_images(
|
|
ref_images[page_idx], gen_images[page_idx], img_dir, page_tag
|
|
)
|
|
rel = img_dir.relative_to(project_root)
|
|
failures.append(
|
|
f" {page_label}: Ähnlichkeit {similarity:.1%} "
|
|
f"(Schwelle {SIMILARITY_THRESHOLD:.0%}) → Bilder: {rel}/"
|
|
)
|
|
|
|
if failures:
|
|
failure_list = "\n".join(failures)
|
|
pytest.fail(
|
|
f"\nPixel-Diff-Fehler in Fixture '{fixture_dir.name}':\n{failure_list}"
|
|
) |