312 lines
11 KiB
Python
312 lines
11 KiB
Python
"""Regression tests: pixel-diff comparison of script-generated vs. manually created reference PDFs.
|
|
|
|
Fixture layout
|
|
--------------
|
|
tests/fixtures/
|
|
YYYY_MMM/ ← directory name encodes year and Excel sheet name
|
|
reference/
|
|
(1) Name.pdf ← manually created ground-truth PDFs
|
|
diff/ ← diff images written here on failure
|
|
|
|
A fixture is discovered automatically when:
|
|
- The directory name matches YYYY_MMM (e.g. 2026_Feb)
|
|
- At least one *.xlsx / *.xlsm file exists in the directory or in input/
|
|
- At least one *.pdf file exists in reference/
|
|
|
|
Rendering
|
|
---------
|
|
Both reference and generated PDFs are rendered to images using pymupdf.
|
|
The images are then compared pixel-by-pixel.
|
|
|
|
Similarity threshold
|
|
--------------------
|
|
SIMILARITY_THRESHOLD (100%) — fraction of pixels whose R, G, B channels
|
|
each differ by at most PIXEL_TOLERANCE (default 5) between reference and generated PDF.
|
|
|
|
Diff images
|
|
-----------
|
|
When a page fails the threshold, three images are saved to
|
|
tests/fixtures/YYYY_MMM/diff/<pdf-stem>/seite_NN/
|
|
- seite_NN_reference.png
|
|
- seite_NN_generated.png
|
|
- seite_NN_diff.png (heatmap: red = difference)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from pypdf import PdfReader, PdfWriter
|
|
from pypdf.generic import BooleanObject, NameObject
|
|
from PIL import Image
|
|
|
|
from src.pdf_render import render_pdf_pages
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
FIXTURES_DIR = Path(__file__).parent / "fixtures"
|
|
SIMILARITY_THRESHOLD = 1.00 # minimum fraction of matching pixels per page
|
|
PIXEL_TOLERANCE = 5 # max per-channel delta still counted as "same"
|
|
RENDER_DPI = 150 # rendering resolution for PDF → image
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture discovery (runs at collection time)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _discover_fixture_params() -> list:
|
|
"""Return pytest.param entries for every ready fixture directory.
|
|
|
|
Excel file lookup order:
|
|
1. Any *.xlsx / *.xlsm directly inside the fixture directory.
|
|
2. Any *.xlsx / *.xlsm inside the project-level ``input/`` directory
|
|
whose filename contains the fixture year (e.g. "2026").
|
|
"""
|
|
if not FIXTURES_DIR.exists():
|
|
return []
|
|
|
|
project_root = FIXTURES_DIR.parent.parent
|
|
input_dir = project_root / "input"
|
|
|
|
params = []
|
|
for d in sorted(FIXTURES_DIR.iterdir()):
|
|
if not d.is_dir():
|
|
continue
|
|
|
|
parts = d.name.split("_", 1)
|
|
if len(parts) != 2 or not parts[0].isdigit() or len(parts[0]) != 4:
|
|
continue
|
|
year_str, month = parts
|
|
|
|
# 1. Excel file in fixture directory
|
|
xlsx_files = list(d.glob("*.xlsx")) + list(d.glob("*.xlsm"))
|
|
|
|
# 2. Fallback: input/ directory
|
|
if not xlsx_files and input_dir.exists():
|
|
xlsx_files = [
|
|
f for f in (list(input_dir.glob("*.xlsx")) + list(input_dir.glob("*.xlsm")))
|
|
if year_str in f.stem
|
|
]
|
|
|
|
if not xlsx_files:
|
|
continue
|
|
|
|
ref_dir = d / "reference"
|
|
if not ref_dir.exists() or not list(ref_dir.glob("*.pdf")):
|
|
continue
|
|
|
|
params.append(
|
|
pytest.param(d, xlsx_files[0], month, ref_dir, id=d.name)
|
|
)
|
|
|
|
return params
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Comparison helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _resize_to_match(img: np.ndarray, target_shape: tuple) -> np.ndarray:
|
|
"""Resize *img* so its (h, w) matches *target_shape*."""
|
|
h, w = target_shape[:2]
|
|
pil = Image.fromarray(img).resize((w, h), Image.LANCZOS)
|
|
return np.array(pil)
|
|
|
|
|
|
def _set_need_appearances(pdf_path: Path, enabled: bool = True) -> None:
|
|
"""Set /AcroForm /NeedAppearances on *pdf_path* in-place."""
|
|
reader = PdfReader(str(pdf_path))
|
|
writer = PdfWriter()
|
|
writer.clone_document_from_reader(reader)
|
|
|
|
if "/AcroForm" in writer._root_object:
|
|
writer._root_object["/AcroForm"][NameObject("/NeedAppearances")] = BooleanObject(enabled)
|
|
|
|
with pdf_path.open("wb") as fh:
|
|
writer.write(fh)
|
|
|
|
|
|
def _compute_similarity(ref: np.ndarray, gen: np.ndarray, mask_footer: bool = False) -> tuple[float, int, int]:
|
|
"""Return (similarity, different_pixels, total_pixels).
|
|
|
|
If mask_footer=True, ignores the bottom ~20% of each image (footer area).
|
|
"""
|
|
if ref.shape != gen.shape:
|
|
gen = _resize_to_match(gen, ref.shape)
|
|
|
|
# Apply footer mask if requested
|
|
if mask_footer:
|
|
h = ref.shape[0]
|
|
mask_start = int(h * 0.80) # Ignore bottom 20%
|
|
ref = ref[:mask_start]
|
|
gen = gen[:mask_start]
|
|
|
|
diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
|
|
matching = np.all(diff <= PIXEL_TOLERANCE, axis=2)
|
|
similar_count = np.sum(matching)
|
|
total_count = matching.size
|
|
|
|
return (similar_count / total_count, total_count - similar_count, total_count)
|
|
|
|
|
|
def _save_diff_images(ref: np.ndarray, gen: np.ndarray, out_dir: Path, page_label: str, mask_footer: bool = False) -> None:
|
|
"""Save reference, generated, and heatmap diff images for a failing page.
|
|
|
|
If mask_footer=True, the bottom ~20% (footer area) is shown in gray in the diff image.
|
|
The diff is computed BEFORE masking to show actual differences, then the masked area is grayed out.
|
|
"""
|
|
if ref.shape != gen.shape:
|
|
gen = _resize_to_match(gen, ref.shape)
|
|
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Save full reference and generated images
|
|
Image.fromarray(ref).save(str(out_dir / f"{page_label}_reference.png"))
|
|
Image.fromarray(gen).save(str(out_dir / f"{page_label}_generated.png"))
|
|
|
|
# Heatmap: identical areas dimmed, differences highlighted in red
|
|
# NOTE: diff is computed on FULL images, not masked, to show all actual differences
|
|
diff = np.abs(ref.astype(np.int32) - gen.astype(np.int32))
|
|
max_diff_per_pixel = diff.max(axis=2)
|
|
severity = max_diff_per_pixel / 255.0
|
|
|
|
base = (ref * 0.5).astype(np.uint8)
|
|
overlay = np.zeros_like(base)
|
|
overlay[:, :, 0] = (severity * 255).astype(np.uint8)
|
|
alpha = severity[:, :, np.newaxis]
|
|
blended = (base * (1.0 - alpha) + overlay * alpha).astype(np.uint8)
|
|
blended[max_diff_per_pixel > 30] = [255, 0, 0]
|
|
|
|
# Gray out masked footer region in diff image
|
|
if mask_footer:
|
|
h = ref.shape[0]
|
|
mask_start = int(h * 0.99)
|
|
blended[mask_start:] = (180, 180, 180) # Gray footer region
|
|
|
|
Image.fromarray(blended).save(str(out_dir / f"{page_label}_diff.png"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main regression test
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("fixture_dir,excel_file,month,ref_dir", _discover_fixture_params())
|
|
def test_pdf_matches_reference(
|
|
fixture_dir: Path,
|
|
excel_file: Path,
|
|
month: str,
|
|
ref_dir: Path,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Generate PDFs for one fixture month and compare pixel-by-pixel against reference PDFs."""
|
|
|
|
project_root = Path(__file__).parent.parent
|
|
|
|
# Load config
|
|
config_path = project_root / "config.json"
|
|
with config_path.open() as f:
|
|
config = json.load(f)
|
|
regression_cfg = config.get("regression", {})
|
|
|
|
if not regression_cfg.get("enabled", False):
|
|
pytest.skip("Regression-Bildvergleich deaktiviert (regression.enabled=false in config.json)")
|
|
|
|
mask_footer = regression_cfg.get("mask_footer", True)
|
|
|
|
# ------------------------------------------------------------------
|
|
# 1. Generate PDFs via the CLI into a temporary directory
|
|
# ------------------------------------------------------------------
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable, "generate.py", "generate",
|
|
"--file", str(excel_file),
|
|
"--month", month,
|
|
"--out", str(tmp_path),
|
|
],
|
|
input="j\n",
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=str(project_root),
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
pytest.fail(
|
|
f"generate.py schlug fehl für Fixture '{fixture_dir.name}':\n"
|
|
f"STDOUT:\n{result.stdout}\n"
|
|
f"STDERR:\n{result.stderr}"
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# 2. Compare each reference PDF against the generated counterpart
|
|
# ------------------------------------------------------------------
|
|
diff_dir = fixture_dir / "diff"
|
|
reference_pdfs = sorted(ref_dir.glob("*.pdf"))
|
|
failures: list[str] = []
|
|
|
|
for ref_pdf in reference_pdfs:
|
|
# Skip PDFs with only a sequence number and no name
|
|
if re.fullmatch(r'\(\d+\)', ref_pdf.stem.strip()):
|
|
continue
|
|
|
|
# Strip manual markers (e.g. " #") from stem to find the generated file
|
|
clean_stem = re.sub(r'\s*#\s*$', '', ref_pdf.stem).rstrip()
|
|
generated_pdf = tmp_path / (clean_stem + ref_pdf.suffix)
|
|
|
|
if not generated_pdf.exists():
|
|
generated_pdf = tmp_path / ref_pdf.name
|
|
|
|
if not generated_pdf.exists():
|
|
failures.append(f" FEHLT: '{ref_pdf.name}' wurde nicht generiert.")
|
|
continue
|
|
|
|
# Regression-specific render behavior: keep NeedAppearances disabled.
|
|
# With pymupdf, NeedAppearances=True can hide widget AP content
|
|
# (footer and bottom-right buttons) during rasterization.
|
|
_set_need_appearances(generated_pdf, enabled=False)
|
|
|
|
# Render both PDFs to images
|
|
ref_images = render_pdf_pages(ref_pdf, RENDER_DPI)
|
|
gen_images = render_pdf_pages(generated_pdf, RENDER_DPI)
|
|
page_count = max(len(ref_images), len(gen_images))
|
|
|
|
for page_idx in range(page_count):
|
|
page_label = f"'{ref_pdf.name}' Seite {page_idx + 1}"
|
|
|
|
if page_idx >= len(ref_images):
|
|
failures.append(f" {page_label}: Seite nur in generierter PDF vorhanden.")
|
|
continue
|
|
if page_idx >= len(gen_images):
|
|
failures.append(f" {page_label}: Seite fehlt in generierter PDF.")
|
|
continue
|
|
|
|
ref_img = ref_images[page_idx]
|
|
gen_img = gen_images[page_idx]
|
|
|
|
similarity, diff_pixels, total_pixels = _compute_similarity(ref_img, gen_img, mask_footer=mask_footer)
|
|
|
|
if similarity < SIMILARITY_THRESHOLD:
|
|
page_tag = f"seite_{page_idx + 1:02d}"
|
|
img_dir = diff_dir / clean_stem / page_tag
|
|
_save_diff_images(
|
|
ref_img, gen_img, img_dir, page_tag, mask_footer=mask_footer
|
|
)
|
|
rel = img_dir.relative_to(project_root)
|
|
failures.append(
|
|
f" {page_label}: Ähnlichkeit {similarity:.2%} "
|
|
f"({diff_pixels:,} / {total_pixels:,} Pixel unterschiedlich, "
|
|
f"Schwelle {SIMILARITY_THRESHOLD:.0%}) → Bilder: {rel}/"
|
|
)
|
|
|
|
if failures:
|
|
failure_list = "\n".join(failures)
|
|
pytest.fail(
|
|
f"\nPixel-Diff-Fehler in Fixture '{fixture_dir.name}':\n{failure_list}"
|
|
) |