Fix inherited contract handling and document smoke test blockers

This commit is contained in:
Rob committed 2026-04-21 15:48:52 +02:00
1 parent 0be2646952
commit 418d8336f5
4 files changed
+258 -210

No files matched your search

+3 -1
View File
@@ -88,8 +88,10 @@ Projekt vollstaendig umsetzen: aus PS_PW.xlsx und Stellenbeschreibungen.xlsx mon
## Phase 8 - Tests und Abnahme
- [ ] T15: Smoke-Test fuer End-to-End-Lauf
- [ ] Beispielmonat ausfuehren
- [x] Beispielmonat ausfuehren
- [ ] Dateien in `output/` entstehen erwartungsgemaess
- [ ] Blocker: Masterdata-Matches fehlen fuer mehrere aktive Namen.
- [ ] Blocker: Template-Regel verletzt (aktuell 5 Sheets statt 1) und template_targets sind noch nicht konfiguriert.
- [ ] T16: Fachtests definieren und ausfuehren
- [ ] Mehrfachzuordnung eines Mitarbeiters
- [ ] Fehlender Match
+33 -4
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import re
import unicodedata
from typing import Any
from .errors import ValidationError
@@ -21,7 +22,30 @@ def calculate_foerderquote_percent(allocation: AllocationRecord, precision: int
def _normalize_name(name: str) -> str:
compact = re.sub(r"\s+", " ", name.strip())
return compact.casefold()
no_accents = "".join(
ch for ch in unicodedata.normalize("NFKD", compact) if not unicodedata.combining(ch)
)
return no_accents.casefold()
def _name_keys(name: str) -> set[str]:
normalized = _normalize_name(name)
keys = {normalized}
if "," in normalized:
last_name, first_name = [part.strip() for part in normalized.split(",", 1)]
if first_name and last_name:
keys.add(f"{first_name} {last_name}")
keys.add(f"{last_name}, {first_name}")
else:
tokens = normalized.split()
if len(tokens) >= 2:
first_name = tokens[0]
last_name = tokens[-1]
keys.add(f"{first_name} {last_name}")
keys.add(f"{last_name}, {first_name}")
return {re.sub(r"\s+", " ", key).strip() for key in keys if key.strip()}
def _with_tolerance(
@@ -43,14 +67,19 @@ def match_assignments(
master_by_name: dict[str, list[MasterDataRecord]] = {}
for record in masterdata_records:
master_by_name.setdefault(_normalize_name(record.name), []).append(record)
for key in _name_keys(record.name):
master_by_name.setdefault(key, []).append(record)
matched: list[MatchedAssignment] = []
issues: list[ValidationIssue] = []
for allocation in allocation_records:
normalized_name = _normalize_name(allocation.employee_name)
candidates = master_by_name.get(normalized_name, [])
candidate_map: dict[str, MasterDataRecord] = {}
for key in _name_keys(allocation.employee_name):
for candidate in master_by_name.get(key, []):
candidate_map[candidate.record_id] = candidate
candidates = list(candidate_map.values())
if not candidates:
issues.append(
ValidationIssue(
+208 -200
View File
@@ -1,4 +1,4 @@
"""Parsing and normalization for source workbooks."""
"""Parsing and normalization for source workbooks."""
from __future__ import annotations
@@ -14,262 +14,270 @@ from .models import AllocationRecord, MasterDataRecord
def _to_text(value: Any) -> str:
if value is None:
return ""
return str(value).strip()
if value is None:
return ""
return str(value).strip()
def _to_float(value: Any, field_name: str) -> float:
if value is None or str(value).strip() == "":
raise ValidationError(f"Missing numeric value for {field_name}.")
if value is None or str(value).strip() == "":
raise ValidationError(f"Missing numeric value for {field_name}.")
text = str(value).strip().replace(",", ".")
try:
return float(text)
except ValueError as exc:
raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc
text = str(value).strip().replace(",", ".")
try:
return float(text)
except ValueError as exc:
raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc
def _to_float_or_none(value: Any) -> float | None:
if value is None or str(value).strip() == "":
return None
text = str(value).strip().replace(",", ".")
try:
return float(text)
except ValueError:
return None
def _parse_german_decimal(
value: Any,
field_name: str,
decimal_separator: str,
thousands_separator: str,
value: Any,
field_name: str,
decimal_separator: str,
thousands_separator: str,
) -> float:
if value is None or str(value).strip() == "":
raise ValidationError(f"Missing numeric value for {field_name}.")
if value is None or str(value).strip() == "":
raise ValidationError(f"Missing numeric value for {field_name}.")
text = str(value).strip()
if thousands_separator:
text = text.replace(thousands_separator, "")
if decimal_separator:
text = text.replace(decimal_separator, ".")
text = str(value).strip()
if thousands_separator:
text = text.replace(thousands_separator, "")
if decimal_separator:
text = text.replace(decimal_separator, ".")
try:
return float(text)
except ValueError as exc:
raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc
try:
return float(text)
except ValueError as exc:
raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc
def _parse_month_marker(value: Any, prefix: str, date_format: str) -> datetime | None:
marker = _to_text(value)
if not marker:
return None
marker = _to_text(value)
if not marker:
return None
if not marker.lower().startswith(prefix.lower()):
return None
if not marker.lower().startswith(prefix.lower()):
return None
date_text = marker[len(prefix) :].strip()
return datetime.strptime(date_text, date_format)
date_text = marker[len(prefix) :].strip()
return datetime.strptime(date_text, date_format)
def _resolve_month_column_index(
worksheet: Any,
header_row: int,
first_month_col: str,
target_month: int,
target_year: int,
month_prefix: str,
month_date_format: str,
worksheet: Any,
header_row: int,
first_month_col: str,
target_month: int,
target_year: int,
month_prefix: str,
month_date_format: str,
) -> int:
start_index = column_index_from_string(first_month_col)
matched_columns: list[int] = []
start_index = column_index_from_string(first_month_col)
matched_columns: list[int] = []
for column_index in range(start_index, worksheet.max_column + 1):
value = worksheet.cell(row=header_row, column=column_index).value
parsed = _parse_month_marker(value, month_prefix, month_date_format)
if parsed is None:
continue
if parsed.month == target_month and parsed.year == target_year:
matched_columns.append(column_index)
for column_index in range(start_index, worksheet.max_column + 1):
value = worksheet.cell(row=header_row, column=column_index).value
parsed = _parse_month_marker(value, month_prefix, month_date_format)
if parsed is None:
continue
if parsed.month == target_month and parsed.year == target_year:
matched_columns.append(column_index)
if len(matched_columns) == 0:
raise ValidationError(
f"No month column found for {target_month:02d}/{target_year} in allocation workbook."
)
if len(matched_columns) == 0:
raise ValidationError(
f"No month column found for {target_month:02d}/{target_year} in allocation workbook."
)
if len(matched_columns) > 1:
raise ValidationError(
f"Multiple month columns found for {target_month:02d}/{target_year} in allocation workbook."
)
if len(matched_columns) > 1:
raise ValidationError(
f"Multiple month columns found for {target_month:02d}/{target_year} in allocation workbook."
)
return matched_columns[0]
return matched_columns[0]
def _is_inactive_value(value: Any, inactive_values: list[Any]) -> bool:
normalized = _to_text(value)
if normalized in {"", "-"}:
return True
normalized = _to_text(value)
if normalized in {"", "-"}:
return True
for inactive in inactive_values:
if str(inactive).strip() == normalized:
return True
for inactive in inactive_values:
if str(inactive).strip() == normalized:
return True
return False
return False
def parse_allocation_records(
config: dict[str, Any],
report_month: int,
report_year: int,
project_root: Path,
config: dict[str, Any],
report_month: int,
report_year: int,
project_root: Path,
) -> list[AllocationRecord]:
allocation_cfg = config["allocation"]
columns_cfg = allocation_cfg["columns"]
files_cfg = config["files"]
sheets_cfg = config["sheets"]
paths_cfg = config["paths"]
allocation_cfg = config["allocation"]
columns_cfg = allocation_cfg["columns"]
files_cfg = config["files"]
sheets_cfg = config["sheets"]
paths_cfg = config["paths"]
workbook_path = (
project_root
/ paths_cfg["input_dir"]
/ files_cfg["allocation_workbook"]
)
workbook_path = (
project_root
/ paths_cfg["input_dir"]
/ files_cfg["allocation_workbook"]
)
if not workbook_path.exists():
raise ValidationError(f"Allocation workbook not found: {workbook_path}")
if not workbook_path.exists():
raise ValidationError(f"Allocation workbook not found: {workbook_path}")
workbook = load_workbook(workbook_path, data_only=True)
sheet_name = sheets_cfg["allocation"]
if sheet_name not in workbook.sheetnames:
raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}")
worksheet = workbook[sheet_name]
workbook = load_workbook(workbook_path, data_only=True)
sheet_name = sheets_cfg["allocation"]
if sheet_name not in workbook.sheetnames:
raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}")
worksheet = workbook[sheet_name]
header_row = allocation_cfg["header_month_marker_row"]
month_marker_cfg = allocation_cfg["month_marker"]
month_column = _resolve_month_column_index(
worksheet=worksheet,
header_row=header_row,
first_month_col=columns_cfg["first_month_column"],
target_month=report_month,
target_year=report_year,
month_prefix=month_marker_cfg["prefix"],
month_date_format=month_marker_cfg["date_format"],
)
header_row = allocation_cfg["header_month_marker_row"]
month_marker_cfg = allocation_cfg["month_marker"]
month_column = _resolve_month_column_index(
worksheet=worksheet,
header_row=header_row,
first_month_col=columns_cfg["first_month_column"],
target_month=report_month,
target_year=report_year,
month_prefix=month_marker_cfg["prefix"],
month_date_format=month_marker_cfg["date_format"],
)
employee_col = column_index_from_string(columns_cfg["employee_name"])
contract_col = column_index_from_string(columns_cfg["contract_hours_weekly"])
project_col = column_index_from_string(columns_cfg["project_key"])
remarks_col = column_index_from_string(columns_cfg["remarks"])
inactive_values = allocation_cfg.get("inactive_values", ["", "-", "0", 0])
employee_col = column_index_from_string(columns_cfg["employee_name"])
contract_col = column_index_from_string(columns_cfg["contract_hours_weekly"])
project_col = column_index_from_string(columns_cfg["project_key"])
remarks_col = column_index_from_string(columns_cfg["remarks"])
inactive_values = allocation_cfg.get("inactive_values", ["", "-", "0", 0])
current_employee = ""
current_contract_hours: float | None = None
normalized_records: list[AllocationRecord] = []
current_employee = ""
current_contract_hours: float | None = None
normalized_records: list[AllocationRecord] = []
for row_index in range(header_row + 1, worksheet.max_row + 1):
row_employee = _to_text(worksheet.cell(row=row_index, column=employee_col).value)
row_contract = worksheet.cell(row=row_index, column=contract_col).value
row_project = _to_text(worksheet.cell(row=row_index, column=project_col).value)
for row_index in range(header_row + 1, worksheet.max_row + 1):
row_employee = _to_text(worksheet.cell(row=row_index, column=employee_col).value)
row_contract = worksheet.cell(row=row_index, column=contract_col).value
row_project = _to_text(worksheet.cell(row=row_index, column=project_col).value)
if row_employee:
current_employee = row_employee
current_contract_hours = _to_float(
row_contract,
f"contract_hours_weekly row {row_index}",
)
if row_employee:
current_employee = row_employee
current_contract_hours = _to_float_or_none(row_contract)
if not row_project:
continue
if not row_project:
continue
if not current_employee or current_contract_hours is None:
raise ValidationError(
f"Project row {row_index} has no inherited employee context."
)
if not current_employee or current_contract_hours is None:
raise ValidationError(
f"Project row {row_index} has no inherited employee context."
)
month_value = worksheet.cell(row=row_index, column=month_column).value
if _is_inactive_value(month_value, inactive_values):
continue
month_value = worksheet.cell(row=row_index, column=month_column).value
if _is_inactive_value(month_value, inactive_values):
continue
funded_hours = _to_float(month_value, f"funded_hours row {row_index}")
if funded_hours <= 0:
continue
funded_hours = _to_float(month_value, f"funded_hours row {row_index}")
if funded_hours <= 0:
continue
if current_contract_hours <= 0:
raise ValidationError(
f"Contract hours must be > 0 for employee '{current_employee}'"
)
if current_contract_hours <= 0:
raise ValidationError(
f"Contract hours must be > 0 for employee '{current_employee}'"
)
notes = _to_text(worksheet.cell(row=row_index, column=remarks_col).value)
normalized_records.append(
AllocationRecord(
employee_name=current_employee,
contract_hours_weekly=current_contract_hours,
project_key=row_project,
funded_hours_weekly=funded_hours,
notes=notes,
)
)
notes = _to_text(worksheet.cell(row=row_index, column=remarks_col).value)
normalized_records.append(
AllocationRecord(
employee_name=current_employee,
contract_hours_weekly=current_contract_hours,
project_key=row_project,
funded_hours_weekly=funded_hours,
notes=notes,
)
)
workbook.close()
return normalized_records
workbook.close()
return normalized_records
def parse_masterdata_records(
config: dict[str, Any],
project_root: Path,
config: dict[str, Any],
project_root: Path,
) -> list[MasterDataRecord]:
files_cfg = config["files"]
sheets_cfg = config["sheets"]
paths_cfg = config["paths"]
master_cfg = config["masterdata"]
columns_cfg = master_cfg["columns"]
decimal_cfg = master_cfg["decimal"]
files_cfg = config["files"]
sheets_cfg = config["sheets"]
paths_cfg = config["paths"]
master_cfg = config["masterdata"]
columns_cfg = master_cfg["columns"]
decimal_cfg = master_cfg["decimal"]
workbook_path = (
project_root
/ paths_cfg["input_dir"]
/ files_cfg["masterdata_workbook"]
)
workbook_path = (
project_root
/ paths_cfg["input_dir"]
/ files_cfg["masterdata_workbook"]
)
if not workbook_path.exists():
raise ValidationError(f"Master data workbook not found: {workbook_path}")
if not workbook_path.exists():
raise ValidationError(f"Master data workbook not found: {workbook_path}")
workbook = load_workbook(workbook_path, data_only=True)
sheet_name = sheets_cfg["masterdata"]
if sheet_name not in workbook.sheetnames:
raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}")
worksheet = workbook[sheet_name]
workbook = load_workbook(workbook_path, data_only=True)
sheet_name = sheets_cfg["masterdata"]
if sheet_name not in workbook.sheetnames:
raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}")
worksheet = workbook[sheet_name]
id_col = column_index_from_string(columns_cfg["id"])
antragsnummer_col = column_index_from_string(columns_cfg["antragsnummer"])
name_col = column_index_from_string(columns_cfg["name"])
qualifikation_col = column_index_from_string(columns_cfg["qualifikation"])
stellenanteil_col = column_index_from_string(columns_cfg["stellenanteil"])
funktion_col = column_index_from_string(columns_cfg["funktion"])
taetigkeit_col = column_index_from_string(columns_cfg["taetigkeit"])
id_col = column_index_from_string(columns_cfg["id"])
antragsnummer_col = column_index_from_string(columns_cfg["antragsnummer"])
name_col = column_index_from_string(columns_cfg["name"])
qualifikation_col = column_index_from_string(columns_cfg["qualifikation"])
stellenanteil_col = column_index_from_string(columns_cfg["stellenanteil"])
funktion_col = column_index_from_string(columns_cfg["funktion"])
taetigkeit_col = column_index_from_string(columns_cfg["taetigkeit"])
records: list[MasterDataRecord] = []
records: list[MasterDataRecord] = []
for row_index in range(2, worksheet.max_row + 1):
name = _to_text(worksheet.cell(row=row_index, column=name_col).value)
if not name:
continue
for row_index in range(2, worksheet.max_row + 1):
name = _to_text(worksheet.cell(row=row_index, column=name_col).value)
if not name:
continue
stellenanteil = _parse_german_decimal(
worksheet.cell(row=row_index, column=stellenanteil_col).value,
field_name=f"stellenanteil row {row_index}",
decimal_separator=decimal_cfg.get("decimal_separator", ","),
thousands_separator=decimal_cfg.get("thousands_separator", "."),
)
stellenanteil = _parse_german_decimal(
worksheet.cell(row=row_index, column=stellenanteil_col).value,
field_name=f"stellenanteil row {row_index}",
decimal_separator=decimal_cfg.get("decimal_separator", ","),
thousands_separator=decimal_cfg.get("thousands_separator", "."),
)
records.append(
MasterDataRecord(
record_id=_to_text(worksheet.cell(row=row_index, column=id_col).value),
antragsnummer=_to_text(
worksheet.cell(row=row_index, column=antragsnummer_col).value
),
name=name,
qualifikation=_to_text(
worksheet.cell(row=row_index, column=qualifikation_col).value
),
stellenanteil_percent=stellenanteil,
funktion=_to_text(worksheet.cell(row=row_index, column=funktion_col).value),
taetigkeit=_to_text(
worksheet.cell(row=row_index, column=taetigkeit_col).value
),
)
)
records.append(
MasterDataRecord(
record_id=_to_text(worksheet.cell(row=row_index, column=id_col).value),
antragsnummer=_to_text(
worksheet.cell(row=row_index, column=antragsnummer_col).value
),
name=name,
qualifikation=_to_text(
worksheet.cell(row=row_index, column=qualifikation_col).value
),
stellenanteil_percent=stellenanteil,
funktion=_to_text(worksheet.cell(row=row_index, column=funktion_col).value),
taetigkeit=_to_text(
worksheet.cell(row=row_index, column=taetigkeit_col).value
),
)
)
workbook.close()
return records
workbook.close()
return records
+14 -5
View File
@@ -194,22 +194,31 @@ def validate_sources(
if len(month_matches) == 1:
month_column = month_matches[0]
employee_col = column_index_from_string(allocation_cols["employee_name"])
contract_col = column_index_from_string(allocation_cols["contract_hours_weekly"])
project_col = column_index_from_string(allocation_cols["project_key"])
current_contract_value: float | None = None
for row_idx in range(header_row + 1, allocation_ws.max_row + 1):
employee_value = _to_text(allocation_ws.cell(row=row_idx, column=employee_col).value)
contract_raw = allocation_ws.cell(row=row_idx, column=contract_col).value
parsed_contract = _to_float_or_none(contract_raw)
if employee_value:
current_contract_value = parsed_contract
project_key = _to_text(allocation_ws.cell(row=row_idx, column=project_col).value)
if not project_key:
continue
contract_value = _to_float_or_none(
allocation_ws.cell(row=row_idx, column=contract_col).value
)
if contract_value is None or contract_value <= 0:
if current_contract_value is None or current_contract_value <= 0:
issues.append(
ValidationIssue(
code="INVALID_CONTRACT_HOURS",
message=f"Invalid contract weekly hours at row {row_idx}.",
message=(
f"Invalid inherited contract weekly hours at row {row_idx}."
),
severity="error",
)
)