diff --git a/.github/copilot-tasks.md b/.github/copilot-tasks.md index 468fd6b..a47c1f9 100644 --- a/.github/copilot-tasks.md +++ b/.github/copilot-tasks.md @@ -88,8 +88,10 @@ Projekt vollstaendig umsetzen: aus PS_PW.xlsx und Stellenbeschreibungen.xlsx mon ## Phase 8 - Tests und Abnahme - [ ] T15: Smoke-Test fuer End-to-End-Lauf - - [ ] Beispielmonat ausfuehren + - [x] Beispielmonat ausfuehren - [ ] Dateien in `output/` entstehen erwartungsgemaess + - [ ] Blocker: Masterdata-Matches fehlen fuer mehrere aktive Namen. + - [ ] Blocker: Template-Regel verletzt (aktuell 5 Sheets statt 1) und template_targets sind noch nicht konfiguriert. - [ ] T16: Fachtests definieren und ausfuehren - [ ] Mehrfachzuordnung eines Mitarbeiters - [ ] Fehlender Match diff --git a/src/taetigkeitsnachweis/matcher.py b/src/taetigkeitsnachweis/matcher.py index 39253fa..9a3d82a 100644 --- a/src/taetigkeitsnachweis/matcher.py +++ b/src/taetigkeitsnachweis/matcher.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import unicodedata from typing import Any from .errors import ValidationError @@ -21,7 +22,30 @@ def calculate_foerderquote_percent(allocation: AllocationRecord, precision: int def _normalize_name(name: str) -> str: compact = re.sub(r"\s+", " ", name.strip()) - return compact.casefold() + no_accents = "".join( + ch for ch in unicodedata.normalize("NFKD", compact) if not unicodedata.combining(ch) + ) + return no_accents.casefold() + + +def _name_keys(name: str) -> set[str]: + normalized = _normalize_name(name) + keys = {normalized} + + if "," in normalized: + last_name, first_name = [part.strip() for part in normalized.split(",", 1)] + if first_name and last_name: + keys.add(f"{first_name} {last_name}") + keys.add(f"{last_name}, {first_name}") + else: + tokens = normalized.split() + if len(tokens) >= 2: + first_name = tokens[0] + last_name = tokens[-1] + keys.add(f"{first_name} {last_name}") + keys.add(f"{last_name}, {first_name}") + + return {re.sub(r"\s+", " ", key).strip() for key in keys if key.strip()} def _with_tolerance( @@ -43,14 +67,19 @@ def match_assignments( master_by_name: dict[str, list[MasterDataRecord]] = {} for record in masterdata_records: - master_by_name.setdefault(_normalize_name(record.name), []).append(record) + for key in _name_keys(record.name): + master_by_name.setdefault(key, []).append(record) matched: list[MatchedAssignment] = [] issues: list[ValidationIssue] = [] for allocation in allocation_records: - normalized_name = _normalize_name(allocation.employee_name) - candidates = master_by_name.get(normalized_name, []) + candidate_map: dict[str, MasterDataRecord] = {} + for key in _name_keys(allocation.employee_name): + for candidate in master_by_name.get(key, []): + candidate_map[candidate.record_id] = candidate + + candidates = list(candidate_map.values()) if not candidates: issues.append( ValidationIssue( diff --git a/src/taetigkeitsnachweis/parser.py b/src/taetigkeitsnachweis/parser.py index 45cc403..5d81a5f 100644 --- a/src/taetigkeitsnachweis/parser.py +++ b/src/taetigkeitsnachweis/parser.py @@ -1,4 +1,4 @@ -"""Parsing and normalization for source workbooks.""" +"""Parsing and normalization for source workbooks.""" from __future__ import annotations @@ -14,262 +14,270 @@ from .models import AllocationRecord, MasterDataRecord def _to_text(value: Any) -> str: - if value is None: - return "" - return str(value).strip() + if value is None: + return "" + return str(value).strip() def _to_float(value: Any, field_name: str) -> float: - if value is None or str(value).strip() == "": - raise ValidationError(f"Missing numeric value for {field_name}.") + if value is None or str(value).strip() == "": + raise ValidationError(f"Missing numeric value for {field_name}.") - text = str(value).strip().replace(",", ".") - try: - return float(text) - except ValueError as exc: - raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc + text = str(value).strip().replace(",", ".") + try: + return float(text) + except ValueError as exc: + raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc + + +def _to_float_or_none(value: Any) -> float | None: + if value is None or str(value).strip() == "": + return None + + text = str(value).strip().replace(",", ".") + try: + return float(text) + except ValueError: + return None def _parse_german_decimal( - value: Any, - field_name: str, - decimal_separator: str, - thousands_separator: str, + value: Any, + field_name: str, + decimal_separator: str, + thousands_separator: str, ) -> float: - if value is None or str(value).strip() == "": - raise ValidationError(f"Missing numeric value for {field_name}.") + if value is None or str(value).strip() == "": + raise ValidationError(f"Missing numeric value for {field_name}.") - text = str(value).strip() - if thousands_separator: - text = text.replace(thousands_separator, "") - if decimal_separator: - text = text.replace(decimal_separator, ".") + text = str(value).strip() + if thousands_separator: + text = text.replace(thousands_separator, "") + if decimal_separator: + text = text.replace(decimal_separator, ".") - try: - return float(text) - except ValueError as exc: - raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc + try: + return float(text) + except ValueError as exc: + raise ValidationError(f"Invalid numeric value for {field_name}: {value}") from exc def _parse_month_marker(value: Any, prefix: str, date_format: str) -> datetime | None: - marker = _to_text(value) - if not marker: - return None + marker = _to_text(value) + if not marker: + return None - if not marker.lower().startswith(prefix.lower()): - return None + if not marker.lower().startswith(prefix.lower()): + return None - date_text = marker[len(prefix) :].strip() - return datetime.strptime(date_text, date_format) + date_text = marker[len(prefix) :].strip() + return datetime.strptime(date_text, date_format) def _resolve_month_column_index( - worksheet: Any, - header_row: int, - first_month_col: str, - target_month: int, - target_year: int, - month_prefix: str, - month_date_format: str, + worksheet: Any, + header_row: int, + first_month_col: str, + target_month: int, + target_year: int, + month_prefix: str, + month_date_format: str, ) -> int: - start_index = column_index_from_string(first_month_col) - matched_columns: list[int] = [] + start_index = column_index_from_string(first_month_col) + matched_columns: list[int] = [] - for column_index in range(start_index, worksheet.max_column + 1): - value = worksheet.cell(row=header_row, column=column_index).value - parsed = _parse_month_marker(value, month_prefix, month_date_format) - if parsed is None: - continue - if parsed.month == target_month and parsed.year == target_year: - matched_columns.append(column_index) + for column_index in range(start_index, worksheet.max_column + 1): + value = worksheet.cell(row=header_row, column=column_index).value + parsed = _parse_month_marker(value, month_prefix, month_date_format) + if parsed is None: + continue + if parsed.month == target_month and parsed.year == target_year: + matched_columns.append(column_index) - if len(matched_columns) == 0: - raise ValidationError( - f"No month column found for {target_month:02d}/{target_year} in allocation workbook." - ) + if len(matched_columns) == 0: + raise ValidationError( + f"No month column found for {target_month:02d}/{target_year} in allocation workbook." + ) - if len(matched_columns) > 1: - raise ValidationError( - f"Multiple month columns found for {target_month:02d}/{target_year} in allocation workbook." - ) + if len(matched_columns) > 1: + raise ValidationError( + f"Multiple month columns found for {target_month:02d}/{target_year} in allocation workbook." + ) - return matched_columns[0] + return matched_columns[0] def _is_inactive_value(value: Any, inactive_values: list[Any]) -> bool: - normalized = _to_text(value) - if normalized in {"", "-"}: - return True + normalized = _to_text(value) + if normalized in {"", "-"}: + return True - for inactive in inactive_values: - if str(inactive).strip() == normalized: - return True + for inactive in inactive_values: + if str(inactive).strip() == normalized: + return True - return False + return False def parse_allocation_records( - config: dict[str, Any], - report_month: int, - report_year: int, - project_root: Path, + config: dict[str, Any], + report_month: int, + report_year: int, + project_root: Path, ) -> list[AllocationRecord]: - allocation_cfg = config["allocation"] - columns_cfg = allocation_cfg["columns"] - files_cfg = config["files"] - sheets_cfg = config["sheets"] - paths_cfg = config["paths"] + allocation_cfg = config["allocation"] + columns_cfg = allocation_cfg["columns"] + files_cfg = config["files"] + sheets_cfg = config["sheets"] + paths_cfg = config["paths"] - workbook_path = ( - project_root - / paths_cfg["input_dir"] - / files_cfg["allocation_workbook"] - ) + workbook_path = ( + project_root + / paths_cfg["input_dir"] + / files_cfg["allocation_workbook"] + ) - if not workbook_path.exists(): - raise ValidationError(f"Allocation workbook not found: {workbook_path}") + if not workbook_path.exists(): + raise ValidationError(f"Allocation workbook not found: {workbook_path}") - workbook = load_workbook(workbook_path, data_only=True) - sheet_name = sheets_cfg["allocation"] - if sheet_name not in workbook.sheetnames: - raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}") - worksheet = workbook[sheet_name] + workbook = load_workbook(workbook_path, data_only=True) + sheet_name = sheets_cfg["allocation"] + if sheet_name not in workbook.sheetnames: + raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}") + worksheet = workbook[sheet_name] - header_row = allocation_cfg["header_month_marker_row"] - month_marker_cfg = allocation_cfg["month_marker"] - month_column = _resolve_month_column_index( - worksheet=worksheet, - header_row=header_row, - first_month_col=columns_cfg["first_month_column"], - target_month=report_month, - target_year=report_year, - month_prefix=month_marker_cfg["prefix"], - month_date_format=month_marker_cfg["date_format"], - ) + header_row = allocation_cfg["header_month_marker_row"] + month_marker_cfg = allocation_cfg["month_marker"] + month_column = _resolve_month_column_index( + worksheet=worksheet, + header_row=header_row, + first_month_col=columns_cfg["first_month_column"], + target_month=report_month, + target_year=report_year, + month_prefix=month_marker_cfg["prefix"], + month_date_format=month_marker_cfg["date_format"], + ) - employee_col = column_index_from_string(columns_cfg["employee_name"]) - contract_col = column_index_from_string(columns_cfg["contract_hours_weekly"]) - project_col = column_index_from_string(columns_cfg["project_key"]) - remarks_col = column_index_from_string(columns_cfg["remarks"]) - inactive_values = allocation_cfg.get("inactive_values", ["", "-", "0", 0]) + employee_col = column_index_from_string(columns_cfg["employee_name"]) + contract_col = column_index_from_string(columns_cfg["contract_hours_weekly"]) + project_col = column_index_from_string(columns_cfg["project_key"]) + remarks_col = column_index_from_string(columns_cfg["remarks"]) + inactive_values = allocation_cfg.get("inactive_values", ["", "-", "0", 0]) - current_employee = "" - current_contract_hours: float | None = None - normalized_records: list[AllocationRecord] = [] + current_employee = "" + current_contract_hours: float | None = None + normalized_records: list[AllocationRecord] = [] - for row_index in range(header_row + 1, worksheet.max_row + 1): - row_employee = _to_text(worksheet.cell(row=row_index, column=employee_col).value) - row_contract = worksheet.cell(row=row_index, column=contract_col).value - row_project = _to_text(worksheet.cell(row=row_index, column=project_col).value) + for row_index in range(header_row + 1, worksheet.max_row + 1): + row_employee = _to_text(worksheet.cell(row=row_index, column=employee_col).value) + row_contract = worksheet.cell(row=row_index, column=contract_col).value + row_project = _to_text(worksheet.cell(row=row_index, column=project_col).value) - if row_employee: - current_employee = row_employee - current_contract_hours = _to_float( - row_contract, - f"contract_hours_weekly row {row_index}", - ) + if row_employee: + current_employee = row_employee + current_contract_hours = _to_float_or_none(row_contract) - if not row_project: - continue + if not row_project: + continue - if not current_employee or current_contract_hours is None: - raise ValidationError( - f"Project row {row_index} has no inherited employee context." - ) + if not current_employee or current_contract_hours is None: + raise ValidationError( + f"Project row {row_index} has no inherited employee context." + ) - month_value = worksheet.cell(row=row_index, column=month_column).value - if _is_inactive_value(month_value, inactive_values): - continue + month_value = worksheet.cell(row=row_index, column=month_column).value + if _is_inactive_value(month_value, inactive_values): + continue - funded_hours = _to_float(month_value, f"funded_hours row {row_index}") - if funded_hours <= 0: - continue + funded_hours = _to_float(month_value, f"funded_hours row {row_index}") + if funded_hours <= 0: + continue - if current_contract_hours <= 0: - raise ValidationError( - f"Contract hours must be > 0 for employee '{current_employee}'" - ) + if current_contract_hours <= 0: + raise ValidationError( + f"Contract hours must be > 0 for employee '{current_employee}'" + ) - notes = _to_text(worksheet.cell(row=row_index, column=remarks_col).value) - normalized_records.append( - AllocationRecord( - employee_name=current_employee, - contract_hours_weekly=current_contract_hours, - project_key=row_project, - funded_hours_weekly=funded_hours, - notes=notes, - ) - ) + notes = _to_text(worksheet.cell(row=row_index, column=remarks_col).value) + normalized_records.append( + AllocationRecord( + employee_name=current_employee, + contract_hours_weekly=current_contract_hours, + project_key=row_project, + funded_hours_weekly=funded_hours, + notes=notes, + ) + ) - workbook.close() - return normalized_records + workbook.close() + return normalized_records def parse_masterdata_records( - config: dict[str, Any], - project_root: Path, + config: dict[str, Any], + project_root: Path, ) -> list[MasterDataRecord]: - files_cfg = config["files"] - sheets_cfg = config["sheets"] - paths_cfg = config["paths"] - master_cfg = config["masterdata"] - columns_cfg = master_cfg["columns"] - decimal_cfg = master_cfg["decimal"] + files_cfg = config["files"] + sheets_cfg = config["sheets"] + paths_cfg = config["paths"] + master_cfg = config["masterdata"] + columns_cfg = master_cfg["columns"] + decimal_cfg = master_cfg["decimal"] - workbook_path = ( - project_root - / paths_cfg["input_dir"] - / files_cfg["masterdata_workbook"] - ) + workbook_path = ( + project_root + / paths_cfg["input_dir"] + / files_cfg["masterdata_workbook"] + ) - if not workbook_path.exists(): - raise ValidationError(f"Master data workbook not found: {workbook_path}") + if not workbook_path.exists(): + raise ValidationError(f"Master data workbook not found: {workbook_path}") - workbook = load_workbook(workbook_path, data_only=True) - sheet_name = sheets_cfg["masterdata"] - if sheet_name not in workbook.sheetnames: - raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}") - worksheet = workbook[sheet_name] + workbook = load_workbook(workbook_path, data_only=True) + sheet_name = sheets_cfg["masterdata"] + if sheet_name not in workbook.sheetnames: + raise ValidationError(f"Sheet '{sheet_name}' not found in {workbook_path.name}") + worksheet = workbook[sheet_name] - id_col = column_index_from_string(columns_cfg["id"]) - antragsnummer_col = column_index_from_string(columns_cfg["antragsnummer"]) - name_col = column_index_from_string(columns_cfg["name"]) - qualifikation_col = column_index_from_string(columns_cfg["qualifikation"]) - stellenanteil_col = column_index_from_string(columns_cfg["stellenanteil"]) - funktion_col = column_index_from_string(columns_cfg["funktion"]) - taetigkeit_col = column_index_from_string(columns_cfg["taetigkeit"]) + id_col = column_index_from_string(columns_cfg["id"]) + antragsnummer_col = column_index_from_string(columns_cfg["antragsnummer"]) + name_col = column_index_from_string(columns_cfg["name"]) + qualifikation_col = column_index_from_string(columns_cfg["qualifikation"]) + stellenanteil_col = column_index_from_string(columns_cfg["stellenanteil"]) + funktion_col = column_index_from_string(columns_cfg["funktion"]) + taetigkeit_col = column_index_from_string(columns_cfg["taetigkeit"]) - records: list[MasterDataRecord] = [] + records: list[MasterDataRecord] = [] - for row_index in range(2, worksheet.max_row + 1): - name = _to_text(worksheet.cell(row=row_index, column=name_col).value) - if not name: - continue + for row_index in range(2, worksheet.max_row + 1): + name = _to_text(worksheet.cell(row=row_index, column=name_col).value) + if not name: + continue - stellenanteil = _parse_german_decimal( - worksheet.cell(row=row_index, column=stellenanteil_col).value, - field_name=f"stellenanteil row {row_index}", - decimal_separator=decimal_cfg.get("decimal_separator", ","), - thousands_separator=decimal_cfg.get("thousands_separator", "."), - ) + stellenanteil = _parse_german_decimal( + worksheet.cell(row=row_index, column=stellenanteil_col).value, + field_name=f"stellenanteil row {row_index}", + decimal_separator=decimal_cfg.get("decimal_separator", ","), + thousands_separator=decimal_cfg.get("thousands_separator", "."), + ) - records.append( - MasterDataRecord( - record_id=_to_text(worksheet.cell(row=row_index, column=id_col).value), - antragsnummer=_to_text( - worksheet.cell(row=row_index, column=antragsnummer_col).value - ), - name=name, - qualifikation=_to_text( - worksheet.cell(row=row_index, column=qualifikation_col).value - ), - stellenanteil_percent=stellenanteil, - funktion=_to_text(worksheet.cell(row=row_index, column=funktion_col).value), - taetigkeit=_to_text( - worksheet.cell(row=row_index, column=taetigkeit_col).value - ), - ) - ) + records.append( + MasterDataRecord( + record_id=_to_text(worksheet.cell(row=row_index, column=id_col).value), + antragsnummer=_to_text( + worksheet.cell(row=row_index, column=antragsnummer_col).value + ), + name=name, + qualifikation=_to_text( + worksheet.cell(row=row_index, column=qualifikation_col).value + ), + stellenanteil_percent=stellenanteil, + funktion=_to_text(worksheet.cell(row=row_index, column=funktion_col).value), + taetigkeit=_to_text( + worksheet.cell(row=row_index, column=taetigkeit_col).value + ), + ) + ) - workbook.close() - return records + workbook.close() + return records diff --git a/src/taetigkeitsnachweis/validator.py b/src/taetigkeitsnachweis/validator.py index 396ef4c..75cb6d8 100644 --- a/src/taetigkeitsnachweis/validator.py +++ b/src/taetigkeitsnachweis/validator.py @@ -194,22 +194,31 @@ def validate_sources( if len(month_matches) == 1: month_column = month_matches[0] + employee_col = column_index_from_string(allocation_cols["employee_name"]) contract_col = column_index_from_string(allocation_cols["contract_hours_weekly"]) project_col = column_index_from_string(allocation_cols["project_key"]) + current_contract_value: float | None = None + for row_idx in range(header_row + 1, allocation_ws.max_row + 1): + employee_value = _to_text(allocation_ws.cell(row=row_idx, column=employee_col).value) + contract_raw = allocation_ws.cell(row=row_idx, column=contract_col).value + parsed_contract = _to_float_or_none(contract_raw) + + if employee_value: + current_contract_value = parsed_contract + project_key = _to_text(allocation_ws.cell(row=row_idx, column=project_col).value) if not project_key: continue - contract_value = _to_float_or_none( - allocation_ws.cell(row=row_idx, column=contract_col).value - ) - if contract_value is None or contract_value <= 0: + if current_contract_value is None or current_contract_value <= 0: issues.append( ValidationIssue( code="INVALID_CONTRACT_HOURS", - message=f"Invalid contract weekly hours at row {row_idx}.", + message=( + f"Invalid inherited contract weekly hours at row {row_idx}." + ), severity="error", ) )