Ergebnisse des Crawlers nach dem er wieder ein paar Tage lang lief

05_System/feed_filter.py

@@ -57,6 +57,12 @@ FEEDS = [
         "kind": "arbeitsagentur",
         "url": "https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v6/jobs",
     },
+    {
+        "name": "oeko-stellenmarkt",
+        "kind": "atom",
+        "url": "https://oeko-stellenmarkt.de/?format=feed&type=atom",
+        "region_filter": True,
+    },
 ]
 
 ATOM_NS = {"a": "http://www.w3.org/2005/Atom"}
@@ -99,7 +105,7 @@ def fetch(url: str, headers: dict | None = None) -> bytes:
         return resp.read()
 
 
-def parse_atom(data: bytes) -> list:
+def parse_atom(data: bytes, region_filter: bool = False) -> list:
     root = ET.fromstring(data)
     items = []
     for e in root.findall("a:entry", ATOM_NS):
@@ -107,6 +113,12 @@ def parse_atom(data: bytes) -> list:
         href = link_el.attrib.get("href", "") if link_el is not None else ""
         title = (e.findtext("a:title", default="", namespaces=ATOM_NS) or "").strip()
         updated = (e.findtext("a:updated", default="", namespaces=ATOM_NS) or "").strip()
+        summary = (e.findtext("a:summary", default="", namespaces=ATOM_NS) or "")
+        if region_filter:
+            text = f"{title} {summary}"
+            text = re.sub(r"<[^>]+>", " ", text)  # HTML aus dem Summary entfernen
+            if not REGION.search(text):
+                continue
         items.append({"title": title, "url": href, "updated": updated})
     return items
 
@@ -228,7 +240,7 @@ def main() -> None:
             else:
                 data = fetch(fd["url"])
                 if fd["kind"] == "atom":
-                    items = parse_atom(data)
+                    items = parse_atom(data, fd.get("region_filter", False))
                 else:
                     items = parse_personio(data, fd.get("company", ""))
                 res["status"] = "ok"