From 0fb19123d71c44431137c5706412793b26122c0e Mon Sep 17 00:00:00 2001
From: trnq1ll0 <rob.tranquillo@gmx.de>
Date: Sun, 25 Aug 2024 13:24:38 +0200
Subject: [PATCH] version 0.9

---
 .gitignore       |   5 ++
 README.md        |  43 ++++++++++++
 config.json.tmpl |   8 +++
 requirements.txt |   8 +++
 scrape.py        | 165 +++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 229 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 README.md
 create mode 100644 config.json.tmpl
 create mode 100644 requirements.txt
 create mode 100644 scrape.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0cba234
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+.DS_Store
+.venv
+pages
+images
+config.json
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9cadbb0
--- /dev/null
+++ b/README.md
@@ -0,0 +1,43 @@
+# Ziel
+
+Ziel des Paketes ist es alle Seiten eines MediWikis so zu konvertieren das sie direkt in Obsidian verwendete werden können. 
+
+Das Paket lädt dafür alle Seiten eines MediaWikis herunter und speichert sie in einem Ordner als Markdown file.
+
+## Bestandteile
+
+* Wikiseiten (Lemma) werden mit ihrem Titel als Dateinamen abgelegt 
+* Seiten die eine scheinbare Struktur in Form von slashes (/) im Namen haben werden in entsprechenden Unterordnern abgelegt
+* Es wird die MediaWiki API verwendet um alle Seiten herunterzuladen. 
+* es wird html2text verwendet um die Seiten von HTML in Markdown zu konvertieren
+* Meta-informationen der Seite werden am Ende in einem json gespeichert
+* Dateien werden in einem separaten Ordner gespeichert un im Metadaten json richtig verlinkt
+
+### Metadaten JSON
+
+´´´
+{
+    "metadata": {
+        "categories": [],
+        "internallinks": [],
+        "externallinks": [],
+        "images": [],
+        "showtoc": false
+}
+´´´
+
+
+# Get started
+## Install python environment
+
+```
+python3 -m venv .venv
+
+source .venv/bin/activate  # Für Unix/Mac
+.venv\Scripts\activate     # Für Windows
+
+pip install -r requirements.txt
+```
+## Konfiguration
+
+Kopiere die datei `config.json.tmpl` zu `config.json` und passe die Einstellungen darin an.
diff --git a/config.json.tmpl b/config.json.tmpl
new file mode 100644
index 0000000..6a62975
--- /dev/null
+++ b/config.json.tmpl
@@ -0,0 +1,8 @@
+{
+    "api_url": "https://my-mediwiki/api.php",
+    "pages": "pages",
+    "images": "images",
+    "username": "bob",
+    "password": "password",
+    "maxfiles": -1
+}
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..385d089
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,8 @@
+beautifulsoup4==4.12.3
+certifi==2024.7.4
+charset-normalizer==3.3.2
+html2text==2024.2.26
+idna==3.7
+requests==2.32.3
+soupsieve==2.6
+urllib3==2.2.2
diff --git a/scrape.py b/scrape.py
new file mode 100644
index 0000000..9ac1be7
--- /dev/null
+++ b/scrape.py
@@ -0,0 +1,165 @@
+import os
+import json
+import requests
+from bs4 import BeautifulSoup
+import html2text
+import urllib3
+import warnings
+
+api_url = ""
+images_folder = ""
+
+print("SSL waring ist okay, kann ignoriert werden.")
+
+def load_config(config_file):
+    with open(config_file, 'r') as file:
+        return json.load(file)
+
+def login(session, api_url, username, password):
+    params = {
+        "action": "query",
+        "meta": "tokens",
+        "type": "login",
+        "format": "json"
+    }
+    response = session.get(api_url, params=params)
+    login_token = response.json()['query']['tokens']['logintoken']
+
+    login_payload = {
+        "action": "login",
+        "lgname": username,
+        "lgpassword": password,
+        "lgtoken": login_token,
+        "format": "json"
+    }
+    response = session.post(api_url, data=login_payload)
+    if response.status_code == 200:
+        print("Login successful")
+    else:
+        print("Login failed")
+        response.raise_for_status()
+
+def download_all_pages(session, api_url):
+    all_pages = []
+    params = {
+        'action': 'query',
+        'format': 'json',
+        'list': 'allpages',
+        'aplimit' : 'max'
+    }
+    while True:
+        response = session.get(api_url, params=params)
+        if response.status_code == 200:
+            data = response.json()
+            all_pages.extend(data['query']['allpages'])
+            if 'continue' in data:
+                params['apcontinue'] = data['continue']['apcontinue']
+            else:
+                break
+        else:
+            print(f"Failed to download pages from {api_url}")
+            break
+    return all_pages
+    
+def save_content_to_file(content, download_folder, page_name, extension='md'):
+    file_path = os.path.join(download_folder, f"{page_name}.{extension}")
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    with open(file_path, 'w', encoding='utf-8') as file:
+        file.write(content)
+    print(f"Downloaded {page_name} to {file_path}")
+
+def GenerateMetadataJson(data):
+    metadata = {
+        'metadata' : {
+            'categories' : data['parse']['categories'],
+            'internallinks' : data['parse']['links'],
+            'externallinks' : data['parse']['externallinks'],
+            'images' : data['parse']['images'],
+            'showtoc' : data['parse']['showtoc']
+        }
+    }
+    return "\n\n " + json.dumps(metadata, indent=4)
+
+def StoreImages(session, images_list, json_string):
+    for image in images_list:
+        params = {
+            'action': 'query',
+            'format': 'json',
+            'titles': f'File:{image}',
+            'prop': 'imageinfo',
+            'iiprop': 'url'
+        }
+        response = session.get(api_url, params=params)
+        if response.status_code != 200:
+            print(f"Failed to download image {image}")
+            continue
+        data = response.json()
+        pages = data['query']['pages']
+        page_id = list(pages.keys())[0]
+        image_info = pages[page_id].get('imageinfo', [{}])[0]
+        image_url = image_info.get('url')
+        if not image_url:
+            print(f"No image info found for {image}")
+            continue
+        image_name = image_url.split('/')[-1]
+        response = session.get(image_url)
+        if response.status_code != 200:
+            print(f"Failed to download image {image_name}")
+            continue
+        image_path = os.path.join(images_folder, image_name)
+        with open(image_path, 'wb') as file:
+            file.write(response.content)
+        print(f"Downloaded {image_name}")
+        json_string = ReplaceImagePath(json_string, image_name, image_path)
+    return json_string
+
+def ReplaceImagePath(str, image_name, image_path):
+    return str.replace(f'"{image_name}"', f'"./{image_path}"') 
+
+def main():
+    global api_url 
+    global images_folder
+    config = load_config('config.json')
+    api_url = config['api_url'].rstrip('/')  # Entferne den abschließenden Schrägstrich, falls vorhanden
+    pages_folder = config['pages']
+    images_folder = config['images']
+    username = config['username']
+    password = config['password']
+    maxfiles = config['maxfiles']
+    
+    if not os.path.exists(pages_folder):
+        os.makedirs(pages_folder)
+    if not os.path.exists(images_folder):
+        os.makedirs(images_folder)
+    
+    with requests.Session() as session:
+        login(session, api_url, username, password)
+        all_pages = download_all_pages(session, api_url)
+        print(f"Downloading {len(all_pages)} pages")
+        for page in all_pages:
+            if maxfiles == 0:
+                break
+            maxfiles -= 1
+            page_title = page['title']
+            params = {
+                'formatversion': 2,
+                'action': 'parse',
+                'page': page_title,
+                'format': 'json'
+            }
+            response = session.get(api_url, params=params)
+            if response.status_code == 200:
+                data = response.json()
+                if 'parse' in data:
+                    markdown_content = html2text.html2text(data['parse']['text'])
+                    markdown_content += GenerateMetadataJson(data)
+                    markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
+                    if markdown_content:
+                        save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
+            else:
+                print(f"Failed to parse page {page_title}")
+        else:
+            print(f"Failed to download page {page_title}")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file