MediaWiki-Scraper/scrape.py

import os
import json
import requests
from bs4 import BeautifulSoup
import html2text
import urllib3
import warnings

api_url = ""
images_folder = ""

print("SSL waring ist okay, kann ignoriert werden.")

def load_config(config_file):
    with open(config_file, 'r') as file:
        return json.load(file)

def login(session, api_url, username, password):
    params = {
        "action": "query",
        "meta": "tokens",
        "type": "login",
        "format": "json"
    }
    response = session.get(api_url, params=params)
    login_token = response.json()['query']['tokens']['logintoken']

    login_payload = {
        "action": "login",
        "lgname": username,
        "lgpassword": password,
        "lgtoken": login_token,
        "format": "json"
    }
    response = session.post(api_url, data=login_payload)
    if response.status_code == 200:
        print("Login successful")
    else:
        print("Login failed")
        response.raise_for_status()

def download_all_pages(session, api_url):
    all_pages = []
    params = {
        'action': 'query',
        'format': 'json',
        'list': 'allpages',
        'aplimit' : 'max'
    }
    while True:
        response = session.get(api_url, params=params)
        if response.status_code == 200:
            data = response.json()
            all_pages.extend(data['query']['allpages'])
            if 'continue' in data:
                params['apcontinue'] = data['continue']['apcontinue']
            else:
                break
        else:
            print(f"Failed to download pages from {api_url}")
            break
    return all_pages
    
def save_content_to_file(content, download_folder, page_name, extension='md'):
    file_path = os.path.join(download_folder, f"{page_name}.{extension}")
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w', encoding='utf-8') as file:
        file.write(content)
    print(f"Downloaded {page_name} to {file_path}")

def GenerateMetadataJson(data):
    metadata = {
        'metadata' : {
            'categories' : data['parse']['categories'],
            'internallinks' : data['parse']['links'],
            'externallinks' : data['parse']['externallinks'],
            'images' : data['parse']['images'],
            'showtoc' : data['parse']['showtoc']
        }
    }
    return "\n\n " + json.dumps(metadata, indent=4)

def StoreImages(session, images_list, json_string):
    for image in images_list:
        params = {
            'action': 'query',
            'format': 'json',
            'titles': f'File:{image}',
            'prop': 'imageinfo',
            'iiprop': 'url'
        }
        response = session.get(api_url, params=params)
        if response.status_code != 200:
            print(f"Failed to download image {image}")
            continue
        data = response.json()
        pages = data['query']['pages']
        page_id = list(pages.keys())[0]
        image_info = pages[page_id].get('imageinfo', [{}])[0]
        image_url = image_info.get('url')
        if not image_url:
            print(f"No image info found for {image}")
            continue
        image_name = image_url.split('/')[-1]
        response = session.get(image_url)
        if response.status_code != 200:
            print(f"Failed to download image {image_name}")
            continue
        image_path = os.path.join(images_folder, image_name)
        with open(image_path, 'wb') as file:
            file.write(response.content)
        print(f"Downloaded {image_name}")
        json_string = ReplaceImagePath(json_string, image_name, image_path)
    return json_string

def ReplaceImagePath(str, image_name, image_path):
    return str.replace(f'"{image_name}"', f'"./{image_path}"') 

def main():
    global api_url 
    global images_folder
    config = load_config('config.json')
    api_url = config['api_url'].rstrip('/')  # Entferne den abschließenden Schrägstrich, falls vorhanden
    pages_folder = config['pages']
    images_folder = config['images']
    username = config['username']
    password = config['password']
    maxfiles = config['maxfiles']
    
    if not os.path.exists(pages_folder):
        os.makedirs(pages_folder)
    if not os.path.exists(images_folder):
        os.makedirs(images_folder)
    
    with requests.Session() as session:
        login(session, api_url, username, password)
        all_pages = download_all_pages(session, api_url)
        print(f"Downloading {len(all_pages)} pages")
        for page in all_pages:
            if maxfiles == 0:
                break
            maxfiles -= 1
            page_title = page['title']
            params = {
                'formatversion': 2,
                'action': 'parse',
                'page': page_title,
                'format': 'json'
            }
            response = session.get(api_url, params=params)
            if response.status_code == 200:
                data = response.json()
                if 'parse' in data:
                    markdown_content = html2text.html2text(data['parse']['text'])
                    markdown_content += GenerateMetadataJson(data)
                    markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
                    if markdown_content:
                        save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
            else:
                print(f"Failed to parse page {page_title}")
        else:
            print(f"Failed to download page {page_title}")

if __name__ == "__main__":
    main()