import os import json import re import requests from bs4 import BeautifulSoup import html2text import urllib3 import warnings api_url = "" images_folder = "" print("SSL waring ist okay, kann ignoriert werden.") def load_config(config_file): with open(config_file, 'r') as file: return json.load(file) def login(session, api_url, username, password): params = { "action": "query", "meta": "tokens", "type": "login", "format": "json" } response = session.get(api_url, params=params) login_token = response.json()['query']['tokens']['logintoken'] login_payload = { "action": "login", "lgname": username, "lgpassword": password, "lgtoken": login_token, "format": "json" } response = session.post(api_url, data=login_payload) if response.status_code == 200: print("Login successful") else: print("Login failed") response.raise_for_status() def download_all_pages(session, api_url): all_pages = [] params = { 'action': 'query', 'format': 'json', 'list': 'allpages', 'aplimit' : 'max' } while True: response = session.get(api_url, params=params) if response.status_code == 200: data = response.json() all_pages.extend(data['query']['allpages']) if 'continue' in data: params['apcontinue'] = data['continue']['apcontinue'] else: break else: print(f"Failed to download pages from {api_url}") break return all_pages def save_content_to_file(content, download_folder, page_name, extension='md'): file_path = os.path.join(download_folder, f"{page_name}.{extension}") os.makedirs(os.path.dirname(file_path), exist_ok=True) with open(file_path, 'w', encoding='utf-8') as file: file.write(content) def GenerateMetadataJson(session, data): metadata = { 'metadata' : { 'categories' : data['parse']['categories'], 'internallinks' : data['parse']['links'], 'externallinks' : data['parse']['externallinks'], 'images' : data['parse']['images'], 'showtoc' : data['parse']['showtoc'], 'revisions' : ProcessRevisions(session, api_url, data['parse']['pageid']) } } return "\n\n " + json.dumps(metadata, indent=4) def StoreImages(session, images_list, json_string): for image in images_list: params = { 'action': 'query', 'format': 'json', 'titles': f'File:{image}', 'prop': 'imageinfo', 'iiprop': 'url' } response = session.get(api_url, params=params) if response.status_code != 200: print(f"Failed to download image {image}") continue data = response.json() pages = data['query']['pages'] page_id = list(pages.keys())[0] image_info = pages[page_id].get('imageinfo', [{}])[0] image_url = image_info.get('url') if not image_url: print(f"No image info found for {image}") continue image_name = image_url.split('/')[-1] response = session.get(image_url) if response.status_code != 200: print(f"Failed to download image {image_name}") continue image_path = os.path.join(images_folder, image_name) with open(image_path, 'wb') as file: file.write(response.content) print(f"Downloaded {image_name}") json_string = ReplaceImagePath(json_string, image_name, image_path) return json_string def ReplaceImagePath(str, image_name, image_path): return str.replace(f'"{image_name}"', f'"./{image_path}"') def HandleExternalLinks(externallinks, markdown_content): # for link in externallinks: # markdown_content = markdown_content.replace(f"<{link}>", f'[{link}]({link})') return markdown_content def HandleInternalLinks(markdown_content): markdown_content = markdown_content.replace('/index.php?title=', '') return markdown_content def RemoveEditLinks(markdown_content): pattern = r'\[\[Bearbeiten\]\(.*?\)\s\|\s\[Quelltext bearbeiten\]\(.*?\)\]' matches = re.findall(pattern, markdown_content) # double check if the match is correct, than remove it for match in matches: if "[Bearbeiten]" in match and "[Quelltext bearbeiten]" in match: markdown_content = markdown_content.replace(match, '') return markdown_content def ProcessRevisions(session, api_url, page_id): params = { 'action': 'query', 'format': 'json', 'prop': 'revisions', 'rvlimit': 500, 'pageids': page_id } response = session.get(api_url, params=params) if response.status_code == 200: data = response.json() first_child_key = next(iter(data['query']['pages'])) revisions = data['query']['pages'][first_child_key]['revisions'] revisons_block = [] # print(f"Found {len(revisions)} revision: {json.dumps(revisions[0])}") for revision in revisions: revisons_block.append({ 'timestamp' : revision['timestamp'], 'comment' : revision['comment'] }) return revisons_block else: print(f"Failed to download revisions for page_id {page_id}") def main(): global api_url global images_folder config = load_config('config.json') api_url = config['api_url'].rstrip('/') # Entferne den abschließenden Schrägstrich, falls vorhanden pages_folder = config['pages'] images_folder = config['images'] username = config['username'] password = config['password'] maxfiles = config['maxfiles'] if not os.path.exists(pages_folder): os.makedirs(pages_folder) if not os.path.exists(images_folder): os.makedirs(images_folder) with requests.Session() as session: login(session, api_url, username, password) all_pages = download_all_pages(session, api_url) ############ #all_pages[0]['title'] = "Haushalt" #all_pages[0]['title'] = "Über ONEH mediawiki" page_count = 0 for page in all_pages: if maxfiles == 0: break maxfiles -= 1 page_title = page['title'] params = { 'formatversion': 2, 'action': 'parse', 'page': page_title, 'format': 'json' } response = session.get(api_url, params=params) if response.status_code == 200: data = response.json() # wiki can have created pages without content, we ignore them here if 'parse' in data: page_count += 1 converter = html2text.HTML2Text() converter.IMAGES_AS_HTML = True markdown_content = converter.handle(data['parse']['text']) markdown_content = RemoveEditLinks(markdown_content) markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content) markdown_content = HandleInternalLinks(markdown_content) markdown_content += GenerateMetadataJson(session, data) markdown_content = StoreImages(session, data['parse']['images'], markdown_content) if markdown_content: save_content_to_file(markdown_content, pages_folder, page_title, extension='txt') print(f"Downloaded page {page_count}: {page_title}") else: print(f"Failed to download page {page_title}") if __name__ == "__main__": main()