From 760ba9231665f997c0fb68979cc1a6a9a68660d0 Mon Sep 17 00:00:00 2001 From: trnq1ll0 Date: Sat, 31 Aug 2024 12:49:57 +0200 Subject: [PATCH] add lots of metadata including revision data --- README.md | 4 +++ scrape.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9cadbb0..fac0686 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ Das Paket lädt dafür alle Seiten eines MediaWikis herunter und speichert sie i * es wird html2text verwendet um die Seiten von HTML in Markdown zu konvertieren * Meta-informationen der Seite werden am Ende in einem json gespeichert * Dateien werden in einem separaten Ordner gespeichert un im Metadaten json richtig verlinkt +* Wikiseiten die zwar erstellt sind aber keinen Inhalt haben werden ignoriert +* entfernt aktuell nur deutschsprachigen links zum Abschnitt bearbeiten: "[Bearbeiten]" und "[Quelltext bearbeiten] +* Tabellen werden aktuell noch nicht richtig importiert +* revisionen werden in Metadata gespeichert mit timestamp und comment (falls vorhanden) ### Metadaten JSON diff --git a/scrape.py b/scrape.py index 9ac1be7..5924761 100644 --- a/scrape.py +++ b/scrape.py @@ -1,5 +1,6 @@ import os import json +import re import requests from bs4 import BeautifulSoup import html2text @@ -66,16 +67,16 @@ def save_content_to_file(content, download_folder, page_name, extension='md'): os.makedirs(os.path.dirname(file_path), exist_ok=True) with open(file_path, 'w', encoding='utf-8') as file: file.write(content) - print(f"Downloaded {page_name} to {file_path}") -def GenerateMetadataJson(data): +def GenerateMetadataJson(session, data): metadata = { 'metadata' : { 'categories' : data['parse']['categories'], 'internallinks' : data['parse']['links'], 'externallinks' : data['parse']['externallinks'], 'images' : data['parse']['images'], - 'showtoc' : data['parse']['showtoc'] + 'showtoc' : data['parse']['showtoc'], + 'revisions' : ProcessRevisions(session, api_url, data['parse']['pageid']) } } return "\n\n " + json.dumps(metadata, indent=4) @@ -116,6 +117,49 @@ def StoreImages(session, images_list, json_string): def ReplaceImagePath(str, image_name, image_path): return str.replace(f'"{image_name}"', f'"./{image_path}"') +def HandleExternalLinks(externallinks, markdown_content): + # for link in externallinks: + # markdown_content = markdown_content.replace(f"<{link}>", f'[{link}]({link})') + return markdown_content + +def HandleInternalLinks(markdown_content): + markdown_content = markdown_content.replace('/index.php?title=', '') + return markdown_content + +def RemoveEditLinks(markdown_content): + pattern = r'\[\[Bearbeiten\]\(.*?\)\s\|\s\[Quelltext bearbeiten\]\(.*?\)\]' + matches = re.findall(pattern, markdown_content) + # double check if the match is correct, than remove it + for match in matches: + if "[Bearbeiten]" in match and "[Quelltext bearbeiten]" in match: + markdown_content = markdown_content.replace(match, '') + return markdown_content + +def ProcessRevisions(session, api_url, page_id): + params = { + 'action': 'query', + 'format': 'json', + 'prop': 'revisions', + 'rvlimit': 500, + 'pageids': page_id + } + response = session.get(api_url, params=params) + if response.status_code == 200: + data = response.json() + first_child_key = next(iter(data['query']['pages'])) + revisions = data['query']['pages'][first_child_key]['revisions'] + revisons_block = [] + # print(f"Found {len(revisions)} revision: {json.dumps(revisions[0])}") + for revision in revisions: + revisons_block.append({ + 'timestamp' : revision['timestamp'], + 'comment' : revision['comment'] + }) + return revisons_block + else: + print(f"Failed to download revisions for page_id {page_id}") + + def main(): global api_url global images_folder @@ -135,7 +179,12 @@ def main(): with requests.Session() as session: login(session, api_url, username, password) all_pages = download_all_pages(session, api_url) - print(f"Downloading {len(all_pages)} pages") + + ############ + #all_pages[0]['title'] = "Haushalt" + #all_pages[0]['title'] = "Über ONEH mediawiki" + + page_count = 0 for page in all_pages: if maxfiles == 0: break @@ -150,16 +199,23 @@ def main(): response = session.get(api_url, params=params) if response.status_code == 200: data = response.json() + # wiki can have created pages without content, we ignore them here if 'parse' in data: - markdown_content = html2text.html2text(data['parse']['text']) - markdown_content += GenerateMetadataJson(data) + page_count += 1 + converter = html2text.HTML2Text() + converter.IMAGES_AS_HTML = True + markdown_content = converter.handle(data['parse']['text']) + + markdown_content = RemoveEditLinks(markdown_content) + markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content) + markdown_content = HandleInternalLinks(markdown_content) + markdown_content += GenerateMetadataJson(session, data) markdown_content = StoreImages(session, data['parse']['images'], markdown_content) if markdown_content: - save_content_to_file(markdown_content, pages_folder, page_title, extension='md') + save_content_to_file(markdown_content, pages_folder, page_title, extension='txt') + print(f"Downloaded page {page_count}: {page_title}") else: - print(f"Failed to parse page {page_title}") - else: - print(f"Failed to download page {page_title}") + print(f"Failed to download page {page_title}") if __name__ == "__main__": main() \ No newline at end of file