add lots of metadata including revision data

scrape.py

@@ -1,5 +1,6 @@
 import os
 import json
+import re
 import requests
 from bs4 import BeautifulSoup
 import html2text
@@ -66,16 +67,16 @@ def save_content_to_file(content, download_folder, page_name, extension='md'):
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
     with open(file_path, 'w', encoding='utf-8') as file:
         file.write(content)
-    print(f"Downloaded {page_name} to {file_path}")
 
-def GenerateMetadataJson(data):
+def GenerateMetadataJson(session, data):
     metadata = {
         'metadata' : {
             'categories' : data['parse']['categories'],
             'internallinks' : data['parse']['links'],
             'externallinks' : data['parse']['externallinks'],
             'images' : data['parse']['images'],
-            'showtoc' : data['parse']['showtoc']
+            'showtoc' : data['parse']['showtoc'],
+            'revisions' : ProcessRevisions(session, api_url, data['parse']['pageid'])
         }
     }
     return "\n\n " + json.dumps(metadata, indent=4)
@@ -116,6 +117,49 @@ def StoreImages(session, images_list, json_string):
 def ReplaceImagePath(str, image_name, image_path):
     return str.replace(f'"{image_name}"', f'"./{image_path}"') 
 
+def HandleExternalLinks(externallinks, markdown_content):
+    # for link in externallinks:
+    #     markdown_content = markdown_content.replace(f"<{link}>", f'[{link}]({link})')
+    return markdown_content
+
+def HandleInternalLinks(markdown_content):
+    markdown_content = markdown_content.replace('/index.php?title=', '')
+    return markdown_content
+
+def RemoveEditLinks(markdown_content):
+    pattern = r'\[\[Bearbeiten\]\(.*?\)\s\|\s\[Quelltext bearbeiten\]\(.*?\)\]'
+    matches = re.findall(pattern, markdown_content)
+    # double check if the match is correct, than remove it
+    for match in matches:
+        if "[Bearbeiten]" in match and "[Quelltext bearbeiten]" in match:
+            markdown_content = markdown_content.replace(match, '')
+    return markdown_content
+
+def ProcessRevisions(session, api_url, page_id):
+    params = {
+        'action': 'query',
+        'format': 'json',
+        'prop': 'revisions',
+        'rvlimit': 500,
+        'pageids': page_id
+    }
+    response = session.get(api_url, params=params)
+    if response.status_code == 200:
+        data = response.json()
+        first_child_key = next(iter(data['query']['pages']))
+        revisions = data['query']['pages'][first_child_key]['revisions']
+        revisons_block = []
+        # print(f"Found {len(revisions)} revision: {json.dumps(revisions[0])}")
+        for revision in revisions:
+            revisons_block.append({
+                'timestamp' : revision['timestamp'],
+                'comment' : revision['comment']
+            })
+        return revisons_block
+    else:
+        print(f"Failed to download revisions for page_id {page_id}")
+
+
 def main():
     global api_url 
     global images_folder
@@ -135,7 +179,12 @@ def main():
     with requests.Session() as session:
         login(session, api_url, username, password)
         all_pages = download_all_pages(session, api_url)
-        print(f"Downloading {len(all_pages)} pages")
+
+        ############
+        #all_pages[0]['title'] = "Haushalt"
+        #all_pages[0]['title'] = "Über ONEH mediawiki"
+
+        page_count = 0
         for page in all_pages:
             if maxfiles == 0:
                 break
@@ -150,16 +199,23 @@ def main():
             response = session.get(api_url, params=params)
             if response.status_code == 200:
                 data = response.json()
+                # wiki can have created pages without content, we ignore them here 
                 if 'parse' in data:
-                    markdown_content = html2text.html2text(data['parse']['text'])
-                    markdown_content += GenerateMetadataJson(data)
+                    page_count += 1
+                    converter = html2text.HTML2Text()
+                    converter.IMAGES_AS_HTML = True
+                    markdown_content = converter.handle(data['parse']['text'])
+                    
+                    markdown_content = RemoveEditLinks(markdown_content)
+                    markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content)
+                    markdown_content = HandleInternalLinks(markdown_content)
+                    markdown_content += GenerateMetadataJson(session, data)
                     markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
                     if markdown_content:
-                        save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
+                        save_content_to_file(markdown_content, pages_folder, page_title, extension='txt')
+                        print(f"Downloaded page {page_count}: {page_title}")
             else:
-                print(f"Failed to parse page {page_title}")
-        else:
-            print(f"Failed to download page {page_title}")
+                print(f"Failed to download page {page_title}")
 
 if __name__ == "__main__":
     main()