220 lines
7.6 KiB
Python
220 lines
7.6 KiB
Python
import os
|
|
import json
|
|
import re
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import html2text
|
|
import urllib3
|
|
import warnings
|
|
|
|
api_url = ""
|
|
images_folder = ""
|
|
|
|
print("SSL waring ist okay, kann ignoriert werden.")
|
|
|
|
def load_config(config_file):
|
|
with open(config_file, 'r') as file:
|
|
return json.load(file)
|
|
|
|
def login(session, api_url, username, password):
|
|
params = {
|
|
"action": "query",
|
|
"meta": "tokens",
|
|
"type": "login",
|
|
"format": "json"
|
|
}
|
|
response = session.get(api_url, params=params)
|
|
login_token = response.json()['query']['tokens']['logintoken']
|
|
|
|
login_payload = {
|
|
"action": "login",
|
|
"lgname": username,
|
|
"lgpassword": password,
|
|
"lgtoken": login_token,
|
|
"format": "json"
|
|
}
|
|
response = session.post(api_url, data=login_payload)
|
|
if response.status_code == 200:
|
|
print("Login successful")
|
|
else:
|
|
print("Login failed")
|
|
response.raise_for_status()
|
|
|
|
def download_all_pages(session, api_url):
|
|
all_pages = []
|
|
params = {
|
|
'action': 'query',
|
|
'format': 'json',
|
|
'list': 'allpages',
|
|
'aplimit' : 'max'
|
|
}
|
|
while True:
|
|
response = session.get(api_url, params=params)
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
all_pages.extend(data['query']['allpages'])
|
|
if 'continue' in data:
|
|
params['apcontinue'] = data['continue']['apcontinue']
|
|
else:
|
|
break
|
|
else:
|
|
print(f"Failed to download pages from {api_url}")
|
|
break
|
|
return all_pages
|
|
|
|
def save_content_to_file(content, download_folder, page_title, extension='md'):
|
|
file_path = os.path.join(download_folder, f"{page_title}.{extension}")
|
|
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
|
with open(file_path, 'w', encoding='utf-8') as file:
|
|
file.write(content)
|
|
|
|
def GenerateMetadataJson(session, data):
|
|
metadata = {
|
|
'metadata' : {
|
|
'categories' : data['parse']['categories'],
|
|
'internallinks' : data['parse']['links'],
|
|
'externallinks' : data['parse']['externallinks'],
|
|
'images' : data['parse']['images'],
|
|
'showtoc' : data['parse']['showtoc'],
|
|
'revisions' : GetRevisionData(session, api_url, data['parse']['pageid'])
|
|
}
|
|
}
|
|
return "\n\n " + json.dumps(metadata, indent=4)
|
|
|
|
def StoreImages(session, images_list, json_string):
|
|
for image in images_list:
|
|
params = {
|
|
'action': 'query',
|
|
'format': 'json',
|
|
'titles': f'File:{image}',
|
|
'prop': 'imageinfo',
|
|
'iiprop': 'url'
|
|
}
|
|
response = session.get(api_url, params=params)
|
|
if response.status_code != 200:
|
|
print(f"Failed to download image {image}")
|
|
continue
|
|
data = response.json()
|
|
pages = data['query']['pages']
|
|
page_id = list(pages.keys())[0]
|
|
image_info = pages[page_id].get('imageinfo', [{}])[0]
|
|
image_url = image_info.get('url')
|
|
if not image_url:
|
|
print(f"No image info found for {image}")
|
|
continue
|
|
image_name = image_url.split('/')[-1]
|
|
response = session.get(image_url)
|
|
if response.status_code != 200:
|
|
print(f"Failed to download image {image_name}")
|
|
continue
|
|
image_path = os.path.join(images_folder, image_name)
|
|
with open(image_path, 'wb') as file:
|
|
file.write(response.content)
|
|
print(f"Downloaded {image_name}")
|
|
json_string = ReplaceImagePath(json_string, image_name, image_path)
|
|
return json_string
|
|
|
|
def ReplaceImagePath(str, image_name, image_path):
|
|
return str.replace(f'"{image_name}"', f'"./{image_path}"')
|
|
|
|
def HandleExternalLinks(externallinks, markdown_content):
|
|
# for link in externallinks:
|
|
# markdown_content = markdown_content.replace(f"<{link}>", f'[{link}]({link})')
|
|
return markdown_content
|
|
|
|
def HandleInternalLinks(markdown_content):
|
|
markdown_content = markdown_content.replace('/index.php?title=', '')
|
|
return markdown_content
|
|
|
|
def RemoveEditLinks(markdown_content):
|
|
pattern = r'\[\[Bearbeiten\]\(.*?\)\s\|\s\[Quelltext bearbeiten\]\(.*?\)\]'
|
|
matches = re.findall(pattern, markdown_content)
|
|
# double check if the match is correct, than remove it
|
|
for match in matches:
|
|
if "[Bearbeiten]" in match and "[Quelltext bearbeiten]" in match:
|
|
markdown_content = markdown_content.replace(match, '')
|
|
return markdown_content
|
|
|
|
def GetRevisionData(session, api_url, page_id):
|
|
params = {
|
|
'action': 'query',
|
|
'format': 'json',
|
|
'prop': 'revisions',
|
|
'rvlimit': 500,
|
|
'pageids': page_id
|
|
}
|
|
response = session.get(api_url, params=params)
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
first_child_key = next(iter(data['query']['pages']))
|
|
revisions = data['query']['pages'][first_child_key]['revisions']
|
|
revisons_block = []
|
|
for revision in revisions:
|
|
revisons_block.append({
|
|
'timestamp' : revision['timestamp'],
|
|
'comment' : revision['comment']
|
|
})
|
|
return revisons_block
|
|
else:
|
|
print(f"Failed to download revisions for page_id {page_id}")
|
|
|
|
|
|
def main():
|
|
global api_url
|
|
global images_folder
|
|
config = load_config('config.json')
|
|
api_url = config['api_url'].rstrip('/') # Entferne den abschließenden Schrägstrich, falls vorhanden
|
|
pages_folder = config['pages']
|
|
images_folder = config['images']
|
|
username = config['username']
|
|
password = config['password']
|
|
maxfiles = config['maxfiles']
|
|
|
|
if not os.path.exists(pages_folder):
|
|
os.makedirs(pages_folder)
|
|
if not os.path.exists(images_folder):
|
|
os.makedirs(images_folder)
|
|
|
|
with requests.Session() as session:
|
|
login(session, api_url, username, password)
|
|
all_pages = download_all_pages(session, api_url)
|
|
|
|
############
|
|
#all_pages[0]['title'] = "Haushalt"
|
|
#all_pages[0]['title'] = "Über ONEH mediawiki"
|
|
|
|
page_count = 0
|
|
for page in all_pages:
|
|
if maxfiles == 0:
|
|
break
|
|
maxfiles -= 1
|
|
page_title = page['title']
|
|
params = {
|
|
'formatversion': 2,
|
|
'action': 'parse',
|
|
'page': page_title,
|
|
'format': 'json'
|
|
}
|
|
response = session.get(api_url, params=params)
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
# wiki can have created pages without content, we ignore them here
|
|
if 'parse' in data:
|
|
page_count += 1
|
|
converter = html2text.HTML2Text()
|
|
converter.IMAGES_AS_HTML = True
|
|
markdown_content = converter.handle(data['parse']['text'])
|
|
|
|
markdown_content = RemoveEditLinks(markdown_content)
|
|
markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content)
|
|
markdown_content = HandleInternalLinks(markdown_content)
|
|
markdown_content += GenerateMetadataJson(session, data)
|
|
markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
|
|
if markdown_content:
|
|
save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
|
|
print(f"Downloaded page {page_count}: {page_title}")
|
|
else:
|
|
print(f"Failed to download page {page_title}")
|
|
|
|
if __name__ == "__main__":
|
|
main() |