rob/MediaWiki-Scraper
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename things

Browse Source
This commit is contained in:
trnq1ll0 committed 2024-08-31 14:22:33 +02:00
1 parent 760ba92316
commit c2328ec96e
1 file changed
+5 -6
Download Patch File Download Diff File Expand all files Collapse all files
scrape.py
+5 -6
View File
@@ -62,8 +62,8 @@ def download_all_pages(session, api_url):
break break
return all_pages return all_pages
def save_content_to_file(content, download_folder, page_name, extension='md'): def save_content_to_file(content, download_folder, page_title, extension='md'):
file_path = os.path.join(download_folder, f"{page_name}.{extension}") file_path = os.path.join(download_folder, f"{page_title}.{extension}")
os.makedirs(os.path.dirname(file_path), exist_ok=True) os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'w', encoding='utf-8') as file: with open(file_path, 'w', encoding='utf-8') as file:
file.write(content) file.write(content)
@@ -76,7 +76,7 @@ def GenerateMetadataJson(session, data):
'externallinks' : data['parse']['externallinks'], 'externallinks' : data['parse']['externallinks'],
'images' : data['parse']['images'], 'images' : data['parse']['images'],
'showtoc' : data['parse']['showtoc'], 'showtoc' : data['parse']['showtoc'],
'revisions' : ProcessRevisions(session, api_url, data['parse']['pageid']) 'revisions' : GetRevisionData(session, api_url, data['parse']['pageid'])
} }
} }
return "\n\n " + json.dumps(metadata, indent=4) return "\n\n " + json.dumps(metadata, indent=4)
@@ -135,7 +135,7 @@ def RemoveEditLinks(markdown_content):
markdown_content = markdown_content.replace(match, '') markdown_content = markdown_content.replace(match, '')
return markdown_content return markdown_content
def ProcessRevisions(session, api_url, page_id): def GetRevisionData(session, api_url, page_id):
params = { params = {
'action': 'query', 'action': 'query',
'format': 'json', 'format': 'json',
@@ -149,7 +149,6 @@ def ProcessRevisions(session, api_url, page_id):
first_child_key = next(iter(data['query']['pages'])) first_child_key = next(iter(data['query']['pages']))
revisions = data['query']['pages'][first_child_key]['revisions'] revisions = data['query']['pages'][first_child_key]['revisions']
revisons_block = [] revisons_block = []
# print(f"Found {len(revisions)} revision: {json.dumps(revisions[0])}")
for revision in revisions: for revision in revisions:
revisons_block.append({ revisons_block.append({
'timestamp' : revision['timestamp'], 'timestamp' : revision['timestamp'],
@@ -212,7 +211,7 @@ def main():
markdown_content += GenerateMetadataJson(session, data) markdown_content += GenerateMetadataJson(session, data)
markdown_content = StoreImages(session, data['parse']['images'], markdown_content) markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
if markdown_content: if markdown_content:
save_content_to_file(markdown_content, pages_folder, page_title, extension='txt') save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
print(f"Downloaded page {page_count}: {page_title}") print(f"Downloaded page {page_count}: {page_title}")
else: else:
print(f"Failed to download page {page_title}") print(f"Failed to download page {page_title}")