diff --git a/scrape.py b/scrape.py index cc8d1d2..de3ad4a 100644 --- a/scrape.py +++ b/scrape.py @@ -158,6 +158,25 @@ def GetRevisionData(session, api_url, page_id): else: print(f"Failed to download revisions for page_id {page_id}") +def BuildToc(data): + toc = "" + if data['parse']['showtoc'] == False: + return toc + toc = "## Inhaltsverzeichnis\n\n" + for section in data['parse']['sections']: + intend = "\t" * (section['toclevel'] - 1) + number = section['number'].split('.')[section['toclevel']-1] + toc += f"{intend}{number}) [[#{section['line']}|{section['line']}]]\n" + return toc + "\n" + +def RemoveToc(html, remove_toc): + if remove_toc == False: + return html + soup = BeautifulSoup(html, 'html.parser') + toc_div = soup.find('div', {'id': 'toc'}) + if toc_div: + toc_div.decompose() + return str(soup) def main(): global api_url @@ -180,8 +199,7 @@ def main(): all_pages = download_all_pages(session, api_url) ############ - #all_pages[0]['title'] = "Haushalt" - #all_pages[0]['title'] = "Über ONEH mediawiki" + all_pages[0]['title'] = "Linux" page_count = 0 for page in all_pages: @@ -203,15 +221,19 @@ def main(): page_count += 1 converter = html2text.HTML2Text() converter.IMAGES_AS_HTML = True - markdown_content = converter.handle(data['parse']['text']) - + converter.SKIP_INTERNAL_LINKS = False + converter.IGNORE_ANCHORS = False + converter.SINGLE_LINE_BREAK = True + converter.MARK_CODE = True + html = RemoveToc(data['parse']['text'], data['parse']['showtoc']) + markdown_content = converter.handle(html) markdown_content = RemoveEditLinks(markdown_content) markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content) markdown_content = HandleInternalLinks(markdown_content) markdown_content += GenerateMetadataJson(session, data) markdown_content = StoreImages(session, data['parse']['images'], markdown_content) if markdown_content: - save_content_to_file(markdown_content, pages_folder, page_title, extension='md') + save_content_to_file(BuildToc(data) + markdown_content, pages_folder, page_title, extension='md') print(f"Downloaded page {page_count}: {page_title}") else: print(f"Failed to download page {page_title}")