generate table of content (toc) with section links

scrape.py

@@ -158,6 +158,25 @@ def GetRevisionData(session, api_url, page_id):
     else:
         print(f"Failed to download revisions for page_id {page_id}")
 
+def BuildToc(data):
+    toc = ""
+    if data['parse']['showtoc'] == False:
+        return toc
+    toc = "## Inhaltsverzeichnis\n\n"
+    for section in data['parse']['sections']:
+        intend = "\t" * (section['toclevel'] - 1)
+        number = section['number'].split('.')[section['toclevel']-1]
+        toc += f"{intend}{number}) [[#{section['line']}|{section['line']}]]\n"
+    return toc + "\n"
+
+def RemoveToc(html, remove_toc):
+    if remove_toc == False:
+        return html
+    soup = BeautifulSoup(html, 'html.parser')
+    toc_div = soup.find('div', {'id': 'toc'})
+    if toc_div:
+        toc_div.decompose()
+    return str(soup)
 
 def main():
     global api_url 
@@ -180,8 +199,7 @@ def main():
         all_pages = download_all_pages(session, api_url)
 
         ############
-        #all_pages[0]['title'] = "Haushalt"
+        all_pages[0]['title'] = "Linux"
-        #all_pages[0]['title'] = "Über ONEH mediawiki"
 
         page_count = 0
         for page in all_pages:
@@ -203,15 +221,19 @@ def main():
                     page_count += 1
                     converter = html2text.HTML2Text()
                     converter.IMAGES_AS_HTML = True
-                    markdown_content = converter.handle(data['parse']['text'])
+                    converter.SKIP_INTERNAL_LINKS = False
-                    
+                    converter.IGNORE_ANCHORS = False
+                    converter.SINGLE_LINE_BREAK = True
+                    converter.MARK_CODE = True
+                    html = RemoveToc(data['parse']['text'], data['parse']['showtoc'])
+                    markdown_content = converter.handle(html)
                     markdown_content = RemoveEditLinks(markdown_content)
                     markdown_content = HandleExternalLinks(data['parse']['externallinks'], markdown_content)
                     markdown_content = HandleInternalLinks(markdown_content)
                     markdown_content += GenerateMetadataJson(session, data)
                     markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
                     if markdown_content:
-                        save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
+                        save_content_to_file(BuildToc(data) + markdown_content, pages_folder, page_title, extension='md')
                         print(f"Downloaded page {page_count}: {page_title}")
             else:
                 print(f"Failed to download page {page_title}")