rename things

scrape.py

@@ -62,8 +62,8 @@ def download_all_pages(session, api_url):
             break
     return all_pages
     
-def save_content_to_file(content, download_folder, page_name, extension='md'):
-    file_path = os.path.join(download_folder, f"{page_name}.{extension}")
+def save_content_to_file(content, download_folder, page_title, extension='md'):
+    file_path = os.path.join(download_folder, f"{page_title}.{extension}")
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
     with open(file_path, 'w', encoding='utf-8') as file:
         file.write(content)
@@ -76,7 +76,7 @@ def GenerateMetadataJson(session, data):
             'externallinks' : data['parse']['externallinks'],
             'images' : data['parse']['images'],
             'showtoc' : data['parse']['showtoc'],
-            'revisions' : ProcessRevisions(session, api_url, data['parse']['pageid'])
+            'revisions' : GetRevisionData(session, api_url, data['parse']['pageid'])
         }
     }
     return "\n\n " + json.dumps(metadata, indent=4)
@@ -135,7 +135,7 @@ def RemoveEditLinks(markdown_content):
             markdown_content = markdown_content.replace(match, '')
     return markdown_content
 
-def ProcessRevisions(session, api_url, page_id):
+def GetRevisionData(session, api_url, page_id):
     params = {
         'action': 'query',
         'format': 'json',
@@ -149,7 +149,6 @@ def ProcessRevisions(session, api_url, page_id):
         first_child_key = next(iter(data['query']['pages']))
         revisions = data['query']['pages'][first_child_key]['revisions']
         revisons_block = []
-        # print(f"Found {len(revisions)} revision: {json.dumps(revisions[0])}")
         for revision in revisions:
             revisons_block.append({
                 'timestamp' : revision['timestamp'],
@@ -212,7 +211,7 @@ def main():
                     markdown_content += GenerateMetadataJson(session, data)
                     markdown_content = StoreImages(session, data['parse']['images'], markdown_content)
                     if markdown_content:
-                        save_content_to_file(markdown_content, pages_folder, page_title, extension='txt')
+                        save_content_to_file(markdown_content, pages_folder, page_title, extension='md')
                         print(f"Downloaded page {page_count}: {page_title}")
             else:
                 print(f"Failed to download page {page_title}")