|
| 1 | +import re |
| 2 | +import os |
| 3 | +import random |
| 4 | +import sys |
| 5 | +from datetime import datetime, timezone |
| 6 | + |
| 7 | +import bs4 |
| 8 | +import cloudscraper |
| 9 | +requests = cloudscraper.create_scraper() |
| 10 | + |
| 11 | +if len(sys.argv) != 2: |
| 12 | + print(f"Error: This program needs 1 argument, got {len(sys.argv) - 1}\n") |
| 13 | + print(f"Usage: {sys.argv[0]} <file-to-write-to>\n") |
| 14 | + print("Parses all MCPE releases from Modscraft and writes to specified Markdown file.") |
| 15 | + sys.exit(1) |
| 16 | + |
| 17 | +def pathify(string): |
| 18 | + return re.sub(r'[^a-z0-9_.-]', '', string.replace(' ', '_').lower()) |
| 19 | + |
| 20 | +def create_md_table(data, width): |
| 21 | + table = f"{'| ' * width}|\n{'|-' * width}|\n" |
| 22 | + for i in range(0, len(data), width): |
| 23 | + table += "| " + " | ".join(data[i:i + width]) + " |\n" |
| 24 | + return table |
| 25 | + |
| 26 | +user_agents = [ |
| 27 | + "Mozilla/5.0 (Linux; Android 13; SM-M127G Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/126.0.6478.134 Mobile Safari/537.36", |
| 28 | + "Mozilla/5.0 (Android 11; Mobile; rv:128.0) Gecko/128.0 Firefox/128.0", |
| 29 | + "Mozilla/5.0 (Android 12; Mobile; RV:92.0) Gecko/92.0 Firefox/92.0", |
| 30 | + "Mozilla/5.0 (Linux; Android 10; Pixel 3 Build/QQ2A.200305.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/81.0.4044.138 Mobile Safari/537.36", |
| 31 | + "Mozilla/5.0 (Android 11; Mobile; RV:91.0) Gecko/91.0 Firefox/91.0", |
| 32 | + "Mozilla/5.0 (Linux; Android 9; Galaxy S9 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.116 Mobile Safari/537.36", |
| 33 | + "Mozilla/5.0 (Android 13; Mobile; RV:106.0) Gecko/106.0 Firefox/106.0", |
| 34 | + "Mozilla/5.0 (Linux; Android 8.0.0; Nexus 6P Build/OPR6.170623.013; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36", |
| 35 | + "Mozilla/5.0 (Android 12; Mobile; RV:98.0) Gecko/98.0 Firefox/98.0", |
| 36 | + "Mozilla/5.0 (Linux; Android 11; Galaxy A32 Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/88.0.4324.152 Mobile Safari/537.36", |
| 37 | + "Mozilla/5.0 (Linux; Android 13; Galaxy S21 Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/120.0.6092.0 Mobile Safari/537.36", |
| 38 | + "Mozilla/5.0 (Android 10; Mobile; RV:102.0) Gecko/102.0 Firefox/102.0", |
| 39 | + "Mozilla/5.0 (Linux; Android 8.1.0; LG G6 Build/N2G48H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/83.0.4103.101 Mobile Safari/537.36", |
| 40 | + "Mozilla/5.0 (Linux; Android 13; OnePlus 9 Pro Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/115.0.5790.163 Mobile Safari/537.36", |
| 41 | + "Mozilla/5.0 (Android 12; Mobile; RV:105.0) Gecko/105.0 Firefox/105.0", |
| 42 | + "Mozilla/5.0 (Linux; Android 11; Galaxy A50 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/89.0.4389.90 Mobile Safari/537.36", |
| 43 | + "Mozilla/5.0 (Android 9; Mobile; RV:101.0) Gecko/101.0 Firefox/101.0", |
| 44 | + "Mozilla/5.0 (Linux; Android 10; Galaxy Note 10 Build/QQ2A.200305.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/84.0.4147.125 Mobile Safari/537.36" |
| 45 | +] |
| 46 | +user_agent = random.choice(user_agents) |
| 47 | +print(f"* Parser has started") |
| 48 | +print(f"= User agent for today is \"{user_agent}\"") |
| 49 | +markdown_output = f"- :open_file_folder: Source available at [**ModsCraft.Net**](https://modscraft.net/en/mcpe/)" |
| 50 | +markdown_output += f"\n- :clock2: Updated **every 12 hours** at `00:00 UTC` and `12:00 UTC`" |
| 51 | +markdown_output += f"\n- :rocket: **Last update:** `{datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC`\n" |
| 52 | +print("* Creating directory 'version'") |
| 53 | +writedir = os.path.dirname(sys.argv[1]) |
| 54 | +os.makedirs(os.path.join(writedir, "version"), exist_ok=True) |
| 55 | +print("* Getting releases") |
| 56 | +resp = requests.get("https://modscraft.net/en/mcpe/", headers={"User-Agent": user_agent}) |
| 57 | +if not resp.ok: |
| 58 | + print(f"! ModsCraft returned {resp.status_code}") |
| 59 | + sys.exit(1) |
| 60 | +soup = bs4.BeautifulSoup(resp.text, "html.parser") |
| 61 | +releases = {i.text: i["href"] for i in soup.find("div", class_="versions-history").find_all("a")} |
| 62 | +version_links = [] |
| 63 | +for title, release in releases.items(): |
| 64 | + print(f"\n= Starting work on version {title}") |
| 65 | + ver = requests.get(release, headers={"User-Agent": user_agent}) |
| 66 | + if not ver.ok: |
| 67 | + print(f"! ModsCraft returned {resp.status_code}") |
| 68 | + sys.exit(1) |
| 69 | + rel_soup = bs4.BeautifulSoup(ver.text, "html.parser") |
| 70 | + version_output = f"## Minecraft {title} APKs\n" |
| 71 | + version_output += "| Download | Size |\n" |
| 72 | + version_output += "|----------|------|\n" |
| 73 | + for download in rel_soup.find_all("a", class_="download-item"): |
| 74 | + print("* Adding file ", end='') |
| 75 | + down_req = requests.get(download["href"], headers={"User-Agent": user_agent}) |
| 76 | + if not down_req.ok: |
| 77 | + print(f"! ModsCraft returned {resp.status_code}") |
| 78 | + sys.exit(1) |
| 79 | + apk = bs4.BeautifulSoup(down_req.text, "html.parser") |
| 80 | + download_id = re.search(r'id=(\d+)', download["href"]).group(1) |
| 81 | + down_spans = download.find_all("span") |
| 82 | + file_name = apk.find("p").text |
| 83 | + print(file_name) |
| 84 | + size = down_spans[2].text[1:-1] |
| 85 | + download_link = f"https://modscraft.net/en/downloads/{download_id}" |
| 86 | + version_output += f"| [:package: `{file_name}`]({download_link}) | :floppy_disk: {size} \n" |
| 87 | + print(f"= Finished work on version {title}") |
| 88 | + filename = f"mc{pathify(title)}.md" |
| 89 | + try: |
| 90 | + with open(os.path.join(writedir, "version", filename), "w") as f: |
| 91 | + f.write(version_output) |
| 92 | + except PermissionError: |
| 93 | + print("! Unable to access file, not enough permissions") |
| 94 | + sys.exit(1) |
| 95 | + except IOError as e: |
| 96 | + print(f"! I/O error while writing to file: {e}") |
| 97 | + sys.exit(1) |
| 98 | + print("= Adding to main file") |
| 99 | + version_links.append(f"**[:package: Minecraft {title}](version/{filename})**") |
| 100 | +markdown_output += f"\n{create_md_table(version_links, 3)}" |
| 101 | + |
| 102 | +print("\n= All done, writing to file") |
| 103 | +try: |
| 104 | + with open(os.path.join(sys.argv[1]), "w") as f: |
| 105 | + f.write(markdown_output) |
| 106 | +except PermissionError: |
| 107 | + print("! Unable to access file, not enough permissions") |
| 108 | + sys.exit(1) |
| 109 | +except IOError as e: |
| 110 | + print(f"! I/O error while writing to file: {e}") |
| 111 | + sys.exit(1) |
| 112 | +print(f"* Wrote to '{sys.argv[1]}' successfully") |
0 commit comments