|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Simpler reimplementation of Android's sdkmanager |
| 3 | +# Extra features of this implementation are pinning and mirroring |
| 4 | + |
| 5 | +# These URLs are the Google repositories containing the list of available |
| 6 | +# packages and their versions. The list has been generated by listing the URLs |
| 7 | +# fetched while executing `tools/bin/sdkmanager --list` |
| 8 | +BASE_REPOSITORY = "https://dl.google.com/android/repository/" |
| 9 | +REPOSITORIES = [ |
| 10 | + "sys-img/android/sys-img2-1.xml", |
| 11 | + "sys-img/android-wear/sys-img2-1.xml", |
| 12 | + "sys-img/android-wear-cn/sys-img2-1.xml", |
| 13 | + "sys-img/android-tv/sys-img2-1.xml", |
| 14 | + "sys-img/google_apis/sys-img2-1.xml", |
| 15 | + "sys-img/google_apis_playstore/sys-img2-1.xml", |
| 16 | + "addon2-1.xml", |
| 17 | + "glass/addon2-1.xml", |
| 18 | + "extras/intel/addon2-1.xml", |
| 19 | + "repository2-1.xml", |
| 20 | +] |
| 21 | + |
| 22 | +# Available hosts: linux, macosx and windows |
| 23 | +HOST_OS = "linux" |
| 24 | + |
| 25 | +# Mirroring options |
| 26 | +MIRROR_BUCKET = "rust-lang-ci2" |
| 27 | +MIRROR_BASE_DIR = "rust-ci-mirror/android/" |
| 28 | + |
| 29 | +import argparse |
| 30 | +import hashlib |
| 31 | +import os |
| 32 | +import subprocess |
| 33 | +import sys |
| 34 | +import tempfile |
| 35 | +import urllib.request |
| 36 | +import xml.etree.ElementTree as ET |
| 37 | + |
| 38 | +class Package: |
| 39 | + def __init__(self, path, url, sha1, deps=None): |
| 40 | + if deps is None: |
| 41 | + deps = [] |
| 42 | + self.path = path.strip() |
| 43 | + self.url = url.strip() |
| 44 | + self.sha1 = sha1.strip() |
| 45 | + self.deps = deps |
| 46 | + |
| 47 | + def download(self, base_url): |
| 48 | + _, file = tempfile.mkstemp() |
| 49 | + url = base_url + self.url |
| 50 | + subprocess.run(["curl", "-o", file, url], check=True) |
| 51 | + # Ensure there are no hash mismatches |
| 52 | + with open(file, "rb") as f: |
| 53 | + sha1 = hashlib.sha1(f.read()).hexdigest() |
| 54 | + if sha1 != self.sha1: |
| 55 | + raise RuntimeError( |
| 56 | + "hash mismatch for package " + self.path + ": " + |
| 57 | + sha1 + " vs " + self.sha1 + " (known good)" |
| 58 | + ) |
| 59 | + return file |
| 60 | + |
| 61 | + def __repr__(self): |
| 62 | + return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" |
| 63 | + |
| 64 | +def fetch_url(url): |
| 65 | + page = urllib.request.urlopen(url) |
| 66 | + return page.read() |
| 67 | + |
| 68 | +def fetch_repository(base, repo_url): |
| 69 | + packages = {} |
| 70 | + root = ET.fromstring(fetch_url(base + repo_url)) |
| 71 | + for package in root: |
| 72 | + if package.tag != "remotePackage": |
| 73 | + continue |
| 74 | + path = package.attrib["path"] |
| 75 | + |
| 76 | + for archive in package.find("archives"): |
| 77 | + host_os = archive.find("host-os") |
| 78 | + if host_os is not None and host_os.text != HOST_OS: |
| 79 | + continue |
| 80 | + complete = archive.find("complete") |
| 81 | + url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) |
| 82 | + sha1 = complete.find("checksum").text |
| 83 | + |
| 84 | + deps = [] |
| 85 | + dependencies = package.find("dependencies") |
| 86 | + if dependencies is not None: |
| 87 | + for dep in dependencies: |
| 88 | + deps.append(dep.attrib["path"]) |
| 89 | + |
| 90 | + packages[path] = Package(path, url, sha1, deps) |
| 91 | + break |
| 92 | + |
| 93 | + return packages |
| 94 | + |
| 95 | +def fetch_repositories(): |
| 96 | + packages = {} |
| 97 | + for repo in REPOSITORIES: |
| 98 | + packages.update(fetch_repository(BASE_REPOSITORY, repo)) |
| 99 | + return packages |
| 100 | + |
| 101 | +class Lockfile: |
| 102 | + def __init__(self, path): |
| 103 | + self.path = path |
| 104 | + self.packages = {} |
| 105 | + if os.path.exists(path): |
| 106 | + with open(path) as f: |
| 107 | + for line in f: |
| 108 | + path, url, sha1 = line.split(" ") |
| 109 | + self.packages[path] = Package(path, url, sha1) |
| 110 | + |
| 111 | + def add(self, packages, name, *, update=True): |
| 112 | + if name not in packages: |
| 113 | + raise NameError("package not found: " + name) |
| 114 | + if not update and name in self.packages: |
| 115 | + return |
| 116 | + self.packages[name] = packages[name] |
| 117 | + for dep in packages[name].deps: |
| 118 | + self.add(packages, dep, update=False) |
| 119 | + |
| 120 | + def save(self): |
| 121 | + packages = list(sorted(self.packages.values(), key=lambda p: p.path)) |
| 122 | + with open(self.path, "w") as f: |
| 123 | + for package in packages: |
| 124 | + f.write(package.path + " " + package.url + " " + package.sha1 + "\n") |
| 125 | + |
| 126 | +def cli_add_to_lockfile(args): |
| 127 | + lockfile = Lockfile(args.lockfile) |
| 128 | + packages = fetch_repositories() |
| 129 | + for package in args.packages: |
| 130 | + lockfile.add(packages, package) |
| 131 | + lockfile.save() |
| 132 | + |
| 133 | +def cli_update_mirror(args): |
| 134 | + lockfile = Lockfile(args.lockfile) |
| 135 | + for package in lockfile.packages.values(): |
| 136 | + path = package.download(BASE_REPOSITORY) |
| 137 | + subprocess.run([ |
| 138 | + "aws", "s3", "mv", path, |
| 139 | + "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, |
| 140 | + "--profile=" + args.awscli_profile, |
| 141 | + ], check=True) |
| 142 | + |
| 143 | +def cli_install(args): |
| 144 | + lockfile = Lockfile(args.lockfile) |
| 145 | + for package in lockfile.packages.values(): |
| 146 | + # Download the file from the mirror into a temp file |
| 147 | + url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR |
| 148 | + downloaded = package.download(url) |
| 149 | + # Extract the file in a temporary directory |
| 150 | + extract_dir = tempfile.mkdtemp() |
| 151 | + subprocess.run([ |
| 152 | + "unzip", "-q", downloaded, "-d", extract_dir, |
| 153 | + ], check=True) |
| 154 | + # Figure out the prefix used in the zip |
| 155 | + subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] |
| 156 | + if len(subdirs) != 1: |
| 157 | + raise RuntimeError("extracted directory contains more than one dir") |
| 158 | + # Move the extracted files in the proper directory |
| 159 | + dest = os.path.join(args.dest, package.path.replace(";", "/")) |
| 160 | + os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) |
| 161 | + os.rename(os.path.join(extract_dir, subdirs[0]), dest) |
| 162 | + os.unlink(downloaded) |
| 163 | + |
| 164 | +def cli(): |
| 165 | + parser = argparse.ArgumentParser() |
| 166 | + subparsers = parser.add_subparsers() |
| 167 | + |
| 168 | + add_to_lockfile = subparsers.add_parser("add-to-lockfile") |
| 169 | + add_to_lockfile.add_argument("lockfile") |
| 170 | + add_to_lockfile.add_argument("packages", nargs="+") |
| 171 | + add_to_lockfile.set_defaults(func=cli_add_to_lockfile) |
| 172 | + |
| 173 | + update_mirror = subparsers.add_parser("update-mirror") |
| 174 | + update_mirror.add_argument("lockfile") |
| 175 | + update_mirror.add_argument("--awscli-profile", default="default") |
| 176 | + update_mirror.set_defaults(func=cli_update_mirror) |
| 177 | + |
| 178 | + install = subparsers.add_parser("install") |
| 179 | + install.add_argument("lockfile") |
| 180 | + install.add_argument("dest") |
| 181 | + install.set_defaults(func=cli_install) |
| 182 | + |
| 183 | + args = parser.parse_args() |
| 184 | + if not hasattr(args, "func"): |
| 185 | + print("error: a subcommand is required (see --help)") |
| 186 | + exit(1) |
| 187 | + args.func(args) |
| 188 | + |
| 189 | +if __name__ == "__main__": |
| 190 | + cli() |
0 commit comments