diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 0489f84..efae7f4 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -161,17 +161,22 @@ def create_project( def _get_latest_pyscript_version() -> str: """Get the latest version of PyScript from GitHub.""" - url = "https://api.github.com/repos/pyscript/pyscript/releases/latest" + return _get_latest_repo_version("pyscript", "pyscript", LATEST_PYSCRIPT_VERSION) + + +def _get_latest_repo_version(gh_user: str, gh_repo: str, default: str): + """Get the latest version of given repo from GitHub.""" + url = f"https://api.github.com/repos/{gh_user}/{gh_repo}/releases/latest" try: response = requests.get(url) if not response.ok: - pyscript_version = LATEST_PYSCRIPT_VERSION + version = default else: data = response.json() - pyscript_version = data["tag_name"] + version = data["tag_name"] except Exception: - pyscript_version = LATEST_PYSCRIPT_VERSION + version = default - return pyscript_version + return version diff --git a/src/pyscript/cli.py b/src/pyscript/cli.py index 9757ca3..3eca504 100644 --- a/src/pyscript/cli.py +++ b/src/pyscript/cli.py @@ -8,7 +8,7 @@ from pyscript import __version__, app, console, plugins, typer from pyscript.plugins import hookspecs -DEFAULT_PLUGINS = ["create", "run"] +DEFAULT_PLUGINS = ["create", "run", "convert_offline"] def ok(msg: str = ""): diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py new file mode 100644 index 0000000..c4cbc56 --- /dev/null +++ b/src/pyscript/plugins/convert_offline.py @@ -0,0 +1,177 @@ +import json +import os +import re +import tarfile +import tempfile +from pathlib import Path + +import requests +import toml +import typer + +from pyscript import app, cli +from pyscript._generator import ( + _get_latest_pyscript_version, + _get_latest_repo_version, + save_config_file, +) + + +@app.command() +def convert_offline( + path: str = typer.Argument( + None, help="Path to pyscript project to convert for offline usage" + ), + config_files: str = typer.Option( + "pyscript.toml", "--config-files", help="Comma-separated list of config files" + ), + interpreter: str = typer.Option( + "pyodide", + "--interpreter", + help="Choose which interpreter to configure. Choices are 'pyodide' or 'micropython'", + ), + download_full_pyodide: bool = typer.Option( + False, + "--download-full-pyodide", + help="Download the 200MB+ pyodide libraries instead of just required interpreter", + ), +): + """ + Takes an existing pyscript app and converts it for offline usage + """ + app_path = Path(path) + PYSCRIPT_TAR_URL_BASE = ( + "https://pyscript.net/releases/{pyscript_version}/release.tar" + ) + PYODIDE_TAR_URL_BASE = ( + "https://github.com/pyodide/pyodide/" + "releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" + ) + MPY_BASE_URL = ( + "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" + ) + remote_pyscript_pattern = re.compile(r"https://pyscript.net/releases/[\d.]+/") + + if interpreter not in ("pyodide", "micropython"): + raise cli.Abort("Interpreter must be one of 'pyodide' or 'micropython'") + + # Get first app configuration to pull pyscript version + config_files_list = config_files.split(",") + config_path = app_path / config_files_list[0] + config = _get_config(config_path) + + # Get the required pyscript version based on config + pyscript_version = config.get("version", "latest") + if pyscript_version == "latest": + pyscript_version = _get_latest_pyscript_version() + + pyscript_tar_url = PYSCRIPT_TAR_URL_BASE.format(pyscript_version=pyscript_version) + pyscript_files_dir = app_path / "pyscript" + + # Download and extract pyscript files + print("Downloading pyscript files...") + _download_and_extract_tarfile(pyscript_tar_url, pyscript_files_dir) + print("Downloading and extraction of pyscript files successful.") + + # Download and extract pyodide files + print("Downloading pyodide files...") + pyodide_version = _get_latest_repo_version("pyodide", "pyodide", "") + if not pyodide_version: + raise cli.Abort("Unable to retrieve latest pyodide version from Github") + + # Only download the 200MB+ pyodide libraries if required, else just download + # the core + pyodide_tar_name = "pyodide" if download_full_pyodide else "pyodide-core" + pyodide_tar_url = PYODIDE_TAR_URL_BASE.format( + pyodide_version=pyodide_version, pyodide_tar_name=pyodide_tar_name + ) + _download_and_extract_tarfile(pyodide_tar_url, app_path) + print("Downloading and extraction of pyodide files successful.") + + # Download Micropython files + print("Downloading micropython files...") + mpy_path = app_path / "micropython" + mpy_path.mkdir(exist_ok=True) + files = ("micropython.mjs", "micropython.wasm") + + for file in files: + target_path = mpy_path / file + url = MPY_BASE_URL + file + + # wasm file is bytes format, mjs is text + response = requests.get(url) + if "wasm" in file: + with open(target_path, "wb") as fp: + fp.write(response.content) + else: + with open(target_path, "w") as fp: + fp.write(response.text) + print("Downloading of micropython files sucessful") + + # Finding all HTML files + html_files = [] + for dirname, dirs, filenames in os.walk(app_path): + dirpath = Path(dirname) + html_files.extend([dirpath / f for f in filenames if f.endswith(".html")]) + + # Replace remote resources with freshly downloaded resources + # Also for old config format to warn user + old_config_pattern = re.compile(r"py-config>") + found_old_config = False + + for filepath in html_files: + with open(filepath) as fpi: + content = fpi.read() + + if remote_pyscript_pattern.search(content): + new_content = remote_pyscript_pattern.sub("/pyscript/", content) + with open(filepath, "w") as fpo: + fpo.write(new_content) + print(f"Updated {filepath}") + + found_old_config = found_old_config or bool(old_config_pattern.search(content)) + + # Add/replace interpreter with downloaded interpreter + for config_file in config_files_list: + config_file_path = app_path / config_file + config = _get_config(config_file_path) + config["interpreter"] = f"/{interpreter}/{interpreter}.mjs" + save_config_file(config_file_path, config) + + print(f"Updated {config_file_path}") + + if found_old_config: + print( + "WARNING: and are not currently supported by this tool" + ) + + +def _download_and_extract_tarfile(remote_url: str, extract_dir: Path): + """Downloads the tarfile at `remote_url` and extracts it into `extract_dir` + + Params: + - remote_url(str): URL of the tarball, for example https://example.com/file.tar + - extract_dir(Path): directory to extract the tarball into + """ + with tempfile.TemporaryDirectory() as tempdirname: + tarfile_target = Path(tempdirname) / "temp.tar" + response = requests.get(remote_url, stream=True) + if response.status_code == 200: + with open(tarfile_target, "wb") as fp: + fp.write(response.raw.read()) + else: + raise cli.Abort( + "Unable to download required files. Please check your network connection" + ) + + with tarfile.open(tarfile_target, "r") as tfile: + tfile.extractall(path=extract_dir) + + +def _get_config(config_path: Path): + """Loads the configuration from the given Path""" + if "toml" in str(config_path): + return toml.load(config_path) + elif "json" in str(config_path): + with open(config_path) as fp: + return json.load(fp)