Using Pyodide lockFileURL + packages when a cache exists

Author: WebReflection

docs/index.js

@@ -3,11 +3,14 @@ import { create } from 'gc-hook';
 import { RUNNING_IN_WORKER, createProgress, writeFile } from './_utils.js';
 import { getFormat, loader, loadProgress, registerJSModule, run, runAsync, runEvent } from './_python.js';
 import { stdio } from './_io.js';
-import { isArray } from '../utils.js';
+import { IDBMapSync, isArray } from '../utils.js';
 
 const type = 'pyodide';
 const toJsOptions = { dict_converter: Object.fromEntries };
 
+const { stringify } = JSON;
+
+// REQUIRES INTEGRATION TEST
 /* c8 ignore start */
 let overrideFunction = false;
 const overrideMethod = method => (...args) => {
@@ -75,10 +78,7 @@ const applyOverride = () => {
 };
 
 const progress = createProgress('py');
-/* c8 ignore stop */
 
-// REQUIRES INTEGRATION TEST
-/* c8 ignore start */
 export default {
     type,
     module: (version = '0.26.2') =>
@@ -88,15 +88,43 @@ export default {
         if (!RUNNING_IN_WORKER && config.experimental_create_proxy === 'auto')
             applyOverride();
         progress('Loading Pyodide');
-        const { stderr, stdout, get } = stdio();
+        let { packages } = config;
+        progress('Loading Storage');
         const indexURL = url.slice(0, url.lastIndexOf('/'));
+        // each pyodide version shares its own cache
+        const storage = new IDBMapSync(indexURL);
+        const options = { indexURL };
+        const save = config.packages_cache !== 'never';
+        await storage.sync();
+        // packages_cache = 'never' means: erase the whole DB
+        if (!save) storage.clear();
+        // otherwise check if cache is known
+        else if (packages) {
+            // packages are uniquely stored as JSON key
+            const key = stringify(packages);
+            if (storage.has(key)) {
+                const blob = new Blob(
+                    [storage.get(key)],
+                    { type: 'application/json' },
+                );
+                // this should be used to bootstrap loadPyodide
+                options.lockFileURL = URL.createObjectURL(blob);
+                // no need to use micropip manually here
+                options.packages = packages;
+                packages = null;
+            }
+        }
+        progress('Loaded Storage');
+        const { stderr, stdout, get } = stdio();
         const interpreter = await get(
-            loadPyodide({ stderr, stdout, indexURL }),
+            loadPyodide({ stderr, stdout, ...options }),
         );
         const py_imports = importPackages.bind(interpreter);
         loader.set(interpreter, py_imports);
         await loadProgress(this, progress, interpreter, config, baseURL);
-        if (config.packages) await py_imports(config.packages);
+        // if cache wasn't know, import and freeze it for the next time
+        if (packages) await py_imports(packages, storage, save);
+        await storage.close();
         progress('Loaded Pyodide');
         return interpreter;
     },
@@ -130,7 +158,7 @@ function transform(value) {
 }
 
 // exposed utility to import packages via polyscript.lazy_py_modules
-async function importPackages(packages) {
+async function importPackages(packages, storage, save = false) {
     // temporary patch/fix console.log which is used
     // not only by Pyodide but by micropip too and there's
     // no way to intercept those calls otherwise
@@ -146,6 +174,10 @@ async function importPackages(packages) {
     const micropip = this.pyimport('micropip');
     await micropip.install(packages, { keep_going: true });
     console.log = log;
+    if (save && (storage instanceof IDBMapSync)) {
+        const frozen = micropip.freeze();
+        storage.set(stringify(packages), frozen);
+    }
     micropip.destroy();
 }
 /* c8 ignore stop */

docs/index.js.map

@@ -43,7 +43,6 @@ const parseString = config => {
         return toml(config);
     }
 };
-/* c8 ignore stop */
 
 /**
  * Parse a generic config if it came from an attribute either as URL
@@ -58,10 +57,7 @@ const parseString = config => {
  */
 export const getRuntime = (id, config, configURL, options = {}) => {
     if (config) {
-        // REQUIRES INTEGRATION TEST
-        /* c8 ignore start */
         [options, config] = resolveConfig(config, configURL, options);
-        /* c8 ignore stop */
     }
     return resolve(options).then(options => interpreter[id](options, config));
 };
@@ -73,3 +69,5 @@ export const getRuntime = (id, config, configURL, options = {}) => {
  */
 export const getRuntimeID = (type, version = '') =>
     `${type}@${version}`.replace(/@$/, '');
+
+/* c8 ignore stop */

esm/interpreter/pyodide.js

@@ -8,6 +8,7 @@ import { getRuntime, getRuntimeID } from './loader.js';
 import { registry } from './interpreters.js';
 import { JSModules, isSync, all, dispatch, resolve, defineProperty, nodeInfo, registerJSModules } from './utils.js';
 
+/* c8 ignore start */
 const getRoot = (script) => {
     let parent = script;
     while (parent.parentNode) parent = parent.parentNode;
@@ -47,7 +48,6 @@ export const interpreters = new Map();
 const execute = async (currentScript, source, XWorker, isAsync) => {
     const { type } = currentScript;
     const module = registry.get(type);
-    /* c8 ignore start */
     if (module.experimental)
         console.warn(`The ${type} interpreter is experimental`);
     const [interpreter, content] = await all([
@@ -77,7 +77,6 @@ const execute = async (currentScript, source, XWorker, isAsync) => {
     } finally {
         delete document.currentScript;
     }
-    /* c8 ignore stop */
 };
 
 const getValue = (ref, prefix) => {
@@ -95,10 +94,8 @@ export const getDetails = (type, id, name, version, config, configURL, runtime =
         interpreters.set(id, details);
         // enable sane defaults when single interpreter *of kind* is used in the page
         // this allows `xxx-*` attributes to refer to such interpreter without `env` around
-        /* c8 ignore start *//* this is tested very well in PyScript */
         if (!interpreters.has(type)) interpreters.set(type, details);
         if (!interpreters.has(runtime)) interpreters.set(runtime, details);
-        /* c8 ignore stopt */
     }
     return interpreters.get(id);
 };
@@ -163,7 +160,6 @@ export const handle = async (script) => {
             if (workerName) workers[workerName].resolve(xworker.ready);
             return;
         }
-        /* c8 ignore stop */
 
         const targetValue = getValue(target, '');
         const details = getDetails(type, id, name, versionValue, configValue);
@@ -182,3 +178,4 @@ export const handle = async (script) => {
         );
     }
 };
+/* c8 ignore stop */

esm/loader.js

@@ -17,10 +17,10 @@ const { all, resolve } = new Proxy(Promise, {
     get: ($, name) => $[name].bind($),
 });
 
+/* c8 ignore start */
 const absoluteURL = (path, base = location.href) =>
     new URL(path, base.replace(/^blob:/, '')).href;
 
-/* c8 ignore start */
 let id = 0;
 const nodeInfo = (node, type) => ({
     id: node.id || (node.id = `${type}-w${id++}`),

esm/script-handler.js

@@ -2,13 +2,13 @@ import { INVALID_CONTENT, INVALID_SRC_ATTR, INVALID_WORKER_ATTR } from '../error
 
 import { dedent, unescape } from '../utils.js';
 
+/* c8 ignore start */ // tested via integration
 const hasCommentsOnly = text => !text
     .replace(/\/\*[\s\S]*?\*\//g, '')
     .replace(/^\s*(?:\/\/|#).*/gm, '')
     .trim()
 ;
 
-/* c8 ignore start */ // tested via integration
 export default element => {
   const { src, worker } = element.attributes;
   if (worker) {