Cache pyodide modules (#1113)

Refactored the `PyodideWorker` so that `pyodide` is not completely
reloaded on each code run, but rather the globals representing
user-defined variables and user-imported modules are cleared. This
allows the modules that have already been loaded into `pyodide` to be
cached, massively reducing the latency on subsequent code runs.

Author: loiswells97

CHANGELOG.md

@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Tests for running simple programs in `pyodide` and `skulpt` (#1100)
 - Fall back to `skulpt` if the host is not `crossOriginIsolated` (#1107)
 - `Pyodide` `seaborn` support (#1106)
+- `Pyodide` module caching (#1113)
 
 ### Changed
 

cypress/e2e/spec-wc-pyodide.cy.js

@@ -143,4 +143,43 @@ describe("Running the code with pyodide", () => {
         "ModuleNotFoundError: No module named 'i_do_not_exist' on line 1 of main.py",
       );
   });
+
+  it("clears user-defined variables between code runs", () => {
+    runCode("a = 1\nprint(a)");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".pythonrunner-console-output-line")
+      .should("contain", "1");
+    runCode("print(a)");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".error-message__content")
+      .should("contain", "NameError: name 'a' is not defined");
+  });
+
+  it("clears user-defined functions between code runs", () => {
+    runCode("def my_function():\n\treturn 1\nprint(my_function())");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".pythonrunner-console-output-line")
+      .should("contain", "1");
+    runCode("print(my_function())");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".error-message__content")
+      .should("contain", "NameError: name 'my_function' is not defined");
+  });
+
+  it("clears user-imported modules between code runs", () => {
+    runCode("import math\nprint(math.floor(math.pi))");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".pythonrunner-console-output-line")
+      .should("contain", "3");
+    runCode("print(math.floor(math.pi))");
+    cy.get("editor-wc")
+      .shadow()
+      .find(".error-message__content")
+      .should("contain", "NameError: name 'math' is not defined");
+  });
 });

src/PyodideWorker.js

@@ -65,16 +65,7 @@ const PyodideWorker = () => {
     await pyodide.runPythonAsync(`
     import pyodide_http
     pyodide_http.patch_all()
-
-    old_input = input
-
-    def patched_input(prompt=False):
-        if (prompt):
-            print(prompt)
-        return old_input()
-
-    __builtins__.input = patched_input
-    `);
+  `);
 
     try {
       await withSupportForPackages(python, async () => {
@@ -87,7 +78,7 @@ const PyodideWorker = () => {
       postMessage({ method: "handleError", ...parsePythonError(error) });
     }
 
-    await reloadPyodideToClearState();
+    await clearPyodideData();
   };
 
   const checkIfStopped = () => {
@@ -241,12 +232,12 @@ const PyodideWorker = () => {
         pyodide.runPython(`
         import js
 
-        class DummyDocument:
+        class __DummyDocument__:
             def __init__(self, *args, **kwargs) -> None:
                 return
             def __getattr__(self, __name: str):
-                return DummyDocument
-        js.document = DummyDocument()
+                return __DummyDocument__
+        js.document = __DummyDocument__()
         `);
         await pyodide.loadPackage("matplotlib")?.catch(() => {});
         let pyodidePackage;
@@ -334,7 +325,18 @@ const PyodideWorker = () => {
     },
   };
 
-  const reloadPyodideToClearState = async () => {
+  const clearPyodideData = async () => {
+    postMessage({ method: "handleLoading" });
+    await pyodide.runPythonAsync(`
+        # Clear all user-defined variables and modules
+        for name in dir():
+            if not name.startswith('_'):
+                del globals()[name]
+      `);
+    postMessage({ method: "handleLoaded", stdinBuffer, interruptBuffer });
+  };
+
+  const initialisePyodide = async () => {
     postMessage({ method: "handleLoading" });
 
     pyodidePromise = loadPyodide({
@@ -346,6 +348,15 @@ const PyodideWorker = () => {
 
     pyodide = await pyodidePromise;
 
+    await pyodide.runPythonAsync(`
+    __old_input__ = input
+    def __patched_input__(prompt=False):
+        if (prompt):
+            print(prompt)
+        return __old_input__()
+    __builtins__.input = __patched_input__
+    `);
+
     if (supportsAllFeatures) {
       stdinBuffer =
         stdinBuffer || new Int32Array(new SharedArrayBuffer(1024 * 1024)); // 1 MiB
@@ -409,7 +420,7 @@ const PyodideWorker = () => {
     return { file, line, mistake, type, info };
   };
 
-  reloadPyodideToClearState();
+  initialisePyodide();
 
   return {
     postMessage,

src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js

@@ -96,7 +96,7 @@ describe("PyodideWorker", () => {
       },
     });
     expect(pyodide.runPythonAsync).toHaveBeenCalledWith(
-      expect.stringMatching(/__builtins__.input = patched_input/),
+      expect.stringMatching(/__builtins__.input = __patched_input__/),
     );
   });
 
@@ -190,17 +190,18 @@ describe("PyodideWorker", () => {
     });
   });
 
-  test("it reloads pyodide after running the code", async () => {
-    global.loadPyodide.mockClear();
+  test("it clears the pyodide variables after running the code", async () => {
     await worker.onmessage({
       data: {
         method: "runPython",
         python: "print('hello')",
       },
     });
-    await waitFor(() => {
-      expect(global.loadPyodide).toHaveBeenCalled();
-    });
+    await waitFor(() =>
+      expect(pyodide.runPythonAsync).toHaveBeenCalledWith(
+        expect.stringContaining("del globals()[name]"),
+      ),
+    );
   });
 
   test("it handles stopping by notifying component of an error", async () => {