Move Python SDK from cloudflare.workers to workers.

Author: dom96

WORKSPACE

@@ -91,8 +91,8 @@ http_archive(
 http_archive(
     name = "pyodide",
     build_file = "//:build/BUILD.pyodide",
-    sha256 = "fbda450a64093a8d246c872bb901ee172a57fe594c9f35bba61f36807c73300d",
-    urls = ["https://github.com/pyodide/pyodide/releases/download/0.26.0a2/pyodide-core-0.26.0a2.tar.bz2"],
+    sha256 = "6e45f93c71ed21bff4a06d6d9d8e27e815269dabf8ab34dc400939fd45edc665",
+    urls = ["https://github.com/pyodide/pyodide/releases/download/0.27.1/pyodide-core-0.27.1.tar.bz2"],
 )
 
 http_archive(

src/pyodide/internal/metadata.ts

@@ -10,6 +10,8 @@ export const LOAD_WHEELS_FROM_R2: boolean = IS_WORKERD;
 export const LOAD_WHEELS_FROM_ARTIFACT_BUNDLER =
   MetadataReader.shouldUsePackagesInArtifactBundler();
 export const PACKAGES_VERSION = MetadataReader.getPackagesVersion();
+export const USING_OLDEST_PYODIDE_VERSION =
+  MetadataReader.getPyodideVersion() == '0.26.0a2';
 export const USING_OLDEST_PACKAGES_VERSION = PACKAGES_VERSION === '20240829.4';
 // TODO: pyodide-packages.runtime-playground.workers.dev points at a worker which redirects requests
 // to the public R2 bucket URL at pub-45d734c4145d4285b343833ee450ef38.r2.dev. We should remove

src/pyodide/python-entrypoint-helper.ts

@@ -13,6 +13,7 @@ import {
   LOCKFILE,
   MAIN_MODULE_NAME,
   WORKERD_INDEX_URL,
+  USING_OLDEST_PYODIDE_VERSION,
 } from 'pyodide-internal:metadata';
 import { reportError } from 'pyodide-internal:util';
 import { default as Limiter } from 'pyodide-internal:limiter';
@@ -79,8 +80,13 @@ async function setupPatches(pyodide: Pyodide): Promise<void> {
     pyodide.site_packages = `/lib/python${pymajor}.${pyminor}/site-packages`;
 
     // Inject modules that enable JS features to be used idiomatically from Python.
-    pyodide.FS.mkdir(`${pyodide.site_packages}/cloudflare`);
-    await injectSitePackagesModule(pyodide, 'workers', 'cloudflare/workers');
+    if (USING_OLDEST_PYODIDE_VERSION) {
+      // Inject at cloudflare.workers for backwards compatibility
+      pyodide.FS.mkdir(`${pyodide.site_packages}/cloudflare`);
+      await injectSitePackagesModule(pyodide, 'workers', 'cloudflare/workers');
+    }
+    // The SDK was moved from `cloudflare.workers` to just `workers`.
+    await injectSitePackagesModule(pyodide, 'workers', 'workers');
 
     // Install patches as needed
     if (TRANSITIVE_REQUIREMENTS.has('aiohttp')) {

src/pyodide/types/runtime-generated/metadata.d.ts

@@ -16,6 +16,7 @@ declare namespace MetadataReader {
   const getMemorySnapshotSize: () => number;
   const disposeMemorySnapshot: () => void;
   const shouldUsePackagesInArtifactBundler: () => boolean;
+  const getPyodideVersion: () => string;
   const getPackagesVersion: () => string;
   const getPackagesLock: () => string;
   const read: (index: number, position: number, buffer: Uint8Array) => number;

src/workerd/api/pyodide/pyodide.c++

@@ -524,6 +524,7 @@ jsg::Ref<PyodideMetadataReader> makePyodideMetadataReader(Worker::Reader conf,
     names.finish(),
     contents.finish(),
     requirements.finish(),
+    kj::str(pythonRelease.getPyodide()),
     kj::str(pythonRelease.getPackages()),
     kj::mv(lock),
     true      /* isWorkerd */,

src/workerd/api/pyodide/pyodide.h

@@ -78,6 +78,7 @@ class PyodideMetadataReader: public jsg::Object {
   kj::Array<kj::String> names;
   kj::Array<kj::Array<kj::byte>> contents;
   kj::Array<kj::String> requirements;
+  kj::String pyodideVersion;
   kj::String packagesVersion;
   kj::String packagesLock;
   bool isWorkerdFlag;
@@ -92,6 +93,7 @@ class PyodideMetadataReader: public jsg::Object {
       kj::Array<kj::String> names,
       kj::Array<kj::Array<kj::byte>> contents,
       kj::Array<kj::String> requirements,
+      kj::String pyodideVersion,
       kj::String packagesVersion,
       kj::String packagesLock,
       bool isWorkerd,
@@ -104,6 +106,7 @@ class PyodideMetadataReader: public jsg::Object {
         names(kj::mv(names)),
         contents(kj::mv(contents)),
         requirements(kj::mv(requirements)),
+        pyodideVersion(kj::mv(pyodideVersion)),
         packagesVersion(kj::mv(packagesVersion)),
         packagesLock(kj::mv(packagesLock)),
         isWorkerdFlag(isWorkerd),
@@ -167,6 +170,10 @@ class PyodideMetadataReader: public jsg::Object {
     return usePackagesInArtifactBundler;
   }
 
+  kj::String getPyodideVersion() {
+    return kj::str(pyodideVersion);
+  }
+
   kj::String getPackagesVersion() {
     return kj::str(packagesVersion);
   }
@@ -192,6 +199,7 @@ class PyodideMetadataReader: public jsg::Object {
     JSG_METHOD(disposeMemorySnapshot);
     JSG_METHOD(shouldSnapshotToDisk);
     JSG_METHOD(shouldUsePackagesInArtifactBundler);
+    JSG_METHOD(getPyodideVersion);
     JSG_METHOD(getPackagesVersion);
     JSG_METHOD(getPackagesLock);
     JSG_METHOD(isCreatingBaselineSnapshot);

src/workerd/io/compatibility-date.capnp

@@ -430,7 +430,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   pythonWorkers @43 :Bool
       $compatEnableFlag("python_workers")
       $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
-          packages = "20240829.4", backport = 14,
+          packages = "20240829.4", backport = 17,
           baselineSnapshotHash = "d13ce2f4a0ade2e09047b469874dacf4d071ed3558fec4c26f8d0b99d95f77b5")
       $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
   # Enables Python Workers. Access to this flag is not restricted, instead bundles containing
@@ -684,7 +684,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
       $compatEnableFlag("python_workers_20250116")
       $experimental
       $pythonSnapshotRelease(pyodide = "0.27.1", pyodideRevision = "2025-01-16",
-          packages = "20241218", backport = 2,
+          packages = "20241218", backport = 5,
           baselineSnapshotHash = "TODO");
 
   requestCfOverridesCacheRules @72 :Bool

src/workerd/server/tests/python/BUILD.bazel

@@ -47,6 +47,9 @@ py_wd_test("seek-metadatafs")
         #
         # TODO: The packages below don't currently work in the latest package/python versions. Need
         # to fix them.
+        #
+        # TODO: pydantic_core started to fail with "no module named typing_extensions" after 0.27.1
+        # was rebuilt and backport bumped.
         pkg_skip_versions = {
             "0.27.1": [
                 "micropip",
@@ -63,6 +66,7 @@ py_wd_test("seek-metadatafs")
                 "sqlalchemy",
                 "starlette",
                 "tblib",
+                "pydantic_core",
             ],
         },
     )

src/workerd/server/tests/python/sdk/proxy.js

@@ -10,6 +10,6 @@ export default {
       return env.PYTHON.fetch(req);
     }
 
-    throw new Error('Invalid url: ' + url);
+    throw new Error('Invalid url in proxy.js: ' + url);
   },
 };

src/workerd/server/tests/python/sdk/server.py

@@ -1,4 +1,4 @@
-from cloudflare.workers import Blob, FormData, Response
+from workers import Blob, FormData, Response
 
 
 async def on_fetch(request):

src/workerd/server/tests/python/sdk/worker.py

@@ -2,9 +2,9 @@
 from http import HTTPMethod, HTTPStatus
 
 import js
+from workers import Blob, File, FormData, Response, fetch
 
 import pyodide.http
-from cloudflare.workers import Blob, File, FormData, Response, fetch
 from pyodide.ffi import to_js