diff --git a/build/pyodide_bucket.bzl b/build/pyodide_bucket.bzl index 5059bde8d61..39b438548fb 100644 --- a/build/pyodide_bucket.bzl +++ b/build/pyodide_bucket.bzl @@ -5,170 +5,3 @@ PYODIDE_PACKAGE_BUCKET_URL = "https://pyodide-packages.runtime-playground.worker PYODIDE_GITHUB_RELEASE_URL = "https://github.com/cloudflare/pyodide-build-scripts/releases/download/20240513.2/" PYODIDE_PACKAGES_TAR_ZIP_SHA256 = "b71d4c3cee3b6bd12969a788545f4159fb1eb984a7ca5de2493c4fa8479beeec" PYODIDE_ALL_WHEELS_ZIP_SHA256 = "c17feb45fdcb4b41eab9c719e69c9e062a8fc88344fcb6bbd7de0de92c3ae660" - -# IMPORTANT: when updating this file in git, check the diff to make sure none of the imports below are being removed unexpectedly -PYODIDE_IMPORTS_TO_TEST = { - "aiohttp": [ - "aiohttp", - ], - "aiosignal": [ - "aiosignal", - ], - "annotated-types": [ - "annotated_types", - ], - "anyio": [ - "anyio", - ], - "async-timeout": [ - "async_timeout", - ], - "attrs": [ - "attr", - "attrs", - ], - "certifi": [ - "certifi", - ], - "charset-normalizer": [ - "charset_normalizer", - ], - "distro": [ - "distro", - ], - "fastapi": [ - "fastapi", - ], - "frozenlist": [ - "frozenlist", - ], - "h11": [ - "h11", - ], - "hashlib": [ - "_hashlib", - ], - "httpcore": [ - "httpcore", - ], - "httpx": [ - "httpx", - ], - "idna": [ - "idna", - ], - "jsonpatch": [ - "jsonpatch", - ], - "jsonpointer": [ - "jsonpointer", - ], - "langchain": [ - "langchain", - ], - "langchain-core": [ - "langchain_core", - "langchain_core.callbacks", - "langchain_core.language_models.llms", - "langchain_core.output_parsers", - "langchain_core.prompts", - ], - "langchain_openai": [ - "langchain_openai", - "langchain_openai.chat_models.base", - ], - "langsmith": [ - "langsmith", - "langsmith.client", - ], - "lzma": [ - "_lzma", - "lzma", - ], - "micropip": [ - "micropip", - ], - "multidict": [ - "multidict", - ], - "numpy": [ - "numpy", - ], - "openai": [ - "openai", - ], - "packaging": [ - "packaging", - ], - "pydantic": [ - "pydantic", - ], - "pydantic_core": [ - "pydantic_core", - ], - "pydecimal": [ - "_pydecimal", - ], - "pydoc_data": [ - "pydoc_data", - ], - "pyyaml": [ - "_yaml", - "yaml", - ], - "regex": [ - "regex", - ], - "requests": [ - "requests", - ], - "six": [ - "six", - ], - "sniffio": [ - "sniffio", - ], - "sqlite3": [ - "_sqlite3", - "sqlite3", - ], - "ssl": [ - "_ssl", - "ssl", - ], - "starlette": [ - "starlette", - "starlette.applications", - "starlette.authentication", - "starlette.background", - "starlette.concurrency", - "starlette.config", - "starlette.convertors", - "starlette.datastructures", - "starlette.endpoints", - "starlette.exceptions", - "starlette.formparsers", - "starlette.middleware", - "starlette.middleware.base", - "starlette.requests", - "starlette.responses", - "starlette.routing", - "starlette.schemas", - ], - "tenacity": [ - "tenacity", - ], - "tiktoken": [ - "tiktoken", - "tiktoken_ext", - ], - "typing-extensions": [ - "typing_extensions", - ], - "urllib3": [ - "urllib3", - ], - "yarl": [ - "yarl", - ], -} diff --git a/build/python/packages_20240829_4.bzl b/build/python/packages_20240829_4.bzl new file mode 100644 index 00000000000..6ba6779e0c1 --- /dev/null +++ b/build/python/packages_20240829_4.bzl @@ -0,0 +1,168 @@ +# This file is automatically generated by the Pyodide build script repo +# (https://github.com/cloudflare/pyodide-build-scripts) and should not be manually modified. + +PACKAGES_20240829_4 = { + "aiohttp": [ + "aiohttp", + ], + "aiosignal": [ + "aiosignal", + ], + "annotated-types": [ + "annotated_types", + ], + "anyio": [ + "anyio", + ], + "async-timeout": [ + "async_timeout", + ], + "attrs": [ + "attr", + "attrs", + ], + "certifi": [ + "certifi", + ], + "charset-normalizer": [ + "charset_normalizer", + ], + "distro": [ + "distro", + ], + "fastapi": [ + "fastapi", + ], + "frozenlist": [ + "frozenlist", + ], + "h11": [ + "h11", + ], + "hashlib": [ + "_hashlib", + ], + "httpcore": [ + "httpcore", + ], + "httpx": [ + "httpx", + ], + "idna": [ + "idna", + ], + "jsonpatch": [ + "jsonpatch", + ], + "jsonpointer": [ + "jsonpointer", + ], + "langchain": [ + "langchain", + ], + "langchain-core": [ + "langchain_core", + "langchain_core.callbacks", + "langchain_core.language_models.llms", + "langchain_core.output_parsers", + "langchain_core.prompts", + ], + "langchain_openai": [ + "langchain_openai", + "langchain_openai.chat_models.base", + ], + "langsmith": [ + "langsmith", + "langsmith.client", + ], + "lzma": [ + "_lzma", + "lzma", + ], + "micropip": [ + "micropip", + ], + "multidict": [ + "multidict", + ], + "numpy": [ + "numpy", + ], + "openai": [ + "openai", + ], + "packaging": [ + "packaging", + ], + "pydantic": [ + "pydantic", + ], + "pydantic_core": [ + "pydantic_core", + ], + "pydecimal": [ + "_pydecimal", + ], + "pydoc_data": [ + "pydoc_data", + ], + "pyyaml": [ + "_yaml", + "yaml", + ], + "regex": [ + "regex", + ], + "requests": [ + "requests", + ], + "six": [ + "six", + ], + "sniffio": [ + "sniffio", + ], + "sqlite3": [ + "_sqlite3", + "sqlite3", + ], + "ssl": [ + "_ssl", + "ssl", + ], + "starlette": [ + "starlette", + "starlette.applications", + "starlette.authentication", + "starlette.background", + "starlette.concurrency", + "starlette.config", + "starlette.convertors", + "starlette.datastructures", + "starlette.endpoints", + "starlette.exceptions", + "starlette.formparsers", + "starlette.middleware", + "starlette.middleware.base", + "starlette.requests", + "starlette.responses", + "starlette.routing", + "starlette.schemas", + ], + "tenacity": [ + "tenacity", + ], + "tiktoken": [ + "tiktoken", + "tiktoken_ext", + ], + "typing-extensions": [ + "typing_extensions", + ], + "urllib3": [ + "urllib3", + ], + "yarl": [ + "yarl", + ], +} diff --git a/build/python/packages_20241218.bzl b/build/python/packages_20241218.bzl new file mode 100644 index 00000000000..78ae962910e --- /dev/null +++ b/build/python/packages_20241218.bzl @@ -0,0 +1,225 @@ +# This file is automatically generated by the Pyodide build script repo +# (https://github.com/cloudflare/pyodide-build-scripts) and should not be manually modified. + +PACKAGES_20241218 = { + "PyJWT": [ + "jwt", + ], + "aiohttp": [ + "aiohttp", + ], + "aiohttp-tests": [ + "aiohttp", + ], + "aiosignal": [ + "aiosignal", + ], + "annotated-types": [ + "annotated_types", + ], + "annotated-types-tests": [ + "annotated_types", + ], + "anyio": [ + "anyio", + ], + "async-timeout": [ + "async_timeout", + ], + "attrs": [ + "attr", + "attrs", + ], + "beautifulsoup4": [ + "bs4", + ], + "beautifulsoup4-tests": [ + "bs4", + ], + "certifi": [ + "certifi", + ], + "cffi": [ + "cffi", + ], + "charset-normalizer": [ + "charset_normalizer", + ], + "cryptography": [ + "cryptography", + "cryptography.fernet", + "cryptography.hazmat", + "cryptography.utils", + "cryptography.x509", + ], + "distro": [ + "distro", + ], + "fastapi": [ + "fastapi", + ], + "frozenlist": [ + "frozenlist", + ], + "h11": [ + "h11", + ], + "h11-tests": [ + "h11", + ], + "hashlib": [ + "_hashlib", + ], + "httpcore": [ + "httpcore", + ], + "httpx": [ + "httpx", + ], + "idna": [ + "idna", + ], + "jsonpatch": [ + "jsonpatch", + ], + "jsonpointer": [ + "jsonpointer", + ], + "langchain": [ + "langchain", + ], + "langchain-core": [ + "langchain_core", + "langchain_core.callbacks", + "langchain_core.language_models.llms", + "langchain_core.output_parsers", + "langchain_core.prompts", + ], + "langchain_openai": [ + "langchain_openai", + "langchain_openai.chat_models.base", + ], + "langsmith": [ + "langsmith", + "langsmith.client", + ], + "lzma": [ + "_lzma", + "lzma", + ], + "micropip": [ + "micropip", + ], + "multidict": [ + "multidict", + ], + "numpy": [ + "numpy", + ], + "numpy-tests": [ + "numpy", + ], + "openai": [ + "openai", + ], + "packaging": [ + "packaging", + ], + "pycparser": [ + "pycparser", + ], + "pydantic": [ + "pydantic", + ], + "pydantic_core": [ + "pydantic_core", + ], + "pydecimal": [ + "_pydecimal", + ], + "pydoc_data": [ + "pydoc_data", + ], + "python-multipart": [ + "multipart", + "python_multipart", + ], + "pyyaml": [ + "_yaml", + "yaml", + ], + "regex": [ + "regex", + ], + "regex-tests": [ + "regex", + ], + "requests": [ + "requests", + ], + "six": [ + "six", + ], + "sniffio": [ + "sniffio", + ], + "sniffio-tests": [ + "sniffio", + ], + "soupsieve": [ + "soupsieve", + ], + "sqlalchemy": [ + "sqlalchemy", + ], + "sqlalchemy-tests": [ + "sqlalchemy", + ], + "sqlite3": [ + "_sqlite3", + "sqlite3", + ], + "ssl": [ + "_ssl", + "ssl", + ], + "starlette": [ + "starlette", + "starlette.applications", + "starlette.authentication", + "starlette.background", + "starlette.concurrency", + "starlette.config", + "starlette.converters", + "starlette.datastructures", + "starlette.endpoints", + "starlette.exceptions", + "starlette.formparsers", + "starlette.middleware", + "starlette.middleware.base", + "starlette.requests", + "starlette.responses", + "starlette.routing", + "starlette.schemas", + ], + "tblib": [ + "tblib", + ], + "tenacity": [ + "tenacity", + ], + "tiktoken": [ + "tiktoken", + "tiktoken_ext", + ], + "typing-extensions": [ + "typing_extensions", + ], + "urllib3": [ + "urllib3", + "urllib3.contrib.emscripten", + ], + "yarl": [ + "yarl", + ], +} diff --git a/build/python_metadata.bzl b/build/python_metadata.bzl index 049b7b36d15..8f41f4116e0 100644 --- a/build/python_metadata.bzl +++ b/build/python_metadata.bzl @@ -1,3 +1,6 @@ +load("//:build/python/packages_20240829_4.bzl", "PACKAGES_20240829_4") +load("//:build/python/packages_20241218.bzl", "PACKAGES_20241218") + # The below is a list of pyodide-lock.json files for each package bundle version that we support. # Each of these gets embedded in the workerd and EW binary. # @@ -7,3 +10,40 @@ PYTHON_LOCKFILES = { "20240829.4": "c2d9c67ea55a672b95a3beb8d66bfbe7df736edb4bb657383b263151e7e85ef4", "20241218": "1421e9351baf24ec44d82f78b9ac26e8e0e6595bfe3f626dedb33147bfcd1998", } + +# This is a dictionary mapping a Python version with its packages version. +# +# NOTE: this needs to be kept in sync with compatibility-date.capnp. +PYTHON_VERSION_TO_PACKAGES = { + "0.26.0a2": "20240829.4", + "0.27.1": "20241218", + "development": "20241218", +} + +# This is a dictionary mapping a `packages` date/version (the `packages` field in +# pythonSnapshotRelease) to a list of package names which are in that packages bundle. The list is +# also in the form of a dictionary and maps to the import names for that package which should be +# tested (as some packages may expose more than one module). +# +# IMPORTANT: packages that are present here should never be removed after the package version is +# released to the public. This is so that we don't break workers using those packages. +# +# ORDER MATTERS: the order of the keys in this dictionary matters, older package bundles should come +# first. +PYTHON_IMPORTS_TO_TEST = { + "20240829.4": PACKAGES_20240829_4, + "20241218": PACKAGES_20241218, +} + +# Each new package bundle should contain the same packages as the previous. We verify this +# constraint here. +def verify_no_packages_were_removed(): + package_dates = PYTHON_IMPORTS_TO_TEST.keys() # Assuming dict order is stable in Skylark. + for i in range(0, len(package_dates) - 1): + curr_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i]] + next_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i + 1]] + for pkg in curr_pkgs: + if pkg not in next_pkgs: + fail(pkg + " from packages version ", package_dates[i], " not in ", package_dates[i + 1]) + +verify_no_packages_were_removed() diff --git a/docs/pyodide.md b/docs/pyodide.md index 61cb9aeeecf..9ff7eb0278b 100644 --- a/docs/pyodide.md +++ b/docs/pyodide.md @@ -1,10 +1,14 @@ # Pyodide Package Indices -workerd is linked against a Pyodide lock file, which is located within an R2 bucket. At build time this lock file is fetched and bundled into the binary. (See WORKSPACE and search for `pyodide-lock.json`) +workerd is linked against multiple package lock files, which are located within an R2 bucket. At +build time these lock files are fetched and bundled into the binary. (See WORKSPACE and search for +`pyodide-lock.json`) -If you know where the R2 bucket is (See build/pyodide_bucket.bzl) then the `pyodide-lock.json` file is located inside the root of the R2 directory for the Pyodide package bundle release. +If you know where the R2 bucket is (See build/pyodide_bucket.bzl) then the `pyodide-lock.json` +file is located inside the root of the R2 directory for the Pyodide package bundle release. -This lock file contains some information used by workerd to pull in package requirements, including but not limited to: +This lock file contains some information used by workerd to pull in package requirements, including +but not limited to: - The versions of each package included in the package bundle - The file names and SHA hashes of each package available for download in the bucket @@ -12,4 +16,6 @@ This lock file contains some information used by workerd to pull in package requ ## Generating pyodide_bucket.bzl -We have scripts and GitHub actions set up for building and uploading Pyodide package bundles onto R2. These are available [here](https://github.com/cloudflare/pyodide-build-scripts). Simply follow the instructions on that repo to build a new version of Pyodide or a new package bundle release. +We have scripts and GitHub actions set up for building and uploading Pyodide package bundles onto R2. +These are available [here](https://github.com/cloudflare/pyodide-build-scripts). Simply follow the +instructions on that repo to build a new version of Pyodide or a new package bundle release. diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index a2f066831e6..40b32e2c63d 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -5,7 +5,6 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") load("//:build/capnp_embed.bzl", "capnp_embed") load("//:build/js_file.bzl", "js_file") -load("//:build/pyodide_bucket.bzl", "PYODIDE_PACKAGE_BUCKET_URL") load("//:build/python_metadata.bzl", "PYTHON_LOCKFILES") load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle") @@ -172,16 +171,6 @@ REPLACEMENTS = [ ], ] -PYODIDE_BUCKET_MODULE = json.encode({ - "PYODIDE_PACKAGE_BUCKET_URL": PYODIDE_PACKAGE_BUCKET_URL, -}) - -write_file( - name = "pyodide-bucket.json@rule", - out = "generated/pyodide-bucket.json", - content = [PYODIDE_BUCKET_MODULE], -) - expand_template( name = "pyodide.asm.js@rule", out = "generated/pyodide.asm.js", @@ -255,15 +244,11 @@ wd_ts_bundle( eslintrc_json = "eslint.config.mjs", import_name = "pyodide", internal_data_modules = INTERNAL_DATA_MODULES, - internal_json_modules = [ - "generated/pyodide-bucket.json", - ], internal_modules = INTERNAL_MODULES, js_deps = [ "generated/emscriptenSetup", "pyodide.asm.wasm@rule", "python_stdlib.zip@rule", - "pyodide-bucket.json@rule", ], lint = False, modules = MODULES, @@ -298,7 +283,6 @@ genrule( ":pyodide-internal_generated_emscriptenSetup.js", ":pyodide-internal_generated_pyodide.asm.wasm", ":pyodide-internal_generated_python_stdlib.zip", - ":pyodide-internal_generated_pyodide-bucket.json", ], outs = ["pyodide.capnp.bin"], cmd = " ".join([ diff --git a/src/pyodide/internal/loadPackage.ts b/src/pyodide/internal/loadPackage.ts index d0c599a7d67..e5b893e9d4d 100644 --- a/src/pyodide/internal/loadPackage.ts +++ b/src/pyodide/internal/loadPackage.ts @@ -63,6 +63,9 @@ async function loadBundleFromR2(requirement: string): Promise { // we didn't find it in the disk cache, continue with original fetch const url = new URL(WORKERD_INDEX_URL + filename); const response = await fetch(url); + if (response.status != 200) { + throw new Error('Could not fetch package at url: ' + url); + } const compressed = await response.arrayBuffer(); const decompressed = await decompressArrayBuffer(compressed); diff --git a/src/pyodide/internal/metadata.ts b/src/pyodide/internal/metadata.ts index 662748b5b7d..e41846a55e7 100644 --- a/src/pyodide/internal/metadata.ts +++ b/src/pyodide/internal/metadata.ts @@ -1,5 +1,4 @@ import { default as MetadataReader } from 'pyodide-internal:runtime-generated/metadata'; -import { default as PYODIDE_BUCKET } from 'pyodide-internal:generated/pyodide-bucket.json'; import { default as ArtifactBundler } from 'pyodide-internal:artifacts'; export const IS_WORKERD = MetadataReader.isWorkerd(); @@ -7,11 +6,19 @@ export const IS_TRACING = MetadataReader.isTracing(); export const SHOULD_SNAPSHOT_TO_DISK = MetadataReader.shouldSnapshotToDisk(); export const IS_CREATING_BASELINE_SNAPSHOT = MetadataReader.isCreatingBaselineSnapshot(); -export const WORKERD_INDEX_URL = PYODIDE_BUCKET.PYODIDE_PACKAGE_BUCKET_URL; export const LOAD_WHEELS_FROM_R2: boolean = IS_WORKERD; export const LOAD_WHEELS_FROM_ARTIFACT_BUNDLER = MetadataReader.shouldUsePackagesInArtifactBundler(); export const PACKAGES_VERSION = MetadataReader.getPackagesVersion(); +export const WORKERD_INDEX_URL = + PACKAGES_VERSION == '20240829.4' + ? // TODO: The URL below points at a worker which redirects requests to the public R2 bucket URL + // at pub-45d734c4145d4285b343833ee450ef38.r2.dev. We should remove this worker and point at + // our prod bucket. + 'https://pyodide-packages.runtime-playground.workers.dev/' + + PACKAGES_VERSION + + '/' + : 'https://python-packages.edgeworker.net/' + PACKAGES_VERSION + '/'; export const LOCKFILE: PackageLock = JSON.parse( MetadataReader.getPackagesLock() ); diff --git a/src/pyodide/internal/setupPackages.ts b/src/pyodide/internal/setupPackages.ts index 2ad7dbbc530..9c9580f21c4 100644 --- a/src/pyodide/internal/setupPackages.ts +++ b/src/pyodide/internal/setupPackages.ts @@ -185,7 +185,6 @@ function disabledLoadPackage(): never { function getTransitiveRequirements(): Set { const requirements = REQUIREMENTS.map(canonicalizePackageName); // resolve transitive dependencies of requirements and if IN_WORKERD install them from the cdn. - // TODO(later): use current package's LOCKFILE instead of the global. const packageDatas = recursiveDependencies(LOCKFILE, requirements); return new Set(packageDatas.map(({ name }) => canonicalizePackageName(name))); } diff --git a/src/pyodide/types/pyodide-bucket.d.ts b/src/pyodide/types/pyodide-bucket.d.ts deleted file mode 100644 index 64f4bfb716a..00000000000 --- a/src/pyodide/types/pyodide-bucket.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -interface PYODIDE_BUCKET { - PYODIDE_PACKAGE_BUCKET_URL: string; -} - -declare module 'pyodide-internal:generated/pyodide-bucket.json' { - const bucket: PYODIDE_BUCKET; - export default bucket; -} diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index b4a310495c2..22a7739a760 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -583,7 +583,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { pythonWorkersDevPyodide @58 :Bool $compatEnableFlag("python_workers_development") $pythonSnapshotRelease(pyodide = "dev", pyodideRevision = "dev", - packages = "20240829.4", backport = 0, + packages = "20241218", backport = 0, baselineSnapshotHash = "92859211804cd350f9e14010afad86e584bdd017dc7acfd94709a87f3220afae") $experimental; # Enables Python Workers and uses the bundle from the Pyodide source directory directly. For testing only. diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 6daeb84bef4..21b9c286d89 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -1,5 +1,5 @@ load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("//:build/pyodide_bucket.bzl", "PYODIDE_IMPORTS_TO_TEST") +load("//:build/python_metadata.bzl", "PYTHON_IMPORTS_TO_TEST", "PYTHON_LOCKFILES", "PYTHON_VERSION_TO_PACKAGES") load("//src/workerd/server/tests/python:import_tests.bzl", "gen_import_tests") load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test") @@ -37,21 +37,28 @@ py_wd_test( py_wd_test("seek-metadatafs") -gen_import_tests( - PYODIDE_IMPORTS_TO_TEST, - # TODO: Micropip version mismatch for 0.27.0 - pkg_skip_versions = { - "micropip": [ - "0.27.1", - ], - "langchain-core": [ - "0.27.1", - ], - "langchain_openai": [ - "0.27.1", - ], - "langsmith": [ - "0.27.1", - ], - }, -) +# Each Python version has a different packages bundle. So we generate a separate list of +# import tests for each Python version. We do however verify that newer package bundles contain +# all the packages that are in older bundles (see verify_no_packages_were_removed). +[ + gen_import_tests( + PYTHON_IMPORTS_TO_TEST[package_version], + python_version, + # TODO: Micropip version mismatch for 0.27.0 + pkg_skip_versions = { + "micropip": [ + "0.27.1", + ], + "langchain-core": [ + "0.27.1", + ], + "langchain_openai": [ + "0.27.1", + ], + "langsmith": [ + "0.27.1", + ], + }, + ) + for python_version, package_version in PYTHON_VERSION_TO_PACKAGES.items() +] diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index 15b57692224..54f2ebd69ed 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -32,11 +32,11 @@ def generate_wd_test_file(requirement): return WD_FILE_TEMPLATE.format(requirement, requirement) # to_test is a dictionary from library name to list of imports -def gen_import_tests(to_test, pkg_skip_versions = {}): +def gen_import_tests(to_test, python_version, pkg_skip_versions = {}): for lib in to_test.keys(): prefix = "import/" + lib - worker_py_fname = prefix + "/worker.py" - wd_test_fname = prefix + "/import.wd-test" + worker_py_fname = python_version + "/" + prefix + "/worker.py" + wd_test_fname = python_version + "/" + prefix + "/import.wd-test" write_file( name = worker_py_fname + "@rule", out = worker_py_fname, @@ -52,6 +52,7 @@ def gen_import_tests(to_test, pkg_skip_versions = {}): name = prefix, directory = lib, src = wd_test_fname, + python_flags = [python_version], skip_python_flags = pkg_skip_versions.get(lib, []), args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"], data = [worker_py_fname, "@all_pyodide_wheels//:whls"],