Add support for pip dependencies (#102)

* Allow pre-installing pip packages

* Run blabk

* Iterate

* Iterate

* Iterate

* Fix RECORD

* Run black

* Update comment

Co-authored-by: Jeremy Tuloup <[email protected]>

* Install docs build dependencies from conda-forge

* Update empack dependency

* Use latest jupyterlite-sphinx in docs

* Add some docs about the pip packages support

---------

Co-authored-by: Jeremy Tuloup <[email protected]>

Author: martinRenou

README.md

@@ -66,10 +66,6 @@ You can also pick another name for that environment file (*e.g.* `custom.yml`),
 jupyter lite build --XeusPythonEnv.environment_file=custom.yml
 ```
 
-#### About pip dependencies
-
-It is common to provide `pip` dependencies in a conda environment file, this is currently **not supported** by xeus-python.
-
 ## Contributing
 
 ### Development install

docs/build-environment.yml

@@ -14,7 +14,7 @@ dependencies:
   - yarn
   - jupyterlab >=3.5.3,<3.6
   - jupyterlite-core >=0.1.0,<0.2.0
-  - empack >=3,<4
+  - empack >=3.1.0
   - pip:
-    - jupyterlite-sphinx
+    - jupyterlite-sphinx >=0.9.1
     - ..

docs/conf.py

@@ -1,20 +1,20 @@
 # -*- coding: utf-8 -*-
 
 extensions = [
-    'jupyterlite_sphinx',
-    'myst_parser',
+    "jupyterlite_sphinx",
+    "myst_parser",
 ]
 
 myst_enable_extensions = [
     "linkify",
 ]
 
-master_doc = 'index'
-source_suffix = '.rst'
+master_doc = "index"
+source_suffix = ".rst"
 
-project = 'jupyterlite-xeus-python'
-copyright = 'JupyterLite Team'
-author = 'JupyterLite Team'
+project = "jupyterlite-xeus-python"
+copyright = "JupyterLite Team"
+author = "JupyterLite Team"
 
 exclude_patterns = []
 
@@ -23,8 +23,8 @@
 jupyterlite_dir = "."
 
 html_theme_options = {
-   "logo": {
-      "image_light": "xeus-python.svg",
-      "image_dark": "xeus-python.svg",
-   }
+    "logo": {
+        "image_light": "xeus-python.svg",
+        "image_dark": "xeus-python.svg",
+    }
 }

docs/configuration.md

@@ -13,12 +13,12 @@ Say you want to install `NumPy`, `Matplotlib` and `ipycanvas`, it can be done by
 ```
 name: xeus-python-kernel
 channels:
-- https://repo.mamba.pm/emscripten-forge
-- https://repo.mamba.pm/conda-forge
+  - https://repo.mamba.pm/emscripten-forge
+  - https://repo.mamba.pm/conda-forge
 dependencies:
-- numpy
-- matplotlib
-- ipycanvas
+  - numpy
+  - matplotlib
+  - ipycanvas
 ```
 
 Then you only need to build JupyterLite:
@@ -33,8 +33,8 @@ You can also pick another name for that environment file (*e.g.* `custom.yml`),
 jupyter lite build --XeusPythonEnv.environment_file=custom.yml
 ```
 
-```{note}
-It is common to provide `pip` dependencies in a conda environment file. This is currently **not supported** by xeus-python, but there is a [work-in-progress](https://github.com/jupyterlite/xeus-python-kernel/pull/102) to support it.
+```{warning}
+It is common to provide `pip` dependencies in a conda environment file. This is currently **partially supported** by xeus-python. See "pip packages" section.
 ```
 
 Then those packages are usable directly:
@@ -55,6 +55,31 @@ Then those packages are usable directly:
    plt.show();
 ```
 
+### pip packages
+
+⚠ This feature is experimental. You won't have the same user-experience as when using conda/mamba in a "normal" setup ⚠
+
+`xeus-python` provides a way to install packages with pip.
+
+There are a couple of limitations that you should be aware of:
+- it can only install pure Python packages (Python code + data files)
+- it does not install the package dependencies, you should make sure to install them yourself using conda-forge/emscripten-forge.
+
+For example, if you were to install `ipycanvas` from PyPI, you would need to install the ipycanvas dependencies for it to work (`pillow`, `numpy` and `ipywidgets`):
+
+```
+name: xeus-python-kernel
+channels:
+  - https://repo.mamba.pm/emscripten-forge
+  - https://repo.mamba.pm/conda-forge
+dependencies:
+  - numpy
+  - pillow
+  - ipywidgets
+  - pip:
+    - ipycanvas
+```
+
 ## Advanced Configuration
 
 ```{warning}

docs/environment.yml

@@ -5,4 +5,7 @@ channels:
 dependencies:
   - numpy
   - matplotlib
-  - ipycanvas
+  - pillow
+  - ipywidgets
+  - pip:
+    - ipycanvas

jupyterlite_xeus_python/build.py

@@ -1,11 +1,14 @@
+import csv
 import os
 from copy import copy
 from pathlib import Path
 import requests
 import shutil
 from subprocess import check_call, run, DEVNULL
+from tempfile import TemporaryDirectory
 from typing import List
 from urllib.parse import urlparse
+import sys
 
 import yaml
 
@@ -25,7 +28,9 @@
 MICROMAMBA_COMMAND = shutil.which("micromamba")
 CONDA_COMMAND = shutil.which("conda")
 
-PYTHON_VERSION = "3.10"
+PYTHON_MAJOR = 3
+PYTHON_MINOR = 10
+PYTHON_VERSION = f"{PYTHON_MAJOR}.{PYTHON_MINOR}"
 
 XEUS_PYTHON_VERSION = "0.15.9"
 
@@ -119,6 +124,90 @@ def _create_config(prefix_path):
     os.environ["CONDARC"] = str(prefix_path / ".condarc")
 
 
+def _install_pip_dependencies(prefix_path, dependencies, log=None):
+    # Why is this so damn complicated?
+    # Isn't it easier to download the .whl ourselves? pip is hell
+
+    if log is not None:
+        log.warning(
+            """
+            Installing pip dependencies. This is very much experimental so use this feature at your own risks.
+            Note that you can only install pure-python packages.
+            pip is being run with the --no-deps option to not pull undesired system-specific dependencies, so please
+            install your package dependencies from emscripten-forge or conda-forge.
+            """
+        )
+
+    # Installing with pip in another prefix that has a different Python version IS NOT POSSIBLE
+    # So we need to do this whole mess "manually"
+    pkg_dir = TemporaryDirectory()
+
+    run(
+        [
+            "pip",
+            "install",
+            *dependencies,
+            # Install in a tmp directory while we process it
+            "--target",
+            pkg_dir.name,
+            # Specify the right Python version
+            "--python-version",
+            PYTHON_VERSION,
+            # No dependency installed
+            "--no-deps",
+            "--no-input",
+            "--verbose",
+        ],
+        check=True,
+    )
+
+    # We need to read the RECORD and try to be smart about what goes
+    # under site-packages and what goes where
+    packages_dist_info = Path(pkg_dir.name).glob("*.dist-info")
+
+    for package_dist_info in packages_dist_info:
+        with open(package_dist_info / "RECORD", "r") as record:
+            record_content = record.read()
+            record_csv = csv.reader(record_content.splitlines())
+            all_files = [_file[0] for _file in record_csv]
+
+            non_supported_files = [".so", ".a", ".dylib", ".lib", ".exe" ".dll"]
+
+            # List of tuples: (path: str, inside_site_packages: bool)
+            files = [(_file, not _file.startswith("../../")) for _file in all_files]
+
+            # Why?
+            fixed_record_data = record_content.replace("../../", "../../../")
+
+        # OVERWRITE RECORD file
+        with open(package_dist_info / "RECORD", "w") as record:
+            record.write(fixed_record_data)
+
+        # COPY files under `prefix_path`
+        for _file, inside_site_packages in files:
+            path = Path(_file)
+
+            # FAIL if .so / .a / .dylib / .lib / .exe / .dll
+            if path.suffix in non_supported_files:
+                raise RuntimeError(
+                    "Cannot install binary PyPI package, only pure Python packages are supported"
+                )
+
+            file_path = _file[6:] if not inside_site_packages else _file
+            install_path = (
+                prefix_path
+                if not inside_site_packages
+                else prefix_path / "lib" / f"python{PYTHON_VERSION}" / "site-packages"
+            )
+
+            src_path = Path(pkg_dir.name) / file_path
+            dest_path = install_path / file_path
+
+            os.makedirs(dest_path.parent, exist_ok=True)
+
+            shutil.copy(src_path, dest_path)
+
+
 def build_and_pack_emscripten_env(
     python_version: str = PYTHON_VERSION,
     xeus_python_version: str = XEUS_PYTHON_VERSION,
@@ -130,6 +219,7 @@ def build_and_pack_emscripten_env(
     output_path: str = ".",
     build_worker: bool = False,
     force: bool = False,
+    log=None,
 ):
     """Build a conda environment for the emscripten platform and pack it with empack."""
     channels = copy(CHANNELS)
@@ -146,6 +236,8 @@ def build_and_pack_emscripten_env(
     if packages or xeus_python_version or environment_file:
         bail_early = False
 
+    pip_dependencies = []
+
     # Process environment.yml file
     if environment_file and Path(environment_file).exists():
         bail_early = False
@@ -168,10 +260,7 @@ def build_and_pack_emscripten_env(
                 if isinstance(dependency, str) and dependency not in specs:
                     specs.append(dependency)
                 elif isinstance(dependency, dict) and dependency.get("pip") is not None:
-                    raise RuntimeError(
-                        """Cannot install pip dependencies in the xeus-python Emscripten environment (yet?).
-                        """
-                    )
+                    pip_dependencies = dependency["pip"]
 
     # Bail early if there is nothing to do
     if bail_early and not force:
@@ -192,6 +281,10 @@ def build_and_pack_emscripten_env(
         # Create emscripten env with the given packages
         create_env(env_name, root_prefix, specs, channels)
 
+        # Install pip dependencies
+        if pip_dependencies:
+            _install_pip_dependencies(prefix_path, pip_dependencies, log=log)
+
         pack_kwargs = {}
 
         # Download env filter config
@@ -203,9 +296,7 @@ def build_and_pack_emscripten_env(
                     yaml.safe_load(empack_config_content)
                 )
             else:
-                pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(
-                    empack_config
-                )
+                pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(empack_config)
         else:
             pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(DEFAULT_CONFIG_PATH)
 
@@ -235,13 +326,16 @@ def build_and_pack_emscripten_env(
 
             worker = worker.replace("XEUS_KERNEL_FILE", "'xpython_wasm.js'")
             worker = worker.replace("LANGUAGE_DATA_FILE", "'python_data.js'")
-            worker = worker.replace("importScripts(DATA_FILE);", """
+            worker = worker.replace(
+                "importScripts(DATA_FILE);",
+                """
                 await globalThis.Module.bootstrap_from_empack_packed_environment(
                 `./empack_env_meta.json`, /* packages_json_url */
                 ".",               /* package_tarballs_root_url */
                 false              /* verbose */
             );
-            """)
+            """,
+            )
             with open(Path(output_path) / "worker.ts", "w") as fobj:
                 fobj.write(worker)
 

jupyterlite_xeus_python/env_build_addon.py

@@ -38,7 +38,6 @@ def from_string(self, s):
 
 
 class XeusPythonEnv(FederatedExtensionAddon):
-
     __all__ = ["post_build"]
 
     xeus_python_version = Unicode(XEUS_PYTHON_VERSION).tag(
@@ -86,6 +85,7 @@ def post_build(self, manager):
             environment_file=Path(self.manager.lite_dir) / self.environment_file,
             empack_config=self.empack_config,
             output_path=self.cwd.name,
+            log=self.log,
         )
 
         # Find the federated extensions in the emscripten-env and install them
@@ -99,17 +99,14 @@ def post_build(self, manager):
         dest = self.output_extensions / "@jupyterlite" / "xeus-python-kernel" / "static"
 
         # copy *.tar.gz for all side packages
-        for item in Path(self.cwd.name) .iterdir():
+        for item in Path(self.cwd.name).iterdir():
             if item.suffix == ".gz":
-
                 file = item.name
                 yield dict(
                     name=f"xeus:copy:{file}",
                     actions=[(self.copy_one, [item, dest / file])],
                 )
 
-
-
         for file in [
             "empack_env_meta.json",
             "xpython_wasm.js",

setup.py

@@ -61,7 +61,7 @@
         "traitlets",
         "jupyterlite-core>=0.1.0",
         "requests",
-        "empack>=3,<4",
+        "empack>=3.1,<4",
         "typer",
     ],
     zip_safe=False,

tests/test_xeus_python_env.py

@@ -23,8 +23,12 @@ def test_python_env():
     # Check env
     assert os.path.isdir("/tmp/xeus-python-kernel/envs/xeus-python-kernel")
 
-    assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js")
-    assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm")
+    assert os.path.isfile(
+        "/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.js"
+    )
+    assert os.path.isfile(
+        "/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm"
+    )
 
     # Check empack output
     assert os.path.isfile(Path(addon.cwd.name) / "empack_env_meta.json")
@@ -46,8 +50,12 @@ def test_python_env_from_file_1():
     # Check env
     assert os.path.isdir("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1")
 
-    assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.js")
-    assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.wasm")
+    assert os.path.isfile(
+        "/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.js"
+    )
+    assert os.path.isfile(
+        "/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.wasm"
+    )
 
     # Check empack output
     assert os.path.isfile(Path(addon.cwd.name) / "empack_env_meta.json")