Skip to content

Commit 1bf985e

Browse files
minrkminrk
authored
Merge pull request #536 from minrk/cluster-manager-nbextension
jupyterlab extension based on dask-labextension, new ClusterManager
2 parents 57288d5 + 947be39 commit 1bf985e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+11663
-387
lines changed

‎.eslintignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
node_modules
2+
dist
3+
coverage
4+
**/*.d.ts
5+
tests

‎.eslintrc.js

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
module.exports = {
2+
env: {
3+
browser: true,
4+
es6: true,
5+
commonjs: true,
6+
node: true,
7+
},
8+
root: true,
9+
extends: [
10+
"eslint:recommended",
11+
"plugin:@typescript-eslint/eslint-recommended",
12+
"plugin:@typescript-eslint/recommended",
13+
"prettier/@typescript-eslint",
14+
"plugin:react/recommended",
15+
],
16+
parser: "@typescript-eslint/parser",
17+
parserOptions: {
18+
project: "tsconfig.eslint.json",
19+
},
20+
plugins: ["@typescript-eslint"],
21+
rules: {
22+
"@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }],
23+
"@typescript-eslint/naming-convention": [
24+
"error",
25+
{
26+
selector: "interface",
27+
format: ["PascalCase"],
28+
custom: {
29+
regex: "^I[A-Z]",
30+
match: true,
31+
},
32+
},
33+
],
34+
"@typescript-eslint/no-unused-vars": ["warn", { args: "none" }],
35+
"@typescript-eslint/no-use-before-define": "off",
36+
"@typescript-eslint/camelcase": "off",
37+
"@typescript-eslint/no-explicit-any": "off",
38+
"@typescript-eslint/no-non-null-assertion": "off",
39+
"@typescript-eslint/no-namespace": "off",
40+
"@typescript-eslint/interface-name-prefix": "off",
41+
"@typescript-eslint/explicit-function-return-type": "off",
42+
"@typescript-eslint/ban-ts-comment": ["warn", { "ts-ignore": true }],
43+
"@typescript-eslint/ban-types": "warn",
44+
"@typescript-eslint/no-non-null-asserted-optional-chain": "warn",
45+
"@typescript-eslint/no-var-requires": "off",
46+
"@typescript-eslint/no-empty-interface": "off",
47+
"@typescript-eslint/triple-slash-reference": "warn",
48+
"@typescript-eslint/no-inferrable-types": "off",
49+
"no-inner-declarations": "off",
50+
"no-prototype-builtins": "off",
51+
"no-control-regex": "warn",
52+
"no-undef": "warn",
53+
"no-case-declarations": "warn",
54+
"no-useless-escape": "off",
55+
"prefer-const": "off",
56+
"react/prop-types": "warn",
57+
},
58+
settings: {
59+
react: {
60+
version: "detect",
61+
},
62+
},
63+
};

‎.github/workflows/test.yml

+10-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ env:
1111
jobs:
1212
test:
1313
runs-on: ${{ matrix.runs_on || 'ubuntu-20.04' }}
14-
timeout-minutes: 15
14+
timeout-minutes: 20
1515

1616
strategy:
1717
# Keep running even if one variation of the job fail
@@ -43,6 +43,15 @@ jobs:
4343
~/conda
4444
key: conda
4545

46+
- name: Cache node_modules
47+
uses: actions/cache@v2
48+
with:
49+
path: |
50+
node_modules
51+
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
52+
restore-keys: |
53+
${{ runner.os }}-yarn-
54+
4655
- name: Set environment variables
4756
if: matrix.controller_ip
4857
run: |

‎.gitignore

+6
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,9 @@ __pycache__
2525
htmlcov
2626
id_*sa
2727
.vscode
28+
29+
node_modules
30+
lib
31+
32+
ipyparallel/labextension
33+
tsconfig.tsbuildinfo

‎.pre-commit-config.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,18 @@ repos:
2222
- id: check-case-conflict
2323
- id: check-executables-have-shebangs
2424
- id: requirements-txt-fixer
25+
- repo: https://github.com/pre-commit/mirrors-eslint
26+
rev: v7.14.0
27+
hooks:
28+
- id: eslint
29+
files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
30+
exclude: ipyparallel/nbextension/.*
31+
types: [file]
32+
additional_dependencies:
33+
- "@typescript-eslint/[email protected]"
34+
- "@typescript-eslint/[email protected]"
35+
- eslint@^6.0.0
36+
- [email protected]
37+
- [email protected]
38+
- [email protected]
39+
- [email protected]

‎.prettierignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
node_modules
2+
docs/build
3+
htmlcov
4+
ipyparallel/labextension
5+
**/node_modules
6+
**/lib
7+
**/package.json

‎CONTRIBUTING.md

+42-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,44 @@
11
# Contributing
22

3-
We follow the [IPython Contributing Guide](https://github.com/ipython/ipython/blob/master/CONTRIBUTING.md).
3+
We follow the [Jupyter Contributing Guide](https://jupyter.readthedocs.io/en/latest/contributing/content-contributor.html).
4+
5+
Make sure to follow the [Jupyter Code of Conduct](https://jupyter.org/conduct/).
6+
7+
## Development install
8+
9+
A basic development install of IPython parallel
10+
is the same as almost all Python packages:
11+
12+
```bash
13+
pip install -e .
14+
```
15+
16+
To enable the server extension from a development install:
17+
18+
```bash
19+
# for jupyterlab
20+
jupyter server extension enable --sys-prefix ipyparallel
21+
# for classic notebook
22+
jupyter serverextension enable --sys-prefix ipyparallel
23+
```
24+
25+
As described in the [JupyterLab documentation](https://jupyterlab.readthedocs.io/en/stable/extension/extension_dev.html#developing-a-prebuilt-extension)
26+
for a development install of the jupyterlab extension you can run the following in this directory:
27+
28+
```bash
29+
jlpm # Install npm package dependencies
30+
jlpm build # Compile the TypeScript sources to Javascript
31+
jupyter labextension develop . --overwrite # Install the current directory as an extension
32+
```
33+
34+
If you are working on the lab extension,
35+
you can run jupyterlab in dev mode to always rebuild and reload your extensions.
36+
In two terminals, run:
37+
38+
```bash
39+
[term 1] $ jlpm watch
40+
[term 2] $ jupyter lab --extensions-in-dev-mode
41+
```
42+
43+
You should then be able to refresh the JupyterLab page
44+
and it will pick up the changes to the extension as you work.

‎MANIFEST.in

+11
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,18 @@ include COPYING.md
22
include CONTRIBUTING.md
33
include README.md
44

5+
# enable-at-install config
56
graft etc
67

8+
# lab extension
9+
graft lab
10+
prune lab/lib
11+
include install.json
12+
include package*.json
13+
include tsconfig*.json
14+
graft ipyparallel/labextension
15+
16+
717
# Documentation
818
graft docs
919

@@ -19,3 +29,4 @@ global-exclude *.pyc
1929
global-exclude *.pyo
2030
global-exclude .git
2131
global-exclude .ipynb_checkpoints
32+
global-exclude .DS_Store

‎README.md

+11-26
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,21 @@
11
# Interactive Parallel Computing with IPython
22

3-
ipyparallel is the new home of IPython.parallel. ipyparallel is a Python package and collection of CLI scripts for controlling clusters for Jupyter.
3+
IPython Parallel (`ipyparallel`) is a Python package and collection of CLI scripts for controlling clusters of IPython processes, built on the Jupyter protocol.
44

5-
ipyparallel contains the following CLI scripts:
5+
IPython Parallel provides the following commands:
66

7-
- ipcluster - start/stop a cluster
8-
- ipcontroller - start a scheduler
7+
- ipcluster - start/stop/list clusters
8+
- ipcontroller - start a controller
99
- ipengine - start an engine
1010

1111
## Install
1212

13-
Install ipyparallel:
13+
Install IPython Parallel:
1414

1515
pip install ipyparallel
1616

17-
To enable the `IPython Clusters` tab in Jupyter Notebook:
18-
19-
ipcluster nbextension enable
20-
21-
To disable it again:
22-
23-
ipcluster nbextension disable
24-
25-
See the [documentation on configuring the notebook server](https://jupyter-notebook.readthedocs.io/en/latest/public_server.html)
26-
to find your config or setup your initial `jupyter_notebook_config.py`.
27-
28-
### JupyterHub Install
29-
30-
To install for all users on JupyterHub, as root:
31-
32-
jupyter nbextension install --sys-prefix --py ipyparallel
33-
jupyter nbextension enable --sys-prefix --py ipyparallel
34-
jupyter serverextension enable --sys-prefix --py ipyparallel
17+
This will install and enable the IPython Parallel extensions
18+
for Jupyter Notebook and (as of 7.0) Jupyter Lab 3.0.
3519

3620
## Run
3721

@@ -45,9 +29,10 @@ Use it from Python:
4529
import os
4630
import ipyparallel as ipp
4731

48-
rc = ipp.Client()
49-
ar = rc[:].apply_async(os.getpid)
50-
pid_map = ar.get_dict()
32+
cluster = ipp.Cluster(n=4)
33+
with cluster as rc:
34+
ar = rc[:].apply_async(os.getpid)
35+
pid_map = ar.get_dict()
5136
```
5237

5338
See [the docs](https://ipyparallel.readthedocs.io) for more info.

‎docs/environment.yml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: ipyparallel-docs
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- python=3.8
6+
- pip
7+
8+
- sphinx>=1.8
9+
- sphinx-copybutton
10+
- pydata-sphinx-theme
11+
- nbsphinx
12+
- myst-parser
13+
14+
- matplotlib
15+
# nodejs needed for extension build
16+
- nodejs

‎etc/ipyparallel-serverextension.json ‎etc/jupyter/jupyter_notebook_config.d/ipyparallel.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"NotebookApp": {
33
"nbserver_extensions": {
4-
"ipyparallel.nbextension": true
4+
"ipyparallel": true
55
}
66
}
77
}

‎etc/jupyter/jupyter_server_config.d/ipyparallel.json

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"ServerApp": {
3+
"jpserver_extensions": {
4+
"ipyparallel": true
5+
}
6+
}
7+
}

‎etc/ipyparallel-nbextension.json ‎etc/jupyter/nbconfig/tree.d/ipyparallel.json

File renamed without changes.

‎install.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"packageManager": "python",
3+
"packageName": "ipyparallel",
4+
"uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package ipyparallel"
5+
}

‎ipyparallel/__init__.py

+27-2
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def bind_kernel(**kwargs):
5050
else:
5151
return app.bind_kernel(**kwargs)
5252

53-
raise RuntimeError("bind_kernel be called from an IPEngine instance")
53+
raise RuntimeError("bind_kernel must be called from an IPEngine instance")
5454

5555

5656
def register_joblib_backend(name='ipyparallel', make_default=False):
@@ -62,7 +62,7 @@ def register_joblib_backend(name='ipyparallel', make_default=False):
6262

6363
# nbextension installation (requires notebook ≥ 4.2)
6464
def _jupyter_server_extension_paths():
65-
return [{'module': 'ipyparallel.nbextension'}]
65+
return [{'module': 'ipyparallel'}]
6666

6767

6868
def _jupyter_nbextension_paths():
@@ -74,3 +74,28 @@ def _jupyter_nbextension_paths():
7474
'require': 'ipyparallel/main',
7575
}
7676
]
77+
78+
79+
def _jupyter_labextension_paths():
80+
return [
81+
{
82+
"src": "labextension",
83+
"dest": "ipyparallel-labextension",
84+
}
85+
]
86+
87+
88+
def _load_jupyter_server_extension(app):
89+
"""Load the server extension"""
90+
# localte the appropriate APIHandler base class before importing our handler classes
91+
from .nbextension.base import get_api_handler
92+
93+
get_api_handler(app)
94+
95+
from .nbextension.handlers import load_jupyter_server_extension
96+
97+
return load_jupyter_server_extension(app)
98+
99+
100+
# backward-compat
101+
load_jupyter_server_extension = _load_jupyter_server_extension

‎ipyparallel/apps/baseapp.py

-3
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,6 @@ def _cluster_id_changed(self, change):
125125
else:
126126
self.name = self.__class__.name
127127

128-
def _config_files_default(self):
129-
return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
130-
131128
loop = Instance(IOLoop)
132129

133130
def _loop_default(self):

0 commit comments

Comments
 (0)