Skip to content

Commit 080a3e4

Browse files
sayerhssayerhs
committed
Conda python environment support in exawind-builder
- Add pySTK to exawind-builder
1 parent e102135 commit 080a3e4

11 files changed

+357
-1
lines changed

‎codes/pystk.bash

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
3+
_EXAWIND_PROJECT_CMAKE_RMEXTRA_=(
4+
_skbuild
5+
pySTK.egg-info
6+
)
7+
8+
exawind_proj_env ()
9+
{
10+
echo "==> Initializing python environment for pySTK"
11+
exawind_py_env
12+
13+
echo "==> Loading dependencies for pySTK ... "
14+
exawind_load_deps trilinos
15+
}
16+
17+
exawind_cmake_base ()
18+
{
19+
echo "WARN: Use py_build instead"
20+
exawind_py_build "$@"
21+
}
22+
23+
exawind_make ()
24+
{
25+
echo "WARN: Use py_build instead"
26+
exawind_py_build "$@"
27+
}
28+
29+
exawind_py_build ()
30+
{
31+
python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH=${TRILINOS_ROOT_DIR} "$@"
32+
}
33+
34+
exawind_py_install ()
35+
{
36+
pip install -e .
37+
}
38+
39+
exawind_default_cmd ()
40+
{
41+
exawind_py_build "$@" && exawind_py_install
42+
}

‎core.bash

+4-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ __EXAWIND_CORE_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
44

55
source ${__EXAWIND_CORE_DIR}/src/environment.bash
66
source ${__EXAWIND_CORE_DIR}/src/builder.bash
7+
source ${__EXAWIND_CORE_DIR}/src/python-env.bash
78

89
exawind_help ()
910
{
@@ -21,6 +22,8 @@ Available tasks:
2122
make - compile the code
2223
ctest - run tests (if available)
2324
run - run arbitrary command using the environment used to compile the code
25+
py_build - build a python library
26+
py_install - install a python library (in development mode)
2427
EOF
2528
}
2629

@@ -34,7 +37,7 @@ exawind_save_func ()
3437
exawind_main ()
3538
{
3639
if [ "$#" == "0" ] ; then
37-
exawind_env && exawind_proj_env && exawind_cmake && exawind_make
40+
exawind_env && exawind_proj_env && exawind_default_cmd
3841
else
3942
subcmd=$1
4043

‎create-pyenv.sh

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env bash
2+
3+
# Create a new python environment based on inputs
4+
5+
EXAWIND_SRCDIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
6+
7+
exw_show_help ()
8+
{
9+
cat <<EOF
10+
$(basename ${BASH_SOURCE[0]}) [options] [output_file_name]
11+
12+
Initialize the ExaWind python environment and create a script to source it.
13+
By default, it will create a file called exawind-env-python-$COMPILER.sh
14+
15+
Options:
16+
-h - Show help message and exit
17+
-s <system> - Select system profile (spack, ornl-summit, etc.)
18+
-c <compiler> - Select compiler type (gcc, clang, intel)
19+
-n <name> - Name of the custom python environment (default: exawind)
20+
-f <file> - Conda environment.yml file
21+
-r <env_root> - Absolute path to the root directory for Conda
22+
23+
EOF
24+
}
25+
26+
check_inputs ()
27+
{
28+
local dirname=${EXAWIND_SRCDIR}/$1
29+
local value=$2
30+
local option=$3
31+
32+
local tgt_file=${dirname}/${value}.bash
33+
if [ ! -f ${tgt_file} ] ; then
34+
echo "Invalid value provided for ${option} = ${value}. Valid options are: "
35+
for fname in $(ls ${dirname}); do
36+
echo " - $(basename -s .bash $fname)"
37+
done
38+
err_stat=1
39+
fi
40+
}
41+
42+
main ()
43+
{
44+
# Define defaults
45+
export EXAWIND_SYSTEM=${EXAWIND_SYSTEM:-spack}
46+
export EXAWIND_COMPILER=${EXAWIND_COMPILER:-gcc}
47+
export EXAWIND_PROJECT_DIR=${EXAWIND_PROJECT_DIR:-$(dirname ${EXAWIND_SRCDIR})}
48+
local env_name=""
49+
local env_file=""
50+
51+
# Parse user options
52+
OPTIND=1
53+
while getopts ":s:c:n:f:r:h" opt ; do
54+
case "$opt" in
55+
h)
56+
exw_show_help
57+
exit 0
58+
;;
59+
s)
60+
export EXAWIND_SYSTEM=$OPTARG
61+
;;
62+
c)
63+
export EXAWIND_COMPILER=$OPTARG
64+
;;
65+
n)
66+
env_name=$OPTARG
67+
;;
68+
f)
69+
env_file=$OPTARG
70+
;;
71+
r)
72+
export CONDA_ROOT_DIR=$OPTARG
73+
;;
74+
\?)
75+
echo "ERROR: Invalid argument provided"
76+
exw_show_help
77+
exit 1
78+
;;
79+
esac
80+
done
81+
shift $((OPTIND-1))
82+
[ "$1" == "--" ] && shift
83+
84+
err_stat=0
85+
check_inputs envs ${EXAWIND_SYSTEM} "system"
86+
if [[ err_stat -ne 0 ]] ; then
87+
echo "Invalid system encountered, exiting now"
88+
exit 1
89+
fi
90+
91+
# Source files
92+
source ${EXAWIND_SRCDIR}/src/environment.bash
93+
source ${EXAWIND_SRCDIR}/envs/${EXAWIND_SYSTEM}.bash
94+
source ${EXAWIND_SRCDIR}/src/python-env.bash
95+
96+
exawind_load_user_configs
97+
98+
# Override configuration variables with command line variables
99+
if [ ! -z ${env_name} ] ; then
100+
export EXAWIND_CONDA_ENV=${env_name}
101+
fi
102+
if [ ! -z ${env_file} ] ; then
103+
export EXAWIND_CONDA_ENV_SPEC=${env_file}
104+
fi
105+
106+
exawind_env
107+
# exawind_py_conda_env_create
108+
109+
output_file=${1:-${EXAWIND_PROJECT_DIR}/scripts/exawind-env-python-${EXAWIND_COMPILER}.sh}
110+
tmpl_file=${EXAWIND_SRCDIR}/etc/pyenv_tmpl.bash
111+
112+
sed -e "s#%%SRCDIR%%#${EXAWIND_SRCDIR}#g;s#%%COMPILER%%#${EXAWIND_COMPILER}#g;s#%%SYSTEM%%#${EXAWIND_SYSTEM}#g;s#%%CONDA_ROOT%%#${CONDA_ROOT_DIR}#;s#%%CONDA_ENV%%#${EXAWIND_CONDA_ENV}#" $tmpl_file > $output_file
113+
chmod a+x $output_file
114+
115+
echo "==> Python environment script ${output_file} created succesfully"
116+
}
117+
118+
main "$@"

‎doc/source/installation.rst

+22
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,28 @@ Nalu-Wind.
9292
you have already executed bootstrap and forgot to add the ``-n`` flag, then
9393
use :ref:`get-ninja` to install Ninja for your use.
9494

95+
Setting up custom ExaWind python environment
96+
--------------------------------------------
97+
98+
``exawind-builder`` now supports building certain Python packages (e.g., `pySTK
99+
<https://sayerhs.github.io/pystk/index.html>`_. To enable this capability,
100+
you'll need to set up a custom virtual environment with the necessary python
101+
modules. Currently, ``exawind-builder`` only supports the `Conda
102+
<https://docs.conda.io/en/latest/index.html>`_ python package manager. To enable this capability:
103+
104+
1. Install `Conda <https://docs.conda.io/en/latest/miniconda.html>`_ if you
105+
don't have an existing conda installation.
106+
107+
2. Create a new virtual environment using the `create-pyenv.sh` utility
108+
109+
.. code-block:: console
110+
111+
cd ${EXAWIND_PROJECT_DIR}
112+
./exawind-builder/create-pyenv.sh -s <system> -c <compiler> -r ${CONDA_ROOT_DIR}
113+
114+
Upon successful installation, this creates a new virtual environment ``exawind``
115+
with all the necessary Python modules to build and use ExaWind python libraries.
116+
95117
.. _homebrew-setup:
96118

97119
Initial Homebrew Setup for Mac OS-X Users

‎doc/source/introduction.rst

+1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ configuration variables availble to customize configuration of each project.
7070
Nalu Wind Utilities https://github.com/exawind/wind-utils.git
7171
TIOGA https://github.com/jsitaraman/tioga.git
7272
TIOGA Utilities https://github.com/sayerhs/tioga_utils.git
73+
pySTK https://github.com/sayerhs/pystk.git
7374
HYPRE https://github.com/LLNL/hypre.git
7475
hypre-mini-app https://github.com/exawind/hypre-mini-app.git
7576
ArborX https://github.com/arborx/ArborX.git

‎etc/pyenv_tmpl.bash

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
#
3+
# ExaWind environment source script for system: %%SYSTEM%%
4+
#
5+
# Autogenerated by exawind-builder
6+
#
7+
# 1. See https://exawind-builder.readthedocs.io for documentation
8+
# 2. Use create-pyenv.sh to regenerate this script
9+
#
10+
11+
#
12+
# Setup variables used by functions
13+
#
14+
export EXAWIND_SRCDIR=%%SRCDIR%%
15+
export EXAWIND_SYSTEM=%%SYSTEM%%
16+
export EXAWIND_COMPILER=%%COMPILER%%
17+
export EXAWIND_CFGFILE=exawind-config
18+
export EXAWIND_PROJECT_DIR=${EXAWIND_PROJECT_DIR:-$(dirname ${EXAWIND_SRCDIR})}
19+
20+
# Source scripts necessary for determining environment
21+
source ${EXAWIND_SRCDIR}/src/environment.bash
22+
source ${EXAWIND_SRCDIR}/envs/${EXAWIND_SYSTEM}.bash
23+
source ${EXAWIND_SRCDIR}/src/python-env.bash
24+
25+
export EXAWIND_INSTALL_DIR=${EXAWIND_INSTALL_DIR:-${EXAWIND_PROJECT_DIR}/install/${EXAWIND_COMPILER}}
26+
27+
# Source any user specific configuration
28+
exawind_load_user_configs
29+
30+
export CONDA_ROOT_DIR=${CONDA_ROOT_DIR:-%%CONDA_ROOT%%}
31+
export EXAWIND_CONDA_ENV=${EXAWIND_CONDA_ENV:-%%CONDA_ENV%%}
32+
33+
# Finally load the environment
34+
exawind_env
35+
exawind_py_env

‎etc/python/conda-env-default.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# -*- mode: yaml -*-
2+
#
3+
# Conda environment for ExaWind project
4+
#
5+
6+
name: exawind
7+
8+
dependencies:
9+
- python>=3.6
10+
- pip
11+
- numpy
12+
- scipy
13+
- cython>=0.25.1
14+
- pandas
15+
- matplotlib
16+
- ipython
17+
- pyyaml
18+
- ruamel_yaml
19+
- ply
20+
- pytz
21+
- jinja2
22+
- pytest
23+
- pylint
24+
- sphinx
25+
- pip:
26+
- -r file:conda-pip-extra.txt

‎etc/python/conda-pip-extra.txt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Extra pip directives for conda environment
2+
--no-binary mpi4py,netCDF4
3+
scikit-build
4+
mpi4py
5+
netCDF4

‎etc/python/exawind-pip-requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Exawind pip requirements file

‎src/builder.bash

+5
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,11 @@ exawind_run ()
152152
eval "$@"
153153
}
154154

155+
exawind_default_cmd ()
156+
{
157+
exawind_cmake "$@" && exawind_make
158+
}
159+
155160
exawind_rpath_dirs ()
156161
{
157162
local rpath_dirs=""

0 commit comments

Comments
 (0)