run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [],
   "source": [
    "content=`cat app.py` yq eval '.$graph[1].requirements.InitialWorkDirRequirement.listing[0].entry=strenv(content)' workflow.cwl.template > workflow.cwl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cwlVersion: v1.0\n",
      "$namespaces:\n",
      "  s: https://schema.org/\n",
      "s:softwareVersion: 1.0.0\n",
      "schemas:\n",
      "  - http://schema.org/version/9.0/schemaorg-current-http.rdf\n",
      "$graph:\n",
      "  - class: Workflow\n",
      "    id: main\n",
      "    label: Water bodies detection based on NDWI and the otsu threshold\n",
      "    doc: Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item\n",
      "    requirements: []\n",
      "    inputs:\n",
      "      aoi:\n",
      "        label: area of interest\n",
      "        doc: area of interest as a bounding box\n",
      "        type: string\n",
      "      epsg:\n",
      "        label: EPSG code\n",
      "        doc: EPSG code\n",
      "        type: string\n",
      "        default: \"EPSG:4326\"\n",
      "      bands:\n",
      "        label: bands used for the NDWI\n",
      "        doc: bands used for the NDWI\n",
      "        type: string[]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        default: [\"green\", \"nir\"]\n",
      "      item:\n",
      "        doc: Reference to a STAC item\n",
      "        label: STAC item reference\n",
      "        type: string\n",
      "    outputs:\n",
      "      - id: stac_catalog\n",
      "        outputSource:\n",
      "          - node_detect/stac-catalog\n",
      "        type: Directory\n",
      "    steps:\n",
      "      node_detect:\n",
      "        run: \"#detect-water-body\"\n",
      "        in:\n",
      "          item: item\n",
      "          aoi: aoi\n",
      "          epsg: epsg\n",
      "          band: bands\n",
      "        out:\n",
      "          - stac-catalog\n",
      "  - class: CommandLineTool\n",
      "    id: detect-water-body\n",
      "    requirements:\n",
      "      InlineJavascriptRequirement: {}\n",
      "      EnvVarRequirement:\n",
      "        envDef: {}\n",
      "      ResourceRequirement:\n",
      "        coresMax: 1\n",
      "        ramMax: 512\n",
      "      InitialWorkDirRequirement:\n",
      "        listing:\n",
      "          - entryname: app.py\n",
      "            entry: |-\n",
      "              import os\n",
      "              import click\n",
      "              import pystac\n",
      "              import rasterio\n",
      "              from skimage.filters import threshold_otsu\n",
      "              from rasterio.mask import mask\n",
      "              from pyproj import Transformer\n",
      "              from shapely import box\n",
      "              from loguru import logger\n",
      "              import rasterio\n",
      "              import pystac\n",
      "              import shutil\n",
      "              import rio_stac\n",
      "              import numpy as np\n",
      "\n",
      "              np.seterr(divide=\"ignore\", invalid=\"ignore\")\n",
      "\n",
      "\n",
      "              def crop(asset: pystac.Asset, bbox, epsg):\n",
      "                  \"\"\"_summary_\n",
      "\n",
      "                  Args:\n",
      "                      asset (_type_): _description_\n",
      "                      bbox (_type_): _description_\n",
      "                      epsg (_type_): _description_\n",
      "\n",
      "                  Returns:\n",
      "                      _type_: _description_\n",
      "                  \"\"\"\n",
      "                  with rasterio.open(asset.get_absolute_href()) as src:\n",
      "\n",
      "                      transformer = Transformer.from_crs(epsg, src.crs, always_xy=True)\n",
      "\n",
      "                      minx, miny = transformer.transform(bbox[0], bbox[1])\n",
      "                      maxx, maxy = transformer.transform(bbox[2], bbox[3])\n",
      "\n",
      "                      transformed_bbox = box(minx, miny, maxx, maxy)\n",
      "\n",
      "                      logger.info(f\"Crop {asset.get_absolute_href()}\")\n",
      "\n",
      "                      out_image, out_transform = rasterio.mask.mask(\n",
      "                          src, [transformed_bbox], crop=True\n",
      "                      )\n",
      "                      out_meta = src.meta.copy()\n",
      "\n",
      "                      out_meta.update(\n",
      "                          {\n",
      "                              \"height\": out_image.shape[1],\n",
      "                              \"width\": out_image.shape[2],\n",
      "                              \"transform\": out_transform,\n",
      "                          }\n",
      "                      )\n",
      "\n",
      "                      return out_image.astype(np.float32), out_meta\n",
      "\n",
      "\n",
      "              def threshold(data):\n",
      "                  \"\"\"Returns the Otsu threshold of a numpy array\"\"\"\n",
      "                  return data > threshold_otsu(data[np.isfinite(data)])\n",
      "\n",
      "\n",
      "              def normalized_difference(array1, array2):\n",
      "                  \"\"\"Returns the normalized difference of two numpy arrays\"\"\"\n",
      "                  return (array1 - array2) / (array1 + array2)\n",
      "\n",
      "\n",
      "              def aoi2box(aoi):\n",
      "                  \"\"\"Converts an area of interest expressed as a bounding box to a list of floats\"\"\"\n",
      "                  return [float(c) for c in aoi.split(\",\")]\n",
      "\n",
      "\n",
      "              def get_asset(item, common_name):\n",
      "                  \"\"\"Returns the asset of a STAC Item defined with its common band name\"\"\"\n",
      "                  for _, asset in item.get_assets().items():\n",
      "                      if not \"data\" in asset.to_dict()[\"roles\"]:\n",
      "                          continue\n",
      "\n",
      "                      eo_asset = pystac.extensions.eo.AssetEOExtension(asset)\n",
      "                      if not eo_asset.bands:\n",
      "                          continue\n",
      "                      for b in eo_asset.bands:\n",
      "                          if (\n",
      "                              \"common_name\" in b.properties.keys()\n",
      "                              and b.properties[\"common_name\"] == common_name\n",
      "                          ):\n",
      "                              return asset\n",
      "\n",
      "\n",
      "              @click.command(\n",
      "                  short_help=\"Crop\",\n",
      "                  help=\"Crops a STAC Item asset defined with its common band name\",\n",
      "              )\n",
      "              @click.option(\n",
      "                  \"--input-item\",\n",
      "                  \"item_url\",\n",
      "                  help=\"STAC Item URL or staged STAC catalog\",\n",
      "                  required=True,\n",
      "              )\n",
      "              @click.option(\n",
      "                  \"--aoi\",\n",
      "                  \"aoi\",\n",
      "                  help=\"Area of interest expressed as a bounding box\",\n",
      "                  required=True,\n",
      "              )\n",
      "              @click.option(\n",
      "                  \"--epsg\",\n",
      "                  \"epsg\",\n",
      "                  help=\"EPSG code\",\n",
      "                  required=True,\n",
      "              )\n",
      "              @click.option(\n",
      "                  \"--band\",\n",
      "                  \"bands\",\n",
      "                  help=\"Common band name\",\n",
      "                  required=True,\n",
      "                  multiple=True,\n",
      "              )\n",
      "              def main(item_url, aoi, bands, epsg):\n",
      "\n",
      "                  if os.path.isdir(item_url):\n",
      "                      catalog = pystac.read_file(os.path.join(item_url, \"catalog.json\"))\n",
      "                      item = next(catalog.get_items())\n",
      "                  else:\n",
      "                      item = pystac.read_file(item_url)\n",
      "\n",
      "                  logger.info(f\"Read {item.id} from {item.get_self_href()}\")\n",
      "\n",
      "                  cropped_assets = {}\n",
      "\n",
      "                  for band in bands:\n",
      "                      asset = get_asset(item, band)\n",
      "                      logger.info(f\"Read asset {band} from {asset.get_absolute_href()}\")\n",
      "\n",
      "                      if not asset:\n",
      "                          msg = f\"Common band name {band} not found in the assets\"\n",
      "                          logger.error(msg)\n",
      "                          raise ValueError(msg)\n",
      "\n",
      "                      bbox = aoi2box(aoi)\n",
      "\n",
      "                      out_image, out_meta = crop(asset, bbox, epsg)\n",
      "\n",
      "                      cropped_assets[band] = out_image[0]\n",
      "\n",
      "                  nd = normalized_difference(cropped_assets[bands[0]], cropped_assets[bands[1]])\n",
      "\n",
      "                  water_bodies = threshold(nd)\n",
      "\n",
      "                  out_meta.update(\n",
      "                      {\n",
      "                          \"dtype\": \"uint8\",\n",
      "                          \"driver\": \"COG\",\n",
      "                          \"tiled\": True,\n",
      "                          \"compress\": \"lzw\",\n",
      "                          \"blockxsize\": 256,\n",
      "                          \"blockysize\": 256,\n",
      "                      }\n",
      "                  )\n",
      "\n",
      "                  water_body = \"otsu.tif\"\n",
      "\n",
      "                  with rasterio.open(water_body, \"w\", **out_meta) as dst_dataset:\n",
      "                      logger.info(f\"Write otsu.tif\")\n",
      "                      dst_dataset.write(water_bodies, indexes=1)\n",
      "\n",
      "                  logger.info(f\"Creating a STAC Catalog\")\n",
      "                  cat = pystac.Catalog(id=\"catalog\", description=\"water-bodies\")\n",
      "\n",
      "                  if os.path.isdir(item_url):\n",
      "                      catalog = pystac.read_file(os.path.join(item_url, \"catalog.json\"))\n",
      "                      item = next(catalog.get_items())\n",
      "                  else:\n",
      "                      item = pystac.read_file(item_url)\n",
      "\n",
      "                  os.makedirs(item.id, exist_ok=True)\n",
      "                  shutil.copy(water_body, item.id)\n",
      "\n",
      "                  out_item = rio_stac.stac.create_stac_item(\n",
      "                      source=water_body,\n",
      "                      input_datetime=item.datetime,\n",
      "                      id=item.id,\n",
      "                      asset_roles=[\"data\", \"visual\"],\n",
      "                      asset_href=os.path.basename(water_body),\n",
      "                      asset_name=\"data\",\n",
      "                      with_proj=True,\n",
      "                      with_raster=False,\n",
      "                  )\n",
      "\n",
      "                  cat.add_items([out_item])\n",
      "\n",
      "                  cat.normalize_and_save(\n",
      "                      root_href=\"./\", catalog_type=pystac.CatalogType.SELF_CONTAINED\n",
      "                  )\n",
      "\n",
      "                  os.remove(water_body)\n",
      "\n",
      "                  logger.info(\"Done!\")\n",
      "\n",
      "\n",
      "              if __name__ == \"__main__\":\n",
      "                  main()\n",
      "    hints:\n",
      "      DockerRequirement:\n",
      "        dockerPull: qw:latest\n",
      "    baseCommand: [\"python\", \"-m\", \"app\"]\n",
      "    arguments: []\n",
      "    inputs:\n",
      "      item:\n",
      "        type: string\n",
      "        inputBinding:\n",
      "          prefix: --input-item\n",
      "      aoi:\n",
      "        type: string\n",
      "        inputBinding:\n",
      "          prefix: --aoi\n",
      "      epsg:\n",
      "        type: string\n",
      "        inputBinding:\n",
      "          prefix: --epsg\n",
      "      band:\n",
      "        type:\n",
      "          - type: array\n",
      "            items: string\n",
      "            inputBinding:\n",
      "              prefix: '--band'\n",
      "    outputs:\n",
      "      stac-catalog:\n",
      "        outputBinding:\n",
      "          glob: .\n",
      "        type: Directory\n"
     ]
    }
   ],
   "source": [
    "cat workflow.cwl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item: \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\"\n",
      "aoi: \"-121.399,39.834,-120.74,40.472\"\n",
      "epsg: \"EPSG:4326\"\n"
     ]
    }
   ],
   "source": [
    "cat params.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;30mINFO\u001b[0m /opt/conda/bin/cwltool 3.1.20231114134824\n",
      "\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///workspace/quickwin-inline-code/workflow.cwl'\n",
      "\u001b[1;30mINFO\u001b[0m [workflow ] start\n",
      "\u001b[1;30mINFO\u001b[0m [workflow ] starting step node_detect\n",
      "\u001b[1;30mINFO\u001b[0m [step node_detect] start\n",
      "\u001b[33mWARN\u001b[0m[0000] \"/\" is not a shared mount, this could cause issues or missing mounts with rootless containers \n",
      "Error: error inspecting object: no such object: \"qw:latest\"\n",
      "\u001b[1;30mINFO\u001b[0m ['podman', 'pull', 'qw:latest']\n",
      "Error: short-name \"qw:latest\" did not resolve to an alias and no unqualified-search registries are defined in \"/etc/containers/registries.conf\"\n",
      "\u001b[1;30mERROR\u001b[0m \u001b[31mWorkflow error, try again with --debug for more information:\n",
      "Docker is not available for this tool, try --no-container to disable Docker, or install a user space Docker replacement like uDocker with --user-space-docker-cmd.: Command '['podman', 'pull', 'qw:latest']' returned non-zero exit status 125.\u001b[0m\n"
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "cwltool workflow.cwl params.yaml\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP 1/4: FROM docker.io/python:slim-bullseye\n",
      "Trying to pull docker.io/library/python:slim-bullseye...\n",
      "Getting image source signatures\n",
      "Copying blob 226f2f9a8120 [================================>---] 2.6MiB / 2.9MiB\n",
      "Copying blob 29ff364e54a5 [====================================] 1.0MiB / 1.0MiB\n",
      "Copying blob bfe30d23a9fb [====================================] 244.0b / 244.0b\n",
      "Copying blob c0edef2937fa [==>--------------------------------] 2.3MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 [=====>-----------------------------] 1.8MiB / 10.7MiB\n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 [====================================] 2.9MiB / 2.9MiB\n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [====>------------------------------] 4.4MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 [=============>---------------------] 4.2MiB / 10.7MiB\n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [==========>------------------------] 9.1MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 [============================>------] 8.7MiB / 10.7MiB\n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [=============>--------------------] 12.4MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 [==================================] 10.7MiB / 10.7MiB\n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [====================>-------------] 18.3MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [==========================>-------] 23.7MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [================================>-] 29.1MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa [==================================] 30.0MiB / 30.0MiB\n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "\u001b[5A\u001b[JCopying blob 226f2f9a8120 done  \n",
      "Copying blob 29ff364e54a5 done  \n",
      "Copying blob bfe30d23a9fb done  \n",
      "Copying blob c0edef2937fa done  \n",
      "Copying blob 4e7818858816 done  \n",
      "Copying config 95fd5d5893 [====================================] 6.5KiB / 6.5KiB\n",
      "\u001b[1A\u001b[JCopying config 95fd5d5893 done  \n",
      "Writing manifest to image destination\n",
      "Storing signatures\n",
      "STEP 2/4: ADD requirements.txt /tmp/requirements.txt\n",
      "--> 46b98660fa8\n",
      "STEP 3/4: RUN pip install --no-cache-dir -r /tmp/requirements.txt &&     python -c \"import rasterio\" &&     rm -rf /tmp/requirements.txt\n",
      "Collecting rasterio (from -r /tmp/requirements.txt (line 1))\n",
      "  Downloading rasterio-1.3.9-cp312-cp312-manylinux2014_x86_64.whl.metadata (14 kB)\n",
      "Collecting click (from -r /tmp/requirements.txt (line 2))\n",
      "  Downloading click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting pystac (from -r /tmp/requirements.txt (line 3))\n",
      "  Downloading pystac-1.9.0-py3-none-any.whl.metadata (6.4 kB)\n",
      "Collecting loguru (from -r /tmp/requirements.txt (line 4))\n",
      "  Downloading loguru-0.7.2-py3-none-any.whl.metadata (23 kB)\n",
      "Collecting pyproj (from -r /tmp/requirements.txt (line 5))\n",
      "  Downloading pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (31 kB)\n",
      "Collecting shapely (from -r /tmp/requirements.txt (line 6))\n",
      "  Downloading shapely-2.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.0 kB)\n",
      "Collecting scikit-image (from -r /tmp/requirements.txt (line 7))\n",
      "  Downloading scikit_image-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
      "Collecting rio_stac (from -r /tmp/requirements.txt (line 9))\n",
      "  Downloading rio_stac-0.9.0-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting affine (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading affine-2.4.0-py3-none-any.whl.metadata (4.0 kB)\n",
      "Collecting attrs (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading attrs-23.2.0-py3-none-any.whl.metadata (9.5 kB)\n",
      "Collecting certifi (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading certifi-2024.2.2-py3-none-any.whl.metadata (2.2 kB)\n",
      "Collecting cligj>=0.5 (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading cligj-0.7.2-py3-none-any.whl.metadata (5.0 kB)\n",
      "Collecting numpy (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.0/61.0 kB 160.8 MB/s eta 0:00:00\n",
      "Collecting snuggs>=1.4.1 (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading snuggs-1.4.7-py3-none-any.whl.metadata (3.4 kB)\n",
      "Collecting click-plugins (from rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading click_plugins-1.1.1-py2.py3-none-any.whl.metadata (6.4 kB)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/site-packages (from rasterio->-r /tmp/requirements.txt (line 1)) (69.1.1)\n",
      "Collecting python-dateutil>=2.7.0 (from pystac->-r /tmp/requirements.txt (line 3))\n",
      "  Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)\n",
      "Collecting scipy>=1.8 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.4/60.4 kB 154.2 MB/s eta 0:00:00\n",
      "Collecting networkx>=2.8 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n",
      "Collecting pillow>=9.0.1 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (9.7 kB)\n",
      "Collecting imageio>=2.27 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading imageio-2.34.0-py3-none-any.whl.metadata (4.9 kB)\n",
      "Collecting tifffile>=2022.8.12 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading tifffile-2024.2.12-py3-none-any.whl.metadata (31 kB)\n",
      "Collecting packaging>=21 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading packaging-24.0-py3-none-any.whl.metadata (3.2 kB)\n",
      "Collecting lazy_loader>=0.3 (from scikit-image->-r /tmp/requirements.txt (line 7))\n",
      "  Downloading lazy_loader-0.3-py3-none-any.whl.metadata (4.3 kB)\n",
      "Collecting six>=1.5 (from python-dateutil>=2.7.0->pystac->-r /tmp/requirements.txt (line 3))\n",
      "  Downloading six-1.16.0-py2.py3-none-any.whl.metadata (1.8 kB)\n",
      "Collecting pyparsing>=2.1.6 (from snuggs>=1.4.1->rasterio->-r /tmp/requirements.txt (line 1))\n",
      "  Downloading pyparsing-3.1.2-py3-none-any.whl.metadata (5.1 kB)\n",
      "Downloading rasterio-1.3.9-cp312-cp312-manylinux2014_x86_64.whl (20.7 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 20.7/20.7 MB 155.4 MB/s eta 0:00:00\n",
      "Downloading click-8.1.7-py3-none-any.whl (97 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 97.9/97.9 kB 204.4 MB/s eta 0:00:00\n",
      "Downloading pystac-1.9.0-py3-none-any.whl (181 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 181.6/181.6 kB 187.2 MB/s eta 0:00:00\n",
      "Downloading loguru-0.7.2-py3-none-any.whl (62 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.5/62.5 kB 176.2 MB/s eta 0:00:00\n",
      "Downloading pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.7 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.7/8.7 MB 138.2 MB/s eta 0:00:00\n",
      "Downloading shapely-2.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.5 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.5/2.5 MB 165.2 MB/s eta 0:00:00\n",
      "Downloading scikit_image-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.0 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.0/15.0 MB 142.2 MB/s eta 0:00:00\n",
      "Downloading rio_stac-0.9.0-py3-none-any.whl (11 kB)\n",
      "Downloading cligj-0.7.2-py3-none-any.whl (7.1 kB)\n",
      "Downloading imageio-2.34.0-py3-none-any.whl (313 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 313.4/313.4 kB 184.2 MB/s eta 0:00:00\n",
      "Downloading lazy_loader-0.3-py3-none-any.whl (9.1 kB)\n",
      "Downloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.6/1.6 MB 225.6 MB/s eta 0:00:00\n",
      "Downloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.0 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 18.0/18.0 MB 149.4 MB/s eta 0:00:00\n",
      "Downloading packaging-24.0-py3-none-any.whl (53 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.5/53.5 kB 212.8 MB/s eta 0:00:00\n",
      "Downloading pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl (4.5 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.5/4.5 MB 97.0 MB/s eta 0:00:00\n",
      "Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 229.9/229.9 kB 290.7 MB/s eta 0:00:00\n",
      "Downloading scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (37.8 MB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 37.8/37.8 MB 222.8 MB/s eta 0:00:00\n",
      "Downloading snuggs-1.4.7-py3-none-any.whl (5.4 kB)\n",
      "Downloading tifffile-2024.2.12-py3-none-any.whl (224 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 224.5/224.5 kB 269.9 MB/s eta 0:00:00\n",
      "Downloading affine-2.4.0-py3-none-any.whl (15 kB)\n",
      "Downloading attrs-23.2.0-py3-none-any.whl (60 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.8/60.8 kB 217.0 MB/s eta 0:00:00\n",
      "Downloading certifi-2024.2.2-py3-none-any.whl (163 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 163.8/163.8 kB 260.2 MB/s eta 0:00:00\n",
      "Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)\n",
      "Downloading pyparsing-3.1.2-py3-none-any.whl (103 kB)\n",
      "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 103.2/103.2 kB 214.7 MB/s eta 0:00:00\n",
      "Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
      "Installing collected packages: six, pyparsing, pillow, packaging, numpy, networkx, loguru, lazy_loader, click, certifi, attrs, affine, tifffile, snuggs, shapely, scipy, python-dateutil, pyproj, imageio, cligj, click-plugins, scikit-image, rasterio, pystac, rio_stac\n",
      "Successfully installed affine-2.4.0 attrs-23.2.0 certifi-2024.2.2 click-8.1.7 click-plugins-1.1.1 cligj-0.7.2 imageio-2.34.0 lazy_loader-0.3 loguru-0.7.2 networkx-3.2.1 numpy-1.26.4 packaging-24.0 pillow-10.2.0 pyparsing-3.1.2 pyproj-3.6.1 pystac-1.9.0 python-dateutil-2.9.0.post0 rasterio-1.3.9 rio_stac-0.9.0 scikit-image-0.22.0 scipy-1.12.0 shapely-2.0.3 six-1.16.0 snuggs-1.4.7 tifffile-2024.2.12\n",
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "--> 788352a9a7c\n",
      "STEP 4/4: ENTRYPOINT []\n",
      "COMMIT registry:5000/qw:latest\n",
      "--> 718d307df50\n",
      "Successfully tagged registry:5000/qw:latest\n",
      "718d307df50210593c0bfd873835d7c1f904939eade6ad18dec2a775666e5d5e\n"
     ]
    }
   ],
   "source": [
    "podman build -t registry:5000/qw:latest .\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [],
   "source": [
    "s=\"detect-water-body\" t=\"registry:5000/qw:latest\" yq -i eval '(.$graph[] | select (.id == env(s)) ).hints.DockerRequirement.dockerPull = env(t)' workflow.cwl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;30mINFO\u001b[0m /opt/conda/bin/cwltool 3.1.20231114134824\n",
      "\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///workspace/quickwin-inline-code/workflow.cwl'\n",
      "\u001b[1;30mINFO\u001b[0m [workflow ] start\n",
      "\u001b[1;30mINFO\u001b[0m [workflow ] starting step node_detect\n",
      "\u001b[1;30mINFO\u001b[0m [step node_detect] start\n",
      "\u001b[1;30mWARNING\u001b[0m \u001b[33m[job node_detect] Skipping Docker software container '--memory' limit despite presence of ResourceRequirement with ramMin and/or ramMax setting. Consider running with --strict-memory-limit for increased portability assurance.\u001b[0m\n",
      "\u001b[1;30mWARNING\u001b[0m \u001b[33m[job node_detect] Skipping Docker software container '--cpus' limit despite presence of ResourceRequirement with coresMin and/or coresMax setting. Consider running with --strict-cpu-limit for increased portability assurance.\u001b[0m\n",
      "\u001b[1;30mINFO\u001b[0m [job node_detect] /tmp/q0dc9yel$ podman \\\n",
      "    run \\\n",
      "    -i \\\n",
      "    --userns=keep-id \\\n",
      "    --mount=type=bind,source=/tmp/q0dc9yel,target=/MRdBdC \\\n",
      "    --mount=type=bind,source=/tmp/qezkygg2,target=/tmp \\\n",
      "    --workdir=/MRdBdC \\\n",
      "    --read-only=true \\\n",
      "    --user=1001:100 \\\n",
      "    --rm \\\n",
      "    --cidfile=/tmp/x1crjtpg/20240329161301-904633.cid \\\n",
      "    --env=TMPDIR=/tmp \\\n",
      "    --env=HOME=/MRdBdC \\\n",
      "    registry:5000/qw:latest \\\n",
      "    python \\\n",
      "    -m \\\n",
      "    app \\\n",
      "    --band \\\n",
      "    green \\\n",
      "    --aoi \\\n",
      "    -121.399,39.834,-120.74,40.472 \\\n",
      "    --epsg \\\n",
      "    EPSG:4326 \\\n",
      "    --input-item \\\n",
      "    https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A \\\n",
      "    --band \\\n",
      "    nir\n",
      "2024-03-29 16:13:10.693 | INFO     | __main__:main:126 - Read S2B_10TFK_20210713_0_L2A from https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n",
      "2024-03-29 16:13:10.731 | INFO     | __main__:main:132 - Read asset green from https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/10/T/FK/2021/7/S2B_10TFK_20210713_0_L2A/B03.tif\n",
      "2024-03-29 16:13:12.795 | INFO     | __main__:crop:39 - Crop https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/10/T/FK/2021/7/S2B_10TFK_20210713_0_L2A/B03.tif\n",
      "2024-03-29 16:13:47.758 | INFO     | __main__:main:132 - Read asset nir from https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/10/T/FK/2021/7/S2B_10TFK_20210713_0_L2A/B08.tif\n",
      "2024-03-29 16:13:49.082 | INFO     | __main__:crop:39 - Crop https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/10/T/FK/2021/7/S2B_10TFK_20210713_0_L2A/B08.tif\n",
      "2024-03-29 16:14:09.012 | INFO     | __main__:main:163 - Write otsu.tif\n",
      "2024-03-29 16:14:09.750 | INFO     | __main__:main:166 - Creating a STAC Catalog\n",
      "2024-03-29 16:14:10.210 | INFO     | __main__:main:197 - Done!\n",
      "\u001b[1;30mINFO\u001b[0m [job node_detect] Max memory used: 2419MiB\n",
      "\u001b[1;30mINFO\u001b[0m [job node_detect] completed success\n",
      "\u001b[1;30mINFO\u001b[0m [step node_detect] completed success\n",
      "\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n",
      "{\n",
      "    \"stac_catalog\": {\n",
      "        \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel\",\n",
      "        \"basename\": \"q0dc9yel\",\n",
      "        \"class\": \"Directory\",\n",
      "        \"listing\": [\n",
      "            {\n",
      "                \"class\": \"File\",\n",
      "                \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/app.py\",\n",
      "                \"basename\": \"app.py\",\n",
      "                \"checksum\": \"sha1$294adb71d5b9be55a341eabea813ee08137793a5\",\n",
      "                \"size\": 5102,\n",
      "                \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/app.py\"\n",
      "            },\n",
      "            {\n",
      "                \"class\": \"Directory\",\n",
      "                \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/__pycache__\",\n",
      "                \"basename\": \"__pycache__\",\n",
      "                \"listing\": [\n",
      "                    {\n",
      "                        \"class\": \"File\",\n",
      "                        \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/__pycache__/app.cpython-312.pyc\",\n",
      "                        \"basename\": \"app.cpython-312.pyc\",\n",
      "                        \"checksum\": \"sha1$96b5f957ad05bbbda3e6adb655f6b0f45c9eaf0a\",\n",
      "                        \"size\": 8197,\n",
      "                        \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/__pycache__/app.cpython-312.pyc\"\n",
      "                    }\n",
      "                ],\n",
      "                \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/__pycache__\"\n",
      "            },\n",
      "            {\n",
      "                \"class\": \"File\",\n",
      "                \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/catalog.json\",\n",
      "                \"basename\": \"catalog.json\",\n",
      "                \"checksum\": \"sha1$497593179aa32d377199b10d540a37fd851a1727\",\n",
      "                \"size\": 359,\n",
      "                \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/catalog.json\"\n",
      "            },\n",
      "            {\n",
      "                \"class\": \"Directory\",\n",
      "                \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A\",\n",
      "                \"basename\": \"S2B_10TFK_20210713_0_L2A\",\n",
      "                \"listing\": [\n",
      "                    {\n",
      "                        \"class\": \"File\",\n",
      "                        \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A/otsu.tif\",\n",
      "                        \"basename\": \"otsu.tif\",\n",
      "                        \"checksum\": \"sha1$8cb131413518c30be6ba485ea61764491444cde5\",\n",
      "                        \"size\": 290099,\n",
      "                        \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A/otsu.tif\"\n",
      "                    },\n",
      "                    {\n",
      "                        \"class\": \"File\",\n",
      "                        \"location\": \"file:///workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A/S2B_10TFK_20210713_0_L2A.json\",\n",
      "                        \"basename\": \"S2B_10TFK_20210713_0_L2A.json\",\n",
      "                        \"checksum\": \"sha1$d361f943e969408533c76bff7fa05a3ba88624d6\",\n",
      "                        \"size\": 5560,\n",
      "                        \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A/S2B_10TFK_20210713_0_L2A.json\"\n",
      "                    }\n",
      "                ],\n",
      "                \"path\": \"/workspace/quickwin-inline-code/q0dc9yel/S2B_10TFK_20210713_0_L2A\"\n",
      "            }\n",
      "        ],\n",
      "        \"path\": \"/workspace/quickwin-inline-code/q0dc9yel\"\n",
      "    }\n",
      "}\u001b[1;30mINFO\u001b[0m Final process status is success\n"
     ]
    }
   ],
   "source": [
    "cwltool workflow.cwl params.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/quickwin-inline-code/3edn2u1q\n",
      "├── S2B_10TFK_20210713_0_L2A\n",
      "│   ├── S2B_10TFK_20210713_0_L2A.json\n",
      "│   └── otsu.tif\n",
      "├── __pycache__\n",
      "│   └── app.cpython-312.pyc\n",
      "├── app.py\n",
      "└── catalog.json\n",
      "\n",
      "2 directories, 5 files\n"
     ]
    }
   ],
   "source": [
    "tree -l /workspace/quickwin-inline-code/3edn2u1q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting image source signatures\n",
      "\u001b[33mWARN\u001b[0m[0000] failed, retrying in 1s ... (1/3). Error: trying to reuse blob sha256:3c8879ab2cf25eb049af95711715b5b99d97490ab4eedbe8d0f160f53ef5a645 at destination: pinging container registry registry:5000: Get \"http://registry:5000/v2/\": dial tcp 10.46.78.158:5000: connect: connection refused \n",
      "Getting image source signatures\n",
      "\u001b[33mWARN\u001b[0m[0001] failed, retrying in 1s ... (2/3). Error: trying to reuse blob sha256:3c8879ab2cf25eb049af95711715b5b99d97490ab4eedbe8d0f160f53ef5a645 at destination: pinging container registry registry:5000: Get \"http://registry:5000/v2/\": dial tcp 10.46.78.158:5000: connect: connection refused \n",
      "Getting image source signatures\n",
      "\u001b[33mWARN\u001b[0m[0002] failed, retrying in 1s ... (3/3). Error: trying to reuse blob sha256:3c8879ab2cf25eb049af95711715b5b99d97490ab4eedbe8d0f160f53ef5a645 at destination: pinging container registry registry:5000: Get \"http://registry:5000/v2/\": dial tcp 10.46.78.158:5000: connect: connection refused \n",
      "Getting image source signatures\n",
      "Error: trying to reuse blob sha256:3c8879ab2cf25eb049af95711715b5b99d97490ab4eedbe8d0f160f53ef5a645 at destination: pinging container registry registry:5000: Get \"http://registry:5000/v2/\": dial tcp 10.46.78.158:5000: connect: connection refused\n"
     ]
    },
    {
     "ename": "",
     "evalue": "125",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "podman push registry:5000/qw:latest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "shellscript"
    }
   },
   "outputs": [],
   "source": [
    "/opt/conda/bin/calrissian --pod-nodeselectors /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Bash",
   "language": "bash",
   "name": "bash"
  },
  "language_info": {
   "codemirror_mode": "shell",
   "file_extension": ".sh",
   "mimetype": "text/x-sh",
   "name": "bash"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}