Shorten camels-us install; support local and remote image path; fix bugs

Author: johnkit

dist/neuralhydrology_demo1/install_demo1.sh

@@ -11,15 +11,14 @@ help()
 "
 }
 
-if [[n $# -lt 2 ]]; then
+if [[ $# -lt 2 ]]; then
   help
   exit 0
 fi
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 DATA_DIR=$(realpath $1)
 EXP_DIR=$(realpath $2)
-CAMELS_SCRIPT="wget-camels-20250127T1907.sh"
 
 # Create directories
 mkdir -p ${EXP_DIR}
@@ -31,19 +30,20 @@ echo "Pulling container image ${DEMO_IMAGE} (6 GB) ..."
 docker pull ${DEMO_IMAGE}
 
 # Download CAMELS-US
-echo "Downloading CAMEL-US dataset (14 GB) ..."
-wget_script=${SOURCE_DIR}/.install/${CAMELS_SCRIPT}
-cp ${wget_script} ${DATA_DIR}
-cd ${DATA_DIR} && "./${CAMELS_SCRIPT}"
+echo "Downloading CAMEL-US dataset (3.2 GB) ..."
+zip_filename="basin_timeseries_v1p2_metForcing_obsFlow.zip"
+zip_url="https://gdex.ucar.edu/dataset/camels/file/${zip_filename}"
+cd ${DATA_DIR} && wget ${zip_url}
+
 
 # Unzip contents
 echo "Unzipping data ..."
-cd ${DATA_DIR} && unzip basin_timeseries_v1p2_metForcing_obsFlow.zip
+cd ${DATA_DIR} && unzip ${zip_filename}
 
 # Generate .args.txt file
-ARGS_FILE=${SCRIPT_DIR}/app/demo1/.args.txt
+ARGS_FILE=${SCRIPT_DIR}/app/.args.txt
 echo "--data_dir" > ${ARGS_FILE}
-echo ${DATA_DIR} >> ${ARGS_FILE}/basin_dataset_public_v1p2
+echo ${DATA_DIR}/basin_dataset_public_v1p2 >> ${ARGS_FILE}
 echo "--experiments_dir" >> ${ARGS_FILE}
 echo ${EXP_DIR} >> ${ARGS_FILE}
 

neuralhydrology/demo1.py

@@ -42,17 +42,18 @@ def main():
         print('Exiting')
         sys.exit(2)
 
-    # name_tag = f'{IMAGE_NAME}:{IMAGE_TAG}'
-    runner = DemoRun(IMAGE_NAME, args.data_dir, verbose=args.verbose)
+    image = IMAGE_NAME if args.local_image_build else f'{IMAGE_REGISTRY_USER}/{IMAGE_NAME}'
+    runner = DemoRun(image, args.data_dir, verbose=args.verbose)
     if not runner.is_image_available():
+        print()
         if args.local_image_build:
-            print(f'container image not available: {IMAGE_NAME}')
-            sys.exit(1)
+            print(f'Container image not found: {image}')
         else:
-            print('Future: add code to fetch image')
-            sys.exit(1)
+            print('Container image not found')
+            print(f'In a terminal run `docker pull {image}` and rerun this script')
+        sys.exit(1)
 
-    print('Image was found!')
+    print(f'Container image was found: {image}')
     runner.execute(
         args.basin_id,
         pathlib.Path(args.experiments_dir),

neuralhydrology/package_demo1.py

@@ -8,6 +8,7 @@
 rm -rf neuralhydrology_demo1/app/demo1/__pycache__/ && tar zcf neuralhydrology_demo1.tgz neuralhydrology_demo1/
 """
 
+import os
 import pathlib
 import shutil
 import tarfile
@@ -19,7 +20,8 @@
 # Delete any existing tgz and app files
 tgz_file = source_dir / 'dist/neuralhydrology_demo1.tgz'
 tgz_file.unlink(missing_ok=True)
-shutil.rmtree(app_dir)
+if app_dir.exists():
+    shutil.rmtree(app_dir)
 
 # Copy source files to app dir
 app_dir.mkdir(parents=True, exist_ok=True)
@@ -38,7 +40,10 @@
     shutil.copy2(from_path, app_sub_dir)
 
 # Generate tgz file
-tgz_path = source_dir.parent / 'dist/neuralhydrology_demo1.tgz'
-with tarfile.open(tgz_path, 'w:gz') as tar:
-    tar.add(dist_dir)
+dist_dir = source_dir.parent / 'dist'
+os.chdir(dist_dir)
+tgz_file = 'neuralhydrology_demo1.tgz'
+with tarfile.open(tgz_file, 'w:gz') as tar:
+    tar.add('neuralhydrology_demo1')
+    tgz_path = dist_dir / tgz_file
     print(f'Wrote {tgz_path}')