S3 credentials order/testing and add S3 object URIs support

.codespellrc

@@ -1,6 +1,6 @@
 # Ref: https://github.com/codespell-project/codespell#using-a-config-file
 [codespell]
-skip = .git,*.svg,.codespellrc,./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c,./config/sanitizer/LICENSE,./config/sanitizer/sanitizers.cmake,./tools/test/h5import/testfiles/*.conf,./tools/test/h5repack/testfiles/*.dat,./test/API/driver,./configure,./bin/ltmain.sh,./bin/depcomp,./bin/config.guess,./bin/config.sub,./autom4te.cache,./m4/libtool.m4,./c++/src/*.html,./HDF5Examples/depcomp,./release_docs/HISTORY-*.txt
+skip = .git,*.svg,.codespellrc,./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c,./config/sanitizer/LICENSE,./config/sanitizer/sanitizers.cmake,./tools/test/h5import/testfiles/*.conf,./tools/test/h5repack/testfiles/*.dat,./test/API/driver,./test/testfiles/t8.shakespeare.txt,./c++/src/*.html,./HDF5Examples/depcomp,./release_docs/HISTORY-*.txt
 check-hidden = true
 # ignore-regex =
 ignore-words-list = ot,isnt,inout,nd,parms,parm,ba,offsetP,ser,ois,had,fiter,fo,clude,refere,minnum,offsetp,creat,ans:,eiter,lastr,ans,isn't,ifset,sur,trun,dne,tthe,hda,filname,te,htmp,ake,gord,numer,ro,oce,msdos,TEXTIN,indx,FLE

.github/workflows/cmake.yml

@@ -28,6 +28,10 @@ jobs:
       name: "CMake Special Workflows"
       uses: ./.github/workflows/main-cmake-spc.yml
 
+    call-workflow-s3proxy-cmake:
+      name: "CMake s3proxy Workflows"
+      uses: ./.github/workflows/s3proxy.yml
+
     call-debug-concurrent-cmake:
         name: "CMake Debug Concurrency Workflows"
         uses: ./.github/workflows/main-cmake.yml

.github/workflows/s3proxy.yml

@@ -0,0 +1,62 @@
+name: hdf5 dev CMake s3proxy CI
+
+# Controls when the action will run. Triggers the workflow on a call
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+# A workflow run is made up of one or more jobs that can run sequentially or
+# in parallel. We just have one job, but the matrix items defined below will
+# run in parallel.
+jobs:
+  build_linux_ros3:
+      name: "ros3 proxy tests"
+      runs-on: ubuntu-latest
+      steps:
+        # SETUP
+        - name: Install Linux Dependencies
+          run: |
+            sudo apt-get update
+            sudo apt-get install ninja-build doxygen graphviz
+            sudo apt install libssl3 libssl-dev libcurl4 libcurl4-openssl-dev
+
+        # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+        - name: Get Sources
+          uses: actions/[email protected]
+
+        # CMAKE CONFIGURE
+        - name: CMake Configure
+          run: |
+            mkdir "${{ runner.workspace }}/build"
+            cd "${{ runner.workspace }}/build"
+            cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \
+              -G Ninja \
+              --log-level=VERBOSE \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DBUILD_SHARED_LIBS=ON \
+              -DHDF5_ENABLE_ALL_WARNINGS=ON \
+              -DHDF5_ENABLE_PARALLEL:BOOL=OFF \
+              -DHDF5_BUILD_FORTRAN:BOOL=ON \
+              -DHDF5_BUILD_CPP_LIB:BOOL=ON \
+              -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \
+              -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=ON \
+              -DHDF5_ENABLE_SZIP_ENCODING:BOOL=ON \
+              -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON \
+              -DLIBAEC_USE_LOCALCONTENT=OFF \
+              -DZLIB_USE_LOCALCONTENT=OFF \
+              -DHDF5_ENABLE_ROS3_VFD:BOOL=ON \
+              -DHDF5_ENABLE_DOCKER_PROXY=ON \
+              $GITHUB_WORKSPACE
+          shell: bash
+
+        # BUILD
+        - name: CMake Build
+          run: cmake --build . --parallel 3 --config Release
+          working-directory: ${{ runner.workspace }}/build
+
+        # RUN TESTS
+        - name: CMake Run Tests
+          run: ctest . --parallel 2 -C Release -V
+          working-directory: ${{ runner.workspace }}/build

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required (VERSION 3.18)
 project (HDF5 C)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if (POLICY CMP0074)
   # find_package() uses <PackageName>_ROOT variables.

CMakeTests.cmake

@@ -26,6 +26,13 @@
   option (HDF5_DISABLE_TESTS_REGEX "Regex pattern to set execution of specific tests to DISABLED" "")
   mark_as_advanced (HDF5_DISABLE_TESTS_REGEX)
 
+if (HDF5_ENABLE_ROS3_VFD)
+  if (HDF5_ENABLE_DOCKER_PROXY)
+    # Create a test credentials file
+    file (WRITE "${CMAKE_BINARY_DIR}/credentials" "[default]\naws_access_key_id = remote-identity\naws_secret_access_key = remote-credential\nregion = us-east-1\n\n[ros3_vfd_test]\naws_access_key_id = remote-identity\naws_secret_access_key = remote-credential\nregion = us-east-1\n")
+  endif ()
+endif ()
+
   option (HDF5_TEST_API "Execute HDF5 API tests" ON)
   mark_as_advanced (HDF5_TEST_API)
   if (HDF5_TEST_API)

config/cmake/ConfigureChecks.cmake

@@ -610,17 +610,39 @@ endif ()
 #  Check if ROS3 driver can be built
 #-----------------------------------------------------------------------------
 option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
-  if (HDF5_ENABLE_ROS3_VFD)
-    find_package(CURL REQUIRED)
-    find_package(OpenSSL REQUIRED)
-    if (${CURL_FOUND} AND ${OPENSSL_FOUND})
-      set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
-      list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES})
-      INCLUDE_DIRECTORIES (${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR})
+if (HDF5_ENABLE_ROS3_VFD)
+  find_package(CURL REQUIRED)
+  find_package(OpenSSL REQUIRED)
+  if (${CURL_FOUND} AND ${OPENSSL_FOUND})
+    set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
+    list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES})
+    INCLUDE_DIRECTORIES (${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR})
+  else ()
+    set (HDF5_ENABLE_ROS3_VFD OFF CACHE BOOL "Build the ROS3 Virtual File Driver" FORCE)
+    message (WARNING "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
+  endif ()
+  option (HDF5_ENABLE_DOCKER_PROXY "Use docker for S3 proxy" OFF)
+  if (HDF5_ENABLE_DOCKER_PROXY)
+    #check if docker is available
+    find_program (DOCKER_EXECUTABLE docker)
+    if (DOCKER_EXECUTABLE)
+      execute_process (
+          COMMAND ${DOCKER_EXECUTABLE} info
+          RESULT_VARIABLE DOCKER_CHECK_RESULT
+          OUTPUT_VARIABLE DOCKER_CHECK_OUTPUT
+          ERROR_VARIABLE DOCKER_CHECK_ERROR
+      )
+      if (DOCKER_CHECK_RESULT EQUAL 0)
+        message (VERBOSE "Docker is installed and running.")
+      else()
+        set (HDF5_ENABLE_DOCKER_PROXY OFF CACHE BOOL "Use docker for S3 proxy" FORCE)
+        message (WARNING "Docker is installed but not running or accessible: ${DOCKER_CHECK_ERROR}")
+      endif ()
     else ()
-      set (HDF5_ENABLE_ROS3_VFD OFF CACHE BOOL "Build the ROS3 Virtual File Driver" FORCE)
-      message (WARNING "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
+      set (HDF5_ENABLE_DOCKER_PROXY OFF CACHE BOOL "Use docker for S3 proxy" FORCE)
+      message (WARNING "Docker is not installed.")
     endif ()
+  endif ()
 endif ()
 
 # ----------------------------------------------------------------------

config/cmake/grepTest.cmake

@@ -14,13 +14,13 @@
 cmake_policy(SET CMP0007 NEW)
 
 # arguments checking
-if (NOT TEST_PROGRAM)
+if (NOT TEST_PROGRAM) #the program to be run
   message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
 endif ()
-if (NOT TEST_FOLDER)
+if (NOT TEST_FOLDER) # this is the folder where the test program is run
   message (FATAL_ERROR "Require TEST_FOLDER to be defined")
 endif ()
-if (NOT TEST_OUTPUT)
+if (NOT TEST_OUTPUT) # the output file to capture the test program output
   message (FATAL_ERROR "Require TEST_OUTPUT to be defined")
 endif ()
 if (NOT TEST_FILTER)
@@ -218,6 +218,7 @@ if (TEST_FILTER)
   endif ()
 endif ()
 
+# cleanup the output files
 if (NOT DEFINED ENV{HDF5_NOCLEANUP})
   if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}" AND NOT TEST_SAVE)
     file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT})

config/cmake/runProxy.cmake

@@ -0,0 +1,183 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the LICENSE file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# [email protected].
+#
+# runProxy.cmake dtarts a docker instance of s3proxy and uploads files.
+# Exit status of command can also be compared.
+
+# arguments checking
+if (NOT TEST_PROGRAM) # currently this is the docker command
+  message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
+endif ()
+if (NOT TEST_PRODUCT) # this is the docker product to be used
+  message (FATAL_ERROR "Require TEST_PRODUCT to be defined")
+endif ()
+if (NOT TEST_PORT) # this is the port for the docker instance
+  message (FATAL_ERROR "Require TEST_PORT to be defined")
+endif ()
+if (NOT TEST_FOLDER) # this is the folder where the test program is run
+  message (FATAL_ERROR "Require TEST_FOLDER to be defined")
+endif ()
+if (NOT TEST_BUCKET)
+  message (FATAL_ERROR "Require TEST_BUCKET to be defined")
+endif ()
+
+
+if (TEST_ENV_VAR)
+  set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}")
+  message (VERBOSE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
+endif ()
+message (STATUS "USING ${TEST_BUCKET} ON COMMAND: docker ${TEST_PRODUCT} ${TEST_ARGS} with creds $ENV{AWS_SHARED_CREDENTIALS_FILE}")
+
+# run the test program to pull the product, capture the stdout/stderr and the result var
+execute_process (
+    COMMAND ${TEST_PROGRAM} pull ${TEST_PRODUCT}
+    WORKING_DIRECTORY ${TEST_FOLDER}
+    RESULT_VARIABLE TEST_RESULT
+    OUTPUT_FILE s3proxy-pull.out
+    ERROR_FILE s3proxy-pull.err
+    OUTPUT_VARIABLE TEST_OUT
+    ERROR_VARIABLE TEST_ERROR
+)
+
+message (VERBOSE "COMMAND Pull Result: ${TEST_RESULT}")
+
+# run the test program to start an instance of the product, capture the stdout/stderr and the result var
+execute_process (
+    COMMAND ${TEST_PROGRAM} run -d --publish ${TEST_PORT}:80 --restart=always --name ${TEST_ARGS} --env S3PROXY_AUTHORIZATION=none --env S3PROXY_ENDPOINT=http://0.0.0.0:80 --env S3PROXY_IDENTITY=remote-identity --env S3PROXY_CREDENTIAL=remote-credential --env S3PROXY_CORS_ALLOW_ALL=true ${TEST_PRODUCT}
+    WORKING_DIRECTORY ${TEST_FOLDER}
+    RESULT_VARIABLE TEST_RESULT
+    OUTPUT_FILE s3proxy-run.out
+    ERROR_FILE s3proxy-run.err
+    OUTPUT_VARIABLE TEST_OUT
+    ERROR_VARIABLE TEST_ERROR
+)
+
+if (EXISTS "${TEST_FOLDER}/s3proxy-run.out")
+  file (READ ${TEST_FOLDER}/s3proxy-run.out TEST_STREAM)
+  message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+endif ()
+message (VERBOSE "COMMAND Run Result: ${TEST_RESULT}")
+
+# if the return value is !=${TEST_EXPECT} bail out
+if (NOT TEST_RESULT EQUAL TEST_EXPECT)
+  if (NOT TEST_NOERRDISPLAY)
+    if (EXISTS "${TEST_FOLDER}/s3proxy-run.err")
+      file (READ ${TEST_FOLDER}/s3proxy-run.err TEST_STREAM)
+      message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+    endif ()
+  endif ()
+  message (FATAL_ERROR "Failed: Test program ${TEST_PRODUCT} exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
+endif ()
+
+execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+
+# check that the docker instance is running
+execute_process (
+    COMMAND ${TEST_PROGRAM} inspect --format='{{.State.Running}}' ${TEST_ARGS}
+    WORKING_DIRECTORY ${TEST_FOLDER}
+    RESULT_VARIABLE TEST_RESULT
+    OUTPUT_FILE s3proxy-filter.out
+    ERROR_FILE s3proxy-filter.err
+    OUTPUT_VARIABLE TEST_OUT
+    ERROR_VARIABLE TEST_ERROR
+)
+
+message (VERBOSE "COMMAND Inspect Result: ${TEST_RESULT}")
+
+if (NOT TEST_NOERRDISPLAY)
+  if (EXISTS "${TEST_FOLDER}/s3proxy-filter.out")
+    file (READ ${TEST_FOLDER}/s3proxy-filter.out TEST_STREAM)
+    message (STATUS "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+    string (REGEX MATCH "true" REGEX_MATCH ${TEST_STREAM})
+    string (COMPARE EQUAL "${REGEX_MATCH}" "true" REGEX_RESULT)
+    if (NOT REGEX_RESULT)
+      message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain true")
+  endif ()
+  endif ()
+endif ()
+# if the return value is !=${TEST_EXPECT} bail out
+if (NOT TEST_RESULT EQUAL TEST_EXPECT)
+  if (NOT TEST_NOERRDISPLAY)
+    if (EXISTS "${TEST_FOLDER}/s3proxy-filter.err")
+      file (READ ${TEST_FOLDER}/s3proxy-filter.err TEST_STREAM)
+      message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+    endif ()
+  endif ()
+  message (FATAL_ERROR "Failed: Test program ${TEST_PRODUCT} exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
+endif ()
+
+# create the bucket to be used
+execute_process (
+    COMMAND aws s3api create-bucket --endpoint-url=http://localhost:${TEST_PORT} --bucket ${TEST_BUCKET}
+    WORKING_DIRECTORY ${TEST_FOLDER}
+    RESULT_VARIABLE TEST_RESULT
+    OUTPUT_FILE s3proxy-bucket.out
+    ERROR_FILE s3proxy-bucket.err
+    OUTPUT_VARIABLE TEST_OUT
+    ERROR_VARIABLE TEST_ERROR
+)
+
+if (EXISTS "${TEST_FOLDER}/s3proxy-bucket.out")
+  file (READ ${TEST_FOLDER}/s3proxy-bucket.out TEST_STREAM)
+  message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+endif ()
+message (VERBOSE "COMMAND Bucket Result: ${TEST_RESULT}")
+
+# if the return value is !=${TEST_EXPECT} bail out
+if (NOT TEST_RESULT EQUAL TEST_EXPECT)
+  if (NOT TEST_NOERRDISPLAY)
+    if (EXISTS "${TEST_FOLDER}/s3proxy-bucket.err")
+      file (READ ${TEST_FOLDER}/s3proxy-bucket.err TEST_STREAM)
+      message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+    endif ()
+  endif ()
+  message (FATAL_ERROR "Failed: Create-Bucket exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
+endif ()
+
+#upload test files to the bucket
+if (TEST_FILES)
+  foreach (dfile ${TEST_FILES})
+    execute_process (
+        COMMAND aws s3api put-object --endpoint-url=http://localhost:${TEST_PORT} --body ${TEST_FOLDER}/testfiles/${dfile} --bucket ${TEST_BUCKET} --key ${dfile}
+        WORKING_DIRECTORY ${TEST_FOLDER}
+        RESULT_VARIABLE TEST_RESULT
+        OUTPUT_FILE s3proxy-${dfile}.out
+        ERROR_FILE s3proxy-${dfile}.err
+        OUTPUT_VARIABLE TEST_OUT
+        ERROR_VARIABLE TEST_ERROR
+    )
+    if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.out")
+      file (READ ${TEST_FOLDER}/s3proxy-${dfile}.out TEST_STREAM)
+      message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+    endif ()
+    message (VERBOSE "COMMAND Put Result: ${TEST_RESULT}")
+
+    # if the return value is !=${TEST_EXPECT} bail out
+    if (NOT TEST_RESULT EQUAL TEST_EXPECT)
+      if (NOT TEST_NOERRDISPLAY)
+        if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.err")
+          file (READ ${TEST_FOLDER}/s3proxy-${dfile}.err TEST_STREAM)
+          message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
+        endif ()
+      endif ()
+      message (FATAL_ERROR "Failed: Put-Object exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
+    endif ()
+  endforeach ()
+endif ()
+
+# cleanup the output files
+if (NOT DEFINED ENV{HDF5_NOCLEANUP})
+  file (GLOB REMOVE_FILES ${TEST_FOLDER}/s3proxy*)
+  file (REMOVE ${REMOVE_FILES})
+endif ()
+
+# everything went fine...
+message (STATUS "Passed: The ${TEST_PRODUCT} docker used ${TEST_BUCKET}")

config/cmake/runTest.cmake

@@ -14,13 +14,13 @@
 cmake_policy(SET CMP0007 NEW)
 
 # arguments checking
-if (NOT TEST_PROGRAM)
+if (NOT TEST_PROGRAM) #the program to be run
   message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
 endif ()
-if (NOT TEST_FOLDER)
+if (NOT TEST_FOLDER) # this is the folder where the test program is run
   message (FATAL_ERROR "Require TEST_FOLDER to be defined")
 endif ()
-if (NOT TEST_OUTPUT)
+if (NOT TEST_OUTPUT) # the output file to capture the test program output
   message (FATAL_ERROR "Require TEST_OUTPUT to be defined")
 endif ()
 if (NOT TEST_EXPECT)
@@ -49,7 +49,7 @@ endif ()
 
 if (TEST_ENV_VAR)
   set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}")
-  message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
+  message (STATUS "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
 endif ()
 
 if (NOT TEST_INPUT)
@@ -390,6 +390,7 @@ if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY)
   )
 endif ()
 
+# cleanup the output files
 if (NOT DEFINED ENV{HDF5_NOCLEANUP})
   if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}" AND NOT TEST_SAVE)
     file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT})