diff --git a/.github/workflows/zoo_helper.workflow.yml b/.github/workflows/zoo_helper.workflow.yml new file mode 100644 index 000000000..349b692c8 --- /dev/null +++ b/.github/workflows/zoo_helper.workflow.yml @@ -0,0 +1,116 @@ +name: Zoo Helper + +on: + workflow_dispatch: + +env: + CMAKE_WINDOWS_SDK_VERSION: '10.0.18362.0' + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" + +jobs: + build-zoo-helper-linux-x86_64: + runs-on: ubuntu-latest + container: + image: almalinux:8 + steps: + - name: Cache .hunter folder + uses: actions/cache@v3 + with: + path: ~/.hunter + key: hunter-almalinux-8-v3-develop-x86_64 + + - name: Install git + run: dnf install -y git + + - name: Export GitHub Actions cache environment variables + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install dependencies + run: | + dnf update -y + dnf install -y pkgconf-pkg-config bison autoconf libtool libXi-devel libXtst-devel cmake zip perl-core python39 + dnf install -y libXrandr-devel libX11-devel libXft-devel libXext-devel flex systemd-devel + dnf install -y gcc-c++ automake libtool-ltdl-devel wget + wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz && tar -xzf nasm-2.16.01.tar.gz && cd nasm-2.16.01 && ./configure && make && make install && cd .. # install nasm - build from source + pip3 install jinja2 + + - name: Configure project + run: cmake -S . -B build -DDEPTHAI_ENABLE_CURL=ON -DCMAKE_BUILD_TYPE=MinSizeRel + + - name: Build zoo_helper + run: cmake --build build --target zoo_helper --parallel 4 + + - name: Strip zoo_helper + run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html + + - name: Upload zoo_helper to artifactory + run: | + bash ./ci/upload-artifactory-zoo-helper.sh + env: + ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }} + ZOO_HELPER_PLATFORM: linux-x86_64 + + build-zoo-helper-linux-arm64: + runs-on: [self-hosted, linux, ARM64] + container: + image: arm64v8/almalinux:8 + # Mount local hunter cache directory, instead of transfering to Github and back + volumes: + - /.hunter:/github/home/.hunter + env: + # workaround required for cache@v3, https://github.com/actions/cache/issues/1428 + VCPKG_FORCE_SYSTEM_BINARIES: "1" # Needed so vpckg can bootstrap itself + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" + VCPKG_MAX_CONCURRENCY: "2" + steps: + - name: Install git + run: dnf install -y git + + - name: Export GitHub Actions cache environment variables + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install dependencies + run: | + dnf update -y + dnf install -y pkgconf-pkg-config bison autoconf libtool libXi-devel libXtst-devel cmake git zip perl-core python39 + dnf install -y libXrandr-devel libX11-devel libXft-devel libXext-devel flex systemd-devel + dnf install -y gcc-c++ automake libtool-ltdl-devel wget + wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz && tar -xzf nasm-2.16.01.tar.gz && cd nasm-2.16.01 && ./configure && make && make install && cd .. # install nasm - build from source + pip3 install jinja2 + pip3 install ninja # ninja is needed for cmake on arm64 + + - name: Configure project + run: cmake -S . -B build -DDEPTHAI_ENABLE_CURL=ON -DCMAKE_BUILD_TYPE=MinSizeRel + + - name: Build zoo_helper + run: cmake --build build --target zoo_helper --parallel 4 + + - name: Strip zoo_helper + run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html + + - name: Upload zoo_helper to artifactory + run: | + bash ./ci/upload-artifactory-zoo-helper.sh + env: + ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }} + ZOO_HELPER_PLATFORM: linux-arm64 diff --git a/CMakeLists.txt b/CMakeLists.txt index eb92cddff..e48263a28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -503,9 +503,55 @@ set_target_properties(${TARGET_CORE_NAME} PROPERTIES EXPORT_NAME ${TARGET_CORE_A list(APPEND targets_to_export ${TARGET_CORE_NAME}) # Add model_zoo helper binary -add_executable(zoo_helper src/modelzoo/zoo_helper.cpp) +find_package(fmt REQUIRED) +find_package(yaml-cpp REQUIRED) +set(ZOO_HELPER_SOURCES + src/modelzoo/zoo_helper.cpp + src/modelzoo/Zoo.cpp + src/modelzoo/NNModelDescription.cpp + src/utility/Environment.cpp + src/utility/Logging.cpp +) +set(ZOO_HELPER_LINK_LIBRARIES + nlohmann_json::nlohmann_json + fmt::fmt + yaml-cpp::yaml-cpp +) +add_executable(zoo_helper ${ZOO_HELPER_SOURCES}) target_compile_definitions(zoo_helper PRIVATE DEPTHAI_TARGET_CORE) -target_link_libraries(zoo_helper PRIVATE ${TARGET_CORE_NAME}) +target_link_libraries(zoo_helper PRIVATE ${ZOO_HELPER_LINK_LIBRARIES}) +target_include_directories(zoo_helper + PRIVATE + # Relative path to include directories after installed + "$" + "$" + + # Build time path to include directories + "$" + "$" + "$" + + "$" + "$" + "$" + "$" +) +target_include_directories(zoo_helper SYSTEM + PRIVATE + "$" +) + +# Set compiler features (c++17), and disables extensions (g++17) +set_property(TARGET zoo_helper PROPERTY CXX_STANDARD 17) +set_property(TARGET zoo_helper PROPERTY CXX_STANDARD_REQUIRED ON) +set_property(TARGET zoo_helper PROPERTY CXX_EXTENSIONS OFF) +# Add interface transitive property (C++17) +if(${CMAKE_VERSION} VERSION_LESS "3.8.0") + target_compile_features(zoo_helper INTERFACE cxx_generic_lambdas) +else() + target_compile_features(zoo_helper INTERFACE cxx_std_17) +endif() + list(APPEND targets_to_export zoo_helper) # Add default flags to core @@ -799,6 +845,11 @@ if(DEPTHAI_ENABLE_CURL) cpr::cpr ) target_compile_definitions(${TARGET_CORE_NAME} PRIVATE DEPTHAI_ENABLE_CURL) + target_link_libraries(zoo_helper PRIVATE + CURL::libcurl + cpr::cpr + ) + target_compile_definitions(zoo_helper PRIVATE DEPTHAI_ENABLE_CURL) endif() # Add compile & CMake definitions diff --git a/ci/upload-artifactory-zoo-helper.sh b/ci/upload-artifactory-zoo-helper.sh new file mode 100644 index 000000000..5908eac93 --- /dev/null +++ b/ci/upload-artifactory-zoo-helper.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Download jfrog cli +curl -fL https://getcli.jfrog.io | sh + +# Set paths +export PATH_PREFIX=luxonis-depthai-helper-binaries/zoo_helper/$ZOO_HELPER_PLATFORM +export ZOO_HELPER_BINARY_LOCAL_PATH=build/zoo_helper + +# Get git hash +git config --global --add safe.directory $(pwd) +export ZOO_HELPER_GIT_HASH=$(git rev-parse HEAD) + +echo "----------------------------------------" +echo "PATH_PREFIX: $PATH_PREFIX" +echo "ZOO_HELPER_BINARY_LOCAL_PATH: $ZOO_HELPER_BINARY_LOCAL_PATH" +echo "ZOO_HELPER_GIT_HASH: $ZOO_HELPER_GIT_HASH" +echo "zoo_helper binary size: $(du -sh $ZOO_HELPER_BINARY_LOCAL_PATH)" +echo "----------------------------------------" + +# Upload binary to artifactory +./jfrog config add --artifactory-url=$ARTIFACTORY_URL --user=$ARTIFACTORY_USER --password=$ARTIFACTORY_PASS +./jfrog rt u "$ZOO_HELPER_BINARY_LOCAL_PATH" "$PATH_PREFIX/$ZOO_HELPER_GIT_HASH/" diff --git a/src/modelzoo/zoo_helper.cpp b/src/modelzoo/zoo_helper.cpp index 5eeb3fb39..96f05ca0e 100644 --- a/src/modelzoo/zoo_helper.cpp +++ b/src/modelzoo/zoo_helper.cpp @@ -1,21 +1,45 @@ +#include #include -#include #include "depthai/modelzoo/Zoo.hpp" int main(int argc, char* argv[]) { - // Check if the number of arguments is correct - if(argc != 3) { - std::cout << "Usage: zoo_helper folder_with_yaml_files cache_folder" << std::endl; + // Initialize parser + argparse::ArgumentParser program("DepthAI Model Zoo Helper"); + + // Add arguments + const std::string DEFAULT_YAML_FOLDER = "."; + program.add_argument("--yaml_folder").default_value(DEFAULT_YAML_FOLDER).help("Folder with YAML files describing models to download"); + + const std::string DEFAULT_CACHE_FOLDER = dai::MODEL_ZOO_DEFAULT_CACHE_DIRECTORY; + program.add_argument("--cache_folder").default_value(DEFAULT_CACHE_FOLDER).help("Cache folder to download models into"); + + const std::string DEFAULT_API_KEY = ""; + program.add_argument("--api_key").default_value(DEFAULT_API_KEY).help("API key to use for downloading models"); + + // Parse arguments + try { + program.parse_args(argc, argv); + } catch(const std::runtime_error& err) { + std::cerr << err.what() << std::endl; + std::cerr << program; return EXIT_FAILURE; } // Unpack arguments - const std::string yamlFolder = argv[1]; - const std::string cacheFolder = argv[2]; + auto yamlFolder = program.get("--yaml_folder"); + auto cacheFolder = program.get("--cache_folder"); + auto apiKey = program.get("--api_key"); + + // Print arguments + std::cout << "Downloading models defined in yaml files in folder: " << yamlFolder << std::endl; + std::cout << "Downloading models into cache folder: " << cacheFolder << std::endl; + if(!apiKey.empty()) { + std::cout << "Using API key: " << apiKey << std::endl; + } // Download models - dai::downloadModelsFromZoo(yamlFolder, cacheFolder); + dai::downloadModelsFromZoo(yamlFolder, cacheFolder, apiKey); return EXIT_SUCCESS; } diff --git a/vcpkg.json b/vcpkg.json index c39471985..a1cb941c6 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -17,6 +17,8 @@ "fp16", "mp4v2", "libnop", + "argparse", + "fmt", "neargye-semver", "magic-enum", {