Merge pull request #1216 from luxonis/feature/zoo_helper_improvements…

…_centos

Add zoo_helper CI/CD workflows for linux-x86_64 and linux-arm64

.github/workflows/zoo_helper.workflow.yml

@@ -0,0 +1,116 @@
+name: Zoo Helper
+
+on:
+  workflow_dispatch:
+
+env:
+  CMAKE_WINDOWS_SDK_VERSION: '10.0.18362.0'
+  VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
+
+jobs:
+  build-zoo-helper-linux-x86_64:
+    runs-on: ubuntu-latest
+    container:
+      image: almalinux:8
+    steps:
+      - name: Cache .hunter folder
+        uses: actions/cache@v3
+        with:
+          path: ~/.hunter
+          key: hunter-almalinux-8-v3-develop-x86_64
+
+      - name: Install git
+        run: dnf install -y git
+
+      - name: Export GitHub Actions cache environment variables
+        uses: actions/github-script@v7
+        with:
+          script: |
+           core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+           core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Install dependencies
+        run: |
+          dnf update -y
+          dnf install -y pkgconf-pkg-config bison autoconf libtool libXi-devel libXtst-devel cmake zip perl-core python39
+          dnf install -y libXrandr-devel libX11-devel libXft-devel libXext-devel flex systemd-devel
+          dnf install -y gcc-c++ automake libtool-ltdl-devel wget
+          wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz && tar -xzf nasm-2.16.01.tar.gz && cd nasm-2.16.01 && ./configure && make && make install && cd .. # install nasm - build from source
+          pip3 install jinja2
+
+      - name: Configure project
+        run: cmake -S . -B build -DDEPTHAI_ENABLE_CURL=ON -DCMAKE_BUILD_TYPE=MinSizeRel
+
+      - name: Build zoo_helper
+        run: cmake --build build --target zoo_helper --parallel 4
+
+      - name: Strip zoo_helper
+        run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html
+
+      - name: Upload zoo_helper to artifactory
+        run: |
+          bash ./ci/upload-artifactory-zoo-helper.sh
+        env:
+          ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
+          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
+          ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
+          ZOO_HELPER_PLATFORM: linux-x86_64
+
+  build-zoo-helper-linux-arm64:
+    runs-on: [self-hosted, linux, ARM64]
+    container:
+      image: arm64v8/almalinux:8
+      # Mount local hunter cache directory, instead of transfering to Github and back
+      volumes:
+        - /.hunter:/github/home/.hunter
+    env:
+      # workaround required for cache@v3, https://github.com/actions/cache/issues/1428
+      VCPKG_FORCE_SYSTEM_BINARIES: "1" # Needed so vpckg can bootstrap itself
+      VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
+      VCPKG_MAX_CONCURRENCY: "2"
+    steps:
+      - name: Install git
+        run: dnf install -y git
+
+      - name: Export GitHub Actions cache environment variables
+        uses: actions/github-script@v7
+        with:
+          script: |
+           core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+           core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Install dependencies
+        run: |
+          dnf update -y
+          dnf install -y pkgconf-pkg-config bison autoconf libtool libXi-devel libXtst-devel cmake git zip perl-core python39
+          dnf install -y libXrandr-devel libX11-devel libXft-devel libXext-devel flex systemd-devel
+          dnf install -y gcc-c++ automake libtool-ltdl-devel wget
+          wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz && tar -xzf nasm-2.16.01.tar.gz && cd nasm-2.16.01 && ./configure && make && make install && cd .. # install nasm - build from source
+          pip3 install jinja2
+          pip3 install ninja # ninja is needed for cmake on arm64
+
+      - name: Configure project
+        run: cmake -S . -B build -DDEPTHAI_ENABLE_CURL=ON -DCMAKE_BUILD_TYPE=MinSizeRel
+
+      - name: Build zoo_helper
+        run: cmake --build build --target zoo_helper --parallel 4
+
+      - name: Strip zoo_helper
+        run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html
+
+      - name: Upload zoo_helper to artifactory
+        run: |
+          bash ./ci/upload-artifactory-zoo-helper.sh
+        env:
+          ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
+          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
+          ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
+          ZOO_HELPER_PLATFORM: linux-arm64

CMakeLists.txt

@@ -503,9 +503,55 @@ set_target_properties(${TARGET_CORE_NAME} PROPERTIES EXPORT_NAME ${TARGET_CORE_A
 list(APPEND targets_to_export ${TARGET_CORE_NAME})
 
 # Add model_zoo helper binary
-add_executable(zoo_helper src/modelzoo/zoo_helper.cpp)
+find_package(fmt REQUIRED)
+find_package(yaml-cpp REQUIRED)
+set(ZOO_HELPER_SOURCES
+    src/modelzoo/zoo_helper.cpp
+    src/modelzoo/Zoo.cpp
+    src/modelzoo/NNModelDescription.cpp
+    src/utility/Environment.cpp
+    src/utility/Logging.cpp
+)
+set(ZOO_HELPER_LINK_LIBRARIES
+    nlohmann_json::nlohmann_json
+    fmt::fmt
+    yaml-cpp::yaml-cpp
+)
+add_executable(zoo_helper ${ZOO_HELPER_SOURCES})
 target_compile_definitions(zoo_helper PRIVATE DEPTHAI_TARGET_CORE)
-target_link_libraries(zoo_helper PRIVATE ${TARGET_CORE_NAME})
+target_link_libraries(zoo_helper PRIVATE ${ZOO_HELPER_LINK_LIBRARIES})
+target_include_directories(zoo_helper
+    PRIVATE
+        # Relative path to include directories after installed
+        "$<INSTALL_INTERFACE:include>"
+        "$<INSTALL_INTERFACE:include/${DEPTHAI_SHARED_3RDPARTY_HEADERS_PATH}>"
+
+        # Build time path to include directories
+        "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
+        "$<BUILD_INTERFACE:${DEPTHAI_SHARED_PUBLIC_INCLUDE}>"
+        "$<BUILD_INTERFACE:${DEPTHAI_BOOTLOADER_SHARED_PUBLIC_INCLUDE}>"
+
+        "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai>"
+        "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src>"
+        "$<BUILD_INTERFACE:${DEPTHAI_SHARED_INCLUDE}>"
+        "$<BUILD_INTERFACE:${DEPTHAI_BOOTLOADER_SHARED_INCLUDE}>"
+)
+target_include_directories(zoo_helper SYSTEM
+    PRIVATE
+        "$<BUILD_INTERFACE:${DEPTHAI_SHARED_3RDPARTY_INCLUDE}>"
+)
+
+# Set compiler features (c++17), and disables extensions (g++17)
+set_property(TARGET zoo_helper PROPERTY CXX_STANDARD 17)
+set_property(TARGET zoo_helper PROPERTY CXX_STANDARD_REQUIRED ON)
+set_property(TARGET zoo_helper PROPERTY CXX_EXTENSIONS OFF)
+# Add interface transitive property (C++17)
+if(${CMAKE_VERSION} VERSION_LESS "3.8.0")
+    target_compile_features(zoo_helper INTERFACE cxx_generic_lambdas)
+else()
+    target_compile_features(zoo_helper INTERFACE cxx_std_17)
+endif()
+
 list(APPEND targets_to_export zoo_helper)
 
 # Add default flags to core
@@ -799,6 +845,11 @@ if(DEPTHAI_ENABLE_CURL)
         cpr::cpr
     )
     target_compile_definitions(${TARGET_CORE_NAME} PRIVATE DEPTHAI_ENABLE_CURL)
+    target_link_libraries(zoo_helper PRIVATE
+        CURL::libcurl
+        cpr::cpr
+    )
+    target_compile_definitions(zoo_helper PRIVATE DEPTHAI_ENABLE_CURL)
 endif()
 
 # Add compile & CMake definitions

ci/upload-artifactory-zoo-helper.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Download jfrog cli
+curl -fL https://getcli.jfrog.io | sh
+
+# Set paths
+export PATH_PREFIX=luxonis-depthai-helper-binaries/zoo_helper/$ZOO_HELPER_PLATFORM
+export ZOO_HELPER_BINARY_LOCAL_PATH=build/zoo_helper
+
+# Get git hash  
+git config --global --add safe.directory $(pwd)
+export ZOO_HELPER_GIT_HASH=$(git rev-parse HEAD)
+
+echo "----------------------------------------"
+echo "PATH_PREFIX: $PATH_PREFIX"
+echo "ZOO_HELPER_BINARY_LOCAL_PATH: $ZOO_HELPER_BINARY_LOCAL_PATH"
+echo "ZOO_HELPER_GIT_HASH: $ZOO_HELPER_GIT_HASH"
+echo "zoo_helper binary size: $(du -sh $ZOO_HELPER_BINARY_LOCAL_PATH)"
+echo "----------------------------------------"
+
+# Upload binary to artifactory
+./jfrog config add --artifactory-url=$ARTIFACTORY_URL --user=$ARTIFACTORY_USER --password=$ARTIFACTORY_PASS
+./jfrog rt u "$ZOO_HELPER_BINARY_LOCAL_PATH" "$PATH_PREFIX/$ZOO_HELPER_GIT_HASH/"

src/modelzoo/zoo_helper.cpp

@@ -1,21 +1,45 @@
+#include <argparse/argparse.hpp>
 #include <iostream>
-#include <string>
 
 #include "depthai/modelzoo/Zoo.hpp"
 
 int main(int argc, char* argv[]) {
-    // Check if the number of arguments is correct
-    if(argc != 3) {
-        std::cout << "Usage: zoo_helper folder_with_yaml_files cache_folder" << std::endl;
+    // Initialize parser
+    argparse::ArgumentParser program("DepthAI Model Zoo Helper");
+
+    // Add arguments
+    const std::string DEFAULT_YAML_FOLDER = ".";
+    program.add_argument("--yaml_folder").default_value(DEFAULT_YAML_FOLDER).help("Folder with YAML files describing models to download");
+
+    const std::string DEFAULT_CACHE_FOLDER = dai::MODEL_ZOO_DEFAULT_CACHE_DIRECTORY;
+    program.add_argument("--cache_folder").default_value(DEFAULT_CACHE_FOLDER).help("Cache folder to download models into");
+
+    const std::string DEFAULT_API_KEY = "";
+    program.add_argument("--api_key").default_value(DEFAULT_API_KEY).help("API key to use for downloading models");
+
+    // Parse arguments
+    try {
+        program.parse_args(argc, argv);
+    } catch(const std::runtime_error& err) {
+        std::cerr << err.what() << std::endl;
+        std::cerr << program;
         return EXIT_FAILURE;
     }
 
     // Unpack arguments
-    const std::string yamlFolder = argv[1];
-    const std::string cacheFolder = argv[2];
+    auto yamlFolder = program.get<std::string>("--yaml_folder");
+    auto cacheFolder = program.get<std::string>("--cache_folder");
+    auto apiKey = program.get<std::string>("--api_key");
+
+    // Print arguments
+    std::cout << "Downloading models defined in yaml files in folder: " << yamlFolder << std::endl;
+    std::cout << "Downloading models into cache folder: " << cacheFolder << std::endl;
+    if(!apiKey.empty()) {
+        std::cout << "Using API key: " << apiKey << std::endl;
+    }
 
     // Download models
-    dai::downloadModelsFromZoo(yamlFolder, cacheFolder);
+    dai::downloadModelsFromZoo(yamlFolder, cacheFolder, apiKey);
 
     return EXIT_SUCCESS;
 }

vcpkg.json

@@ -17,6 +17,8 @@
         "fp16",
         "mp4v2",
         "libnop",
+        "argparse",
+        "fmt",
         "neargye-semver",
         "magic-enum",
         {