From 8e8689b9dc417101013f80bd9b833791b41de9a5 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Wed, 28 Jun 2023 21:47:12 +0300 Subject: [PATCH 01/26] Improved speed of docker image building - Introduced `docker bake` as main build mechanism with fallback to `docker build` if bake is not available - Adjusted build process by utilising multistage builds and paralleling builds - Optimised cache usage by separating/moving parameters that are changed often (e.g. build hash, stamp) - Fixed bug with `docker/sdk export` when export options was ignored when `-v` option was used at the same time - Added labels for all images containing project revision, project name and SDK revision - Adjusted maintenance mode to be controlled by environment variable: SPRYKER_MAINTENANCE_MODE_ENABLED=1 docker/sdk run --- .dockerignore.default | 1 + bin/command/build/baked/build.sh | 9 +- bin/command/build/baked/export.sh | 48 ++-- bin/command/build/mount/build.sh | 9 +- bin/command/install/bootstrap.sh | 9 +- bin/command/install/config.sh | 5 +- bin/registry/help.sh | 1 + bin/sdk/assets/baked.sh | 75 ++---- bin/sdk/assets/mount.sh | 4 - bin/sdk/compose.sh | 3 +- bin/sdk/images/baked.sh | 27 +- bin/sdk/images/baked/ecr.sh | 64 ----- bin/sdk/images/common.sh | 237 ++---------------- bin/sdk/images/destination/ecr.sh | 5 + bin/sdk/images/destination/print.sh | 5 + bin/sdk/images/engine/bake.sh | 26 ++ bin/sdk/images/engine/build.sh | 82 ++++++ bin/sdk/images/mount.sh | 28 +-- generator/Dockerfile | 2 +- generator/deploy-file-generator/Dockerfile | 2 +- generator/index.php | 33 ++- generator/src/templates/deploy.bash.twig | 29 ++- .../src/templates/docker-compose.yml.twig | 1 + generator/src/templates/mount/nfs.yml.twig | 2 +- .../dashboard/default/dashboard.yml.twig | 8 +- .../tideways/default/tideways.yml.twig | 8 +- images/baked/.gitkeep | 0 .../Dockerfile => application.Dockerfile} | 46 ++-- images/baked/assets/Dockerfile | 57 ++++- images/baked/frontend/Dockerfile | 21 +- .../{cli/Dockerfile => pipeline.Dockerfile} | 27 +- images/common/application-local/Dockerfile | 7 +- ...Dockerfile.twig => application.Dockerfile} | 32 +-- images/common/args.Dockerfile | 4 + .../common/{cli/Dockerfile => cli.Dockerfile} | 49 +--- images/common/frontend.dev.Dockerfile | 2 + images/common/frontend/Dockerfile | 31 +-- images/common/gateway/Dockerfile | 19 +- images/common/node.Dockerfile | 5 + images/common/pipeline.Dockerfile | 41 +++ .../{ => services}/dashboard/Dockerfile | 1 + .../common/services/jenkins/export/Dockerfile | 63 +++-- images/common/services/tideways/Dockerfile | 3 +- .../Dockerfile => application.Dockerfile} | 9 +- images/debug/cli/Dockerfile | 10 +- images/debug/frontend/Dockerfile | 10 +- images/export/.gitkeep | 0 images/mount/.gitkeep | 0 images/mount/cli/Dockerfile | 13 +- images/mount/frontend/Dockerfile | 15 +- images/templates/baked/Dockerfile.twig | 73 ++++++ images/templates/baked/npm.Dockerfile | 9 + images/templates/export/Dockerfile.twig | 52 ++++ images/templates/mount/Dockerfile.twig | 65 +++++ .../mount/application.Dockerfile} | 26 +- images/templates/plans/baked.docker-bake.hcl | 1 + images/templates/plans/baked.sh | 17 ++ images/templates/plans/common.docker-bake.hcl | 59 +++++ images/templates/plans/export.docker-bake.hcl | 45 ++++ images/templates/plans/export.sh | 17 ++ images/templates/plans/mount.docker-bake.hcl | 1 + images/templates/plans/mount.sh | 1 + .../templates/plans/runtime.docker-bake.hcl | 45 ++++ images/templates/plans/tag.ecr.export.twig | 2 + images/templates/plans/tag.print.export.twig | 2 + images/templates/plans/tag.print.runtime.twig | 2 + sdk | 1 + 67 files changed, 959 insertions(+), 647 deletions(-) delete mode 100644 bin/sdk/images/baked/ecr.sh create mode 100644 bin/sdk/images/destination/ecr.sh create mode 100644 bin/sdk/images/destination/print.sh create mode 100644 bin/sdk/images/engine/bake.sh create mode 100644 bin/sdk/images/engine/build.sh create mode 100644 images/baked/.gitkeep rename images/baked/{application/Dockerfile => application.Dockerfile} (71%) rename images/baked/{cli/Dockerfile => pipeline.Dockerfile} (53%) rename images/common/{application/Dockerfile.twig => application.Dockerfile} (61%) create mode 100644 images/common/args.Dockerfile rename images/common/{cli/Dockerfile => cli.Dockerfile} (53%) create mode 100644 images/common/frontend.dev.Dockerfile create mode 100644 images/common/node.Dockerfile create mode 100644 images/common/pipeline.Dockerfile rename images/common/{ => services}/dashboard/Dockerfile (93%) rename images/debug/{application/Dockerfile => application.Dockerfile} (79%) create mode 100644 images/export/.gitkeep create mode 100644 images/mount/.gitkeep create mode 100644 images/templates/baked/Dockerfile.twig create mode 100644 images/templates/baked/npm.Dockerfile create mode 100644 images/templates/export/Dockerfile.twig create mode 100644 images/templates/mount/Dockerfile.twig rename images/{mount/application/Dockerfile => templates/mount/application.Dockerfile} (66%) create mode 100644 images/templates/plans/baked.docker-bake.hcl create mode 100644 images/templates/plans/baked.sh create mode 100644 images/templates/plans/common.docker-bake.hcl create mode 100644 images/templates/plans/export.docker-bake.hcl create mode 100644 images/templates/plans/export.sh create mode 100644 images/templates/plans/mount.docker-bake.hcl create mode 100644 images/templates/plans/mount.sh create mode 100644 images/templates/plans/runtime.docker-bake.hcl create mode 100644 images/templates/plans/tag.ecr.export.twig create mode 100644 images/templates/plans/tag.print.export.twig create mode 100644 images/templates/plans/tag.print.runtime.twig diff --git a/.dockerignore.default b/.dockerignore.default index 7263346bf..19db8e938 100644 --- a/.dockerignore.default +++ b/.dockerignore.default @@ -1,4 +1,5 @@ .git* +**/.git .idea .DS_Store node_modules diff --git a/bin/command/build/baked/build.sh b/bin/command/build/baked/build.sh index 28754a4be..1839a04cb 100644 --- a/bin/command/build/baked/build.sh +++ b/bin/command/build/baked/build.sh @@ -13,18 +13,15 @@ function Command::build() { subCommand=${1} case ${subCommand} in images | image) - Images::buildApplication --force - Assets::build - Images::buildFrontend --force + Images::build --force ;; assets | asset) + Images::build --force Assets::build --force - Images::buildFrontend --force ;; '') - Images::buildApplication --force + Images::build --force Assets::build --force - Images::buildFrontend --force ;; *) Console::error "Unknown build target '${subCommand}' is occurred. No action." >&2 diff --git a/bin/command/build/baked/export.sh b/bin/command/build/baked/export.sh index fafe5d109..f522d635d 100644 --- a/bin/command/build/baked/export.sh +++ b/bin/command/build/baked/export.sh @@ -2,7 +2,7 @@ Registry::addCommand "export" "Command::export" -Registry::Help::command -c "export images" -a "[-t ]" "Builds prod-like images (Yves, Zed, Glue, Frontend)." +Registry::Help::command -c "export images" -a "[-t ] [-d ecr]" "Builds prod-like images (Yves, Zed, Glue, Frontend)." Registry::Help::command -c "export assets" -a "[-t ] [-p ]" "[DEPRECATED] Builds assets and export as archives stored by given path." function _assertDestinationDirectory() { @@ -13,10 +13,12 @@ function _assertDestinationDirectory() { } function Command::export() { + local OPTIND=0 + local opt local subCommand='' local tag=${SPRYKER_DOCKER_TAG} local destinationPath='./' - local pushDestination='' + local pushDestination='print' subCommand=${1} shift || true @@ -32,14 +34,6 @@ function Command::export() { ;; d) pushDestination=${OPTARG} - local pushDestinationPath="sdk/images/baked/${pushDestination}.sh" - local pathToFile="${DEPLOYMENT_PATH}/bin/${pushDestinationPath}" - if [ ! -f "${pathToFile}" ]; then - Console::error "\nUnknown export images destination - '${OPTARG}'." - exit 1 - fi - - import ${pushDestinationPath} ;; # Unknown option specified \?) @@ -60,33 +54,27 @@ function Command::export() { done shift $((OPTIND - 1)) + local pushDestinationPath="sdk/images/destination/${pushDestination}.sh" + if [ ! -f "${DEPLOYMENT_PATH}/bin/${pushDestinationPath}" ]; then + Console::error "\nUnknown export images destination - '${pushDestination}'." + exit 1 + fi + + import ${pushDestinationPath} + case ${subCommand} in asset | assets) - Console::warn 'This command is DEPRECATED. Please, use just "export".' + Console::warn 'This command is DEPRECATED. Please, use just "export images".' _assertDestinationDirectory "${destinationPath}" - Images::buildApplication --force - Assets::build --force - Images::buildFrontend --force - Assets::export "${tag}" "${destinationPath}" + Images::export "${tag}" "print" + Assets::export "${tag}" "print" "${destinationPath}" ;; image | images) - Console::verbose "${INFO}Build and export images${NC}" - Images::buildApplication --force - Images::tagApplications "${tag}" - Assets::build --force - Images::buildFrontend --force - Images::tagFrontend "${tag}" - - if [ -n "${pushDestination}" ]; then - Images::push "${tag}" - fi - - if [ -z "${pushDestination}" ]; then - Images::printAll "${tag}" - fi + Images::export "${tag}" "${pushDestination}" + Images::print "${tag}" "${pushDestination}" ;; *) - Console::error "Unknown export '${subCommand}' is occurred. No action. Usage: ${HELP_SCR}${SELF_SCRIPT} export images [-t ]" >&2 + Console::error "Unknown export '${subCommand}' is occurred. No action. Usage: ${HELP_SCR}${SELF_SCRIPT} export images [-t ] [-d ecr]" >&2 exit 1 ;; esac diff --git a/bin/command/build/mount/build.sh b/bin/command/build/mount/build.sh index e37686ce5..4d3d99a61 100644 --- a/bin/command/build/mount/build.sh +++ b/bin/command/build/mount/build.sh @@ -11,23 +11,18 @@ function Command::build() { subCommand=${1} case ${subCommand} in images | image) - Images::buildApplication --force - Codebase::build - Assets::build - Images::buildFrontend --force + Images::build --force ;; codebase | code) Codebase::build --force ;; assets | asset) Assets::build --force - Images::buildFrontend --force ;; '') - Images::buildApplication --force + Images::build --force Codebase::build --force Assets::build --force - Images::buildFrontend --force ;; *) Console::error "Unknown build target '${subCommand}' is occurred. No action." >&2 diff --git a/bin/command/install/bootstrap.sh b/bin/command/install/bootstrap.sh index d6bc1d208..1fb64a346 100644 --- a/bin/command/install/bootstrap.sh +++ b/bin/command/install/bootstrap.sh @@ -13,6 +13,8 @@ Registry::Help::command -s -c "bootstrap | boot" -a "[-v] " "P Registry::Help::command -s -c "bootstrap | boot" -a "[-v]" "Prepares all the files to run the application based on ${HELP_HIGH}deploy.local.yml${HELP_DESC} or ${HELP_HIGH}deploy.yml${HELP_DESC}." function Command::bootstrap() { + local OPTIND=0 + local opt while getopts ":vsx" opt; do case ${opt} in @@ -42,6 +44,7 @@ function Command::bootstrap() { shift $((OPTIND - 1)) local gitHash=$(git rev-parse --verify HEAD 2>/dev/null || true) + local SPRYKER_SDK_REVISION="$(cd ${SOURCE_DIR} && git rev-parse --short HEAD 2>/dev/null || echo '')" local tmpDeploymentDir="${SOURCE_DIR}/deployment/_tmp" local defaultProjectYaml=$([ -f "./deploy.local.yml" ] && echo -n "./deploy.local.yml" || echo -n "./deploy.yml") local projectYaml=${1:-${defaultProjectYaml}} @@ -75,11 +78,12 @@ function Command::bootstrap() { Console::info "Using ${projectYaml}" local USER_FULL_ID=$(Environment::getFullUserId) + local USER_UID="${USER_FULL_ID%%:*}" Console::verbose::start "Building generator..." docker build -t spryker_docker_sdk \ -f "${SOURCE_DIR}/generator/Dockerfile" \ - --build-arg="USER_UID=${USER_FULL_ID%%:*}" \ + --build-arg="USER_UID=${USER_UID}" \ -q \ "${SOURCE_DIR}/generator" >/dev/null Console::end "[DONE]" @@ -103,12 +107,13 @@ function Command::bootstrap() { # To support root user local userToRun=("-u" "${USER_FULL_ID}") - if [ "${USER_FULL_ID%%:*}" != '0' ]; then + if [ "${USER_UID}" != '0' ]; then userToRun=() fi docker run -i --rm "${userToRun[@]}" \ -e SPRYKER_PLATFORM_IMAGE="${SPRYKER_PLATFORM_IMAGE:-""}" \ -e SPRYKER_DOCKER_SDK_PLATFORM="${_PLATFORM}" \ + -e SPRYKER_SDK_REVISION="${SPRYKER_SDK_REVISION}" \ -e SPRYKER_DOCKER_SDK_DEPLOYMENT_DIR="${DESTINATION_DIR}" \ -e VERBOSE="${VERBOSE}" \ -v "${tmpDeploymentDir}":/data/deployment:rw \ diff --git a/bin/command/install/config.sh b/bin/command/install/config.sh index e34738968..87d512f1b 100644 --- a/bin/command/install/config.sh +++ b/bin/command/install/config.sh @@ -23,11 +23,12 @@ function Command::config() { Console::info "Using ${projectYaml}" local USER_FULL_ID=$(Environment::getFullUserId) + local USER_UID="${USER_FULL_ID%%:*}" Console::verbose::start "Building generator..." docker build -t spryker_docker_sdk \ -f "${SOURCE_DIR}/generator/deploy-file-generator/Dockerfile" \ - --build-arg="USER_UID=${USER_FULL_ID%%:*}" \ + --build-arg="USER_UID=${USER_UID}" \ -q \ "${SOURCE_DIR}/generator" >/dev/null @@ -42,7 +43,7 @@ function Command::config() { # To support root user local userToRun=("-u" "${USER_FULL_ID}") - if [ "${USER_FULL_ID%%:*}" != '0' ]; then + if [ "${USER_UID}" != '0' ]; then userToRun=() fi docker run -i --rm "${userToRun[@]}" \ diff --git a/bin/registry/help.sh b/bin/registry/help.sh index 1c375a9fe..b39da7088 100644 --- a/bin/registry/help.sh +++ b/bin/registry/help.sh @@ -19,6 +19,7 @@ function Registry::Help::separator() { function Registry::Help::command() { local OPTIND=0 + local opt local OPTARG='' local OPTERR='' local envs='' diff --git a/bin/sdk/assets/baked.sh b/bin/sdk/assets/baked.sh index 7005ef6a4..8de068646 100755 --- a/bin/sdk/assets/baked.sh +++ b/bin/sdk/assets/baked.sh @@ -2,10 +2,11 @@ function Assets::export() { # deprecated - local tag=${1} - local destinationPath=${2%/} + local TARGET_TAG=${1:-${SPRYKER_DOCKER_TAG}} + local destination=${2} + local destinationPath=${3%/} - local builderAssetsImage=$(Assets::getImageTag) + local builderAssetsImage local dockerAssetsTmpDirectory="/_tmp" local projectDockerAssetsTmpDirectory=${DEPLOYMENT_DIR}${dockerAssetsTmpDirectory} @@ -19,6 +20,18 @@ function Assets::export() { Console::start "Preparing assets archives..." + source ${DEPLOYMENT_PATH}/images/export/${destination}.sh + + local targetData + for targetData in "${TARGET_TAGS[@]}"; do + eval "${targetData}" + + if [ "$TARGET" == 'frontend' ]; then + builderAssetsImage="${TAGS[0]}" + break + fi + done + # To support root user local userToRun=("-u" "${USER_FULL_ID}") if [ "${USER_FULL_ID%%:*}" != '0' ]; then @@ -29,7 +42,7 @@ function Assets::export() { -v "${DEPLOYMENT_DIR}/bin:/data/standalone" \ -v "${projectDockerAssetsTmpDirectory}:/data${dockerAssetsTmpDirectory}" \ --entrypoint='' \ - --name="${SPRYKER_DOCKER_PREFIX}_builder_assets" \ + --name="${SPRYKER_DOCKER_PREFIX}_frontend" \ "${builderAssetsImage}" \ sh -c "${command}" 2>&1 @@ -50,58 +63,12 @@ function Assets::export() { rm -rf "${projectDockerAssetsTmpDirectory}" } -function Assets::getImageTag() { - echo -n "${SPRYKER_DOCKER_PREFIX}_builder_assets:${SPRYKER_DOCKER_TAG}-${SPRYKER_REPOSITORY_HASH}" -} - function Assets::areBuilt() { - Console::start "Checking assets are built..." - - local builderAssetsImage=$(Assets::getImageTag) - - if docker image inspect "${builderAssetsImage}" >/dev/null 2>&1; then - Console::end "[BUILT]" - return "${TRUE}" - fi - - return "${FALSE}" + # Do nothing as everything is built before. Could be changed once assets are separated + : } function Assets::build() { - - local force='' - if [ "$1" == '--force' ]; then - force=1 - shift || true - fi - - if [ -z "${force}" ] && Assets::areBuilt; then - return "${TRUE}" - fi - - Console::start "Cleaning old assets..." - - # ${XARGS_NO_RUN_IF_EMPTY} must be without quotes - # shellcheck disable=SC2086 - docker images --filter "reference=${SPRYKER_DOCKER_PREFIX}_builder_assets:${SPRYKER_DOCKER_TAG}*" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f - - Console::end "[DONE]" - Console::start "Building assets..." - - local builderAssetsImage=$(Assets::getImageTag) - local cliImage="${SPRYKER_DOCKER_PREFIX}_cli:${SPRYKER_DOCKER_TAG}" - local mode=${SPRYKER_ASSETS_MODE:-development} - - docker build \ - -t "${builderAssetsImage}" \ - -f "${DEPLOYMENT_PATH}/images/baked/assets/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${cliImage}" \ - --build-arg "SPRYKER_ASSETS_MODE=${mode}" \ - --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \ - . 1>&2 - - Console::end "[DONE]" + # Do nothing as everything is built before. Could be changed once assets are separated + : } diff --git a/bin/sdk/assets/mount.sh b/bin/sdk/assets/mount.sh index 439567e0b..c9f512d1a 100644 --- a/bin/sdk/assets/mount.sh +++ b/bin/sdk/assets/mount.sh @@ -6,10 +6,6 @@ function Assets::export() { return "${FALSE}" } -function Assets::getImageTag() { - echo -n "${SPRYKER_DOCKER_PREFIX}_cli:${SPRYKER_DOCKER_TAG}" -} - function Assets::areBuilt() { Console::start "Checking assets are built..." diff --git a/bin/sdk/compose.sh b/bin/sdk/compose.sh index b972a182d..d058a3244 100644 --- a/bin/sdk/compose.sh +++ b/bin/sdk/compose.sh @@ -141,10 +141,9 @@ function Compose::up() { Compose::cleanSourceDirectory fi - Images::buildApplication ${noCache} ${doBuild} + Images::build ${noCache} ${doBuild} Codebase::build ${noCache} ${doBuild} Assets::build ${noCache} ${doAssets} - Images::buildFrontend ${noCache} ${doBuild} Compose::run --build Compose::command restart frontend gateway diff --git a/bin/sdk/images/baked.sh b/bin/sdk/images/baked.sh index a38ac6488..e61571697 100644 --- a/bin/sdk/images/baked.sh +++ b/bin/sdk/images/baked.sh @@ -2,7 +2,7 @@ import sdk/images/common.sh -function Images::buildApplication() { +function Images::build() { for arg in "${@}"; do case "${arg}" in '--force') @@ -12,31 +12,14 @@ function Images::buildApplication() { # TODO implement --no-cache for build images ;; *) - Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::buildApplication." + Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::build." ;; esac done - Images::_buildApp baked - Images::tagApplications "${SPRYKER_DOCKER_TAG}" + Images::_build baked } -function Images::buildFrontend() { - for arg in "${@}"; do - case "${arg}" in - '--force') - # it is always it. - ;; - '--no-cache') - # TODO implement --no-cache for build images - ;; - *) - Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::buildFrontend." - ;; - esac - done - - Images::_buildFrontend baked - Images::_buildGateway - Images::tagFrontend "${SPRYKER_DOCKER_TAG}" +function Images::export() { + Images::_build export "${@}" } diff --git a/bin/sdk/images/baked/ecr.sh b/bin/sdk/images/baked/ecr.sh deleted file mode 100644 index 7931bfc83..000000000 --- a/bin/sdk/images/baked/ecr.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -import sdk/images/baked.sh - -function Images::buildApplication() { - Console::verbose "${INFO}Building application images for AWS ECR${NC}" - - Images::_buildApp baked "${TRUE}" -} - -function Images::buildFrontend() { - Console::verbose "${INFO}Building Frontend image for AWS ECR${NC}" - - Images::_buildFrontend baked - Images::_buildGateway -} - -function Images::tagApplications() { - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - Console::verbose "${INFO}Tag images for AWS ECR${NC}" - for application in "${SPRYKER_APPLICATIONS[@]}"; do - local application="$(echo "$application" | tr '[:upper:]' '[:lower:]')" - docker tag "${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:${tag}" - docker tag "${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:latest" - done - - docker tag "${SPRYKER_DOCKER_PREFIX}_jenkins:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:${tag}" - docker tag "${SPRYKER_DOCKER_PREFIX}_jenkins:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:latest" - docker tag "${SPRYKER_DOCKER_PREFIX}_pipeline:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-pipeline:${tag}" - docker tag "${SPRYKER_DOCKER_PREFIX}_pipeline:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-pipeline:latest" -} - -function Images::tagFrontend() { - Console::verbose "${INFO}Tagging Frontend for AWS ECR${NC}" - - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - docker tag "${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-frontend:${tag}" - docker tag "${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-frontend:latest" - -} - -function Images::push() { - Console::verbose "${INFO}Pushing images to AWS ECR${NC}" - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - docker images | grep ecr - for application in "${SPRYKER_APPLICATIONS[@]}"; do - local application="$(echo "$application" | tr '[:upper:]' '[:lower:]')" - echo "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:latest" - done - echo "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-frontend:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-frontend:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-frontend:latest" - echo "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:latest" - echo "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-pipeline:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-pipeline:${tag}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-pipeline:latest" -} diff --git a/bin/sdk/images/common.sh b/bin/sdk/images/common.sh index 78226c887..53a7dab56 100644 --- a/bin/sdk/images/common.sh +++ b/bin/sdk/images/common.sh @@ -2,6 +2,8 @@ require docker +import sdk/images/destination/print.sh + function Images::pull() { docker pull "${SPRYKER_PLATFORM_IMAGE}" || true } @@ -11,232 +13,47 @@ function Images::destroy() { # ${XARGS_NO_RUN_IF_EMPTY} must be without quotes # shellcheck disable=SC2086 - docker images --filter "reference=${SPRYKER_DOCKER_PREFIX}_*:${SPRYKER_DOCKER_TAG}" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f - docker images --filter "reference=${SPRYKER_DOCKER_PREFIX}_builder_assets*" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f - docker images --filter "reference=spryker_docker_sdk*" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f - - docker rmi -f "${SPRYKER_DOCKER_PREFIX}_cli" || true - docker rmi -f "${SPRYKER_DOCKER_PREFIX}_app" || true - docker rmi -f "${SPRYKER_PLATFORM_IMAGE}" || true + docker images --filter "label=spryker.project=${SPRYKER_DOCKER_PREFIX}" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f 2>/dev/null + docker images --filter "reference=spryker_docker_sdk*" --format "{{.ID}}" | xargs ${XARGS_NO_RUN_IF_EMPTY} docker rmi -f 2>/dev/null + docker rmi -f "${SPRYKER_PLATFORM_IMAGE}" 2>/dev/null || true } -# Using temporary file for secrets as `docker secret` is only available for swarm mode. -function Images::_prepareSecrets() { - env - "${SECRETS_ENVIRONMENT[@]}" env > "${SECRETS_FILE_PATH}" -} +function Images::_build() { -function Images::_buildApp() { - - local -a sshArgument=() - local folder=${1} - local withPushImages=${2:-${FALSE}} - local baseAppImage="${SPRYKER_DOCKER_PREFIX}_base_app:${SPRYKER_DOCKER_TAG}" - local appImage="${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" - local localAppImage="${SPRYKER_DOCKER_PREFIX}_local_app:${SPRYKER_DOCKER_TAG}" - local runtimeImage="${SPRYKER_DOCKER_PREFIX}_run_app:${SPRYKER_DOCKER_TAG}" - local baseCliImage="${SPRYKER_DOCKER_PREFIX}_base_cli:${SPRYKER_DOCKER_TAG}" - local cliImage="${SPRYKER_DOCKER_PREFIX}_cli:${SPRYKER_DOCKER_TAG}" - local pipelineImage="${SPRYKER_DOCKER_PREFIX}_pipeline:${SPRYKER_DOCKER_TAG}" - local runtimeCliImage="${SPRYKER_DOCKER_PREFIX}_run_cli:${SPRYKER_DOCKER_TAG}" - - if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then - sshArgument=('--ssh' 'default') + # Checking availability of docker bake + if docker buildx --help | grep bake >/dev/null 2>&1; then + import sdk/images/engine/bake.sh + else + import sdk/images/engine/build.sh fi + # Using temporary file for secrets as `docker secret` is only available for swarm mode. + function Images::_prepareSecrets() { + env - "${SECRETS_ENVIRONMENT[@]}" env > "${SECRETS_FILE_PATH}" + } + Images::_prepareSecrets Registry::Trap::addExitHook 'removeBuildSecrets' "rm -f ${SECRETS_FILE_PATH}" - Console::verbose "${INFO}Building Application images${NC}" - - docker build \ - -t "${baseAppImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/application/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \ - --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \ - --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ - --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \ - --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \ - --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \ - --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \ - --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \ - --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - - docker build \ - -t "${appImage}" \ - -f "${DEPLOYMENT_PATH}/images/${folder}/application/Dockerfile" \ - "${sshArgument[@]}" \ - --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${baseAppImage}" \ - --build-arg "SPRYKER_DOCKER_PREFIX=${SPRYKER_DOCKER_PREFIX}" \ - --build-arg "SPRYKER_DOCKER_TAG=${SPRYKER_DOCKER_TAG}" \ - --build-arg "USER_UID=${USER_FULL_ID%%:*}" \ - --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \ - --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ - --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \ - --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \ - --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \ - --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \ - . 1>&2 - - docker build \ - -t "${localAppImage}" \ - -t "${runtimeImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/application-local/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${appImage}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - - if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then - docker build \ - -t "${runtimeImage}" \ - -f "${DEPLOYMENT_PATH}/images/debug/application/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${localAppImage}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - fi - - Console::verbose "${INFO}Building CLI images${NC}" - - docker build \ - -t "${baseCliImage}" \ - -t "${pipelineImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/cli/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${localAppImage}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - - docker build \ - -t "${cliImage}" \ - -t "${runtimeCliImage}" \ - -f "${DEPLOYMENT_PATH}/images/${folder}/cli/Dockerfile" \ - "${sshArgument[@]}" \ - --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${baseCliImage}" \ - --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \ - --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \ - . 1>&2 - - if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then - docker build \ - -t "${runtimeCliImage}" \ - -f "${DEPLOYMENT_PATH}/images/debug/cli/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${cliImage}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - fi - - if [ "${withPushImages}" == "${TRUE}" ]; then - local jenkinsImage="${SPRYKER_DOCKER_PREFIX}_jenkins:${SPRYKER_DOCKER_TAG}" - - docker build \ - -t "${jenkinsImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/services/jenkins/export/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${appImage}" \ - "${DEPLOYMENT_PATH}/" 1>&2 - fi + Images::_build::perform "${@}" Registry::Trap::releaseExitHook 'removeBuildSecrets' } -function Images::_buildFrontend() { - local folder=${1} - local cliImage="${SPRYKER_DOCKER_PREFIX}_cli:${SPRYKER_DOCKER_TAG}" - local builderAssetsImage="$(Assets::getImageTag)" - local baseFrontendImage="${SPRYKER_DOCKER_PREFIX}_base_frontend:${SPRYKER_DOCKER_TAG}" - local frontendImage="${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" - local runtimeFrontendImage="${SPRYKER_DOCKER_PREFIX}_run_frontend:${SPRYKER_DOCKER_TAG}" - - Console::verbose "${INFO}Building Frontend images${NC}" - - docker build \ - -t "${baseFrontendImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/frontend/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \ - --build-arg "SPRYKER_MAINTENANCE_MODE_ENABLED=${SPRYKER_MAINTENANCE_MODE_ENABLED}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - - docker build \ - -t "${frontendImage}" \ - -t "${runtimeFrontendImage}" \ - -f "${DEPLOYMENT_PATH}/images/${folder}/frontend/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${baseFrontendImage}" \ - --build-arg "SPRYKER_ASSETS_BUILDER_IMAGE=${builderAssetsImage}" \ - --build-arg "SPRYKER_MAINTENANCE_MODE_ENABLED=${SPRYKER_MAINTENANCE_MODE_ENABLED}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - - if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then - docker build \ - -t "${runtimeFrontendImage}" \ - -f "${DEPLOYMENT_PATH}/images/debug/frontend/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - --build-arg "SPRYKER_PARENT_IMAGE=${frontendImage}" \ - --build-arg "SPRYKER_XDEBUG_MODE_ENABLE=${SPRYKER_XDEBUG_MODE_ENABLE}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 - fi -} +function Images::print() { -function Images::_buildGateway() { - local gatewayImage="${SPRYKER_DOCKER_PREFIX}_gateway:${SPRYKER_DOCKER_TAG}" + local TARGET_TAG=${1:-${SPRYKER_DOCKER_TAG}} + local destination=${2:-print} - Console::verbose "${INFO}Building Gateway image${NC}" + source ${DEPLOYMENT_PATH}/images/export/${destination}.sh - docker build \ - -t "${gatewayImage}" \ - -f "${DEPLOYMENT_PATH}/images/common/gateway/Dockerfile" \ - --progress="${PROGRESS_TYPE}" \ - "${DEPLOYMENT_PATH}/context" 1>&2 -} + local imageData + for imageData in "${IMAGE_TAGS[@]}"; do + eval "${imageData}" -function Images::_tagByApp() { - local applicationName=$1 - local imageName=$2 - local baseImageName=${3:-${imageName}} - local applicationPrefix="$(echo "$applicationName" | tr '[:upper:]' '[:lower:]')" - local tag="${imageName}-${applicationPrefix}" + for tag in "${TAGS[@]}"; do + echo -e "$IMAGE $tag" + done - docker tag "${baseImageName}" "${tag}" -} - -function Images::tagApplications() { - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - for application in "${SPRYKER_APPLICATIONS[@]}"; do - Images::_tagByApp "${application}" "${SPRYKER_DOCKER_PREFIX}_app:${tag}" "${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" - Images::_tagByApp "${application}" "${SPRYKER_DOCKER_PREFIX}_run_app:${tag}" "${SPRYKER_DOCKER_PREFIX}_run_app:${SPRYKER_DOCKER_TAG}" - done - - Images::_tagByApp pipeline "${SPRYKER_DOCKER_PREFIX}_pipeline:${tag}" "${SPRYKER_DOCKER_PREFIX}_pipeline:${SPRYKER_DOCKER_TAG}" -} - -function Images::tagFrontend() { - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - Images::_tagByApp frontend "${SPRYKER_DOCKER_PREFIX}_frontend:${tag}" "${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" -} - -function Images::printAll() { - local tag=${1:-${SPRYKER_DOCKER_TAG}} - - for application in "${SPRYKER_APPLICATIONS[@]}"; do - local applicationPrefix=$(echo "${application}" | tr '[:upper:]' '[:lower:]') - printf "%s %s_app:%s\n" "${application}" "${SPRYKER_DOCKER_PREFIX}" "${tag}-${applicationPrefix}" done - - printf "%s %s_frontend:%s\n" "frontend" "${SPRYKER_DOCKER_PREFIX}" "${tag}-frontend" - printf "%s %s_pipeline:%s\n" "pipeline" "${SPRYKER_DOCKER_PREFIX}" "${tag}-pipeline" } diff --git a/bin/sdk/images/destination/ecr.sh b/bin/sdk/images/destination/ecr.sh new file mode 100644 index 000000000..76f752c53 --- /dev/null +++ b/bin/sdk/images/destination/ecr.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +function Images::needPush() { + return ${TRUE} +} diff --git a/bin/sdk/images/destination/print.sh b/bin/sdk/images/destination/print.sh new file mode 100644 index 000000000..33af82fa1 --- /dev/null +++ b/bin/sdk/images/destination/print.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +function Images::needPush() { + return ${FALSE} +} diff --git a/bin/sdk/images/engine/bake.sh b/bin/sdk/images/engine/bake.sh new file mode 100644 index 000000000..f75b8a936 --- /dev/null +++ b/bin/sdk/images/engine/bake.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +function Images::_build::perform() { + + local folder=${1} + local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}} + local destination=${3:-print} + + local -a arguments=() + + if Images::needPush; then + arguments+=('--push') + Console::verbose "${INFO}Building, tagging and pushing images${NC}" + else + Console::verbose "${INFO}Building and tagging images${NC}" + fi + + export TARGET_TAG + export SPRYKER_BUILD_SSH=$([ -n "${SSH_AUTH_SOCK_IN_CLI}" ] && echo 'default' || echo '') + + docker buildx bake \ + -f ${DEPLOYMENT_PATH}/images/${folder}/${destination}.docker-bake.hcl \ + "${arguments[@]}" \ + --progress="${PROGRESS_TYPE}" + +} diff --git a/bin/sdk/images/engine/build.sh b/bin/sdk/images/engine/build.sh new file mode 100644 index 000000000..0e96c115b --- /dev/null +++ b/bin/sdk/images/engine/build.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +require docker + +function Images::_build::perform() { + + local folder=${1} + local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}} + local destination=${3:-print} + + local targetImage="${SPRYKER_DOCKER_PREFIX}_target:${SPRYKER_DOCKER_TAG}" + + Console::verbose "${INFO}Building images${NC}" + + # Primal build including all stages to fully benefit from multistage features + Images::build::_runBuild --target "target" -t "${targetImage}" + + # Tagging all the images by running the same build targeting different stages + source ${DEPLOYMENT_PATH}/images/${folder}/${destination}.sh + + local -a arguments=('--quiet' '--progress' 'auto') + + if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then + arguments+=('--ssh' 'default') + fi + + if Images::needPush; then + arguments+=('--push') + Console::verbose "${INFO}Tagging and pushing images${NC}" + else + Console::verbose "${INFO}Tagging images${NC}" + fi + + local targetData + for targetData in "${TARGET_TAGS[@]}"; do + eval "${targetData}" + + local -a tagArguments=() + local tag + for tag in "${TAGS[@]}"; do + tagArguments+=('-t' "${tag}") + done + + Console::info "${YELLOW}Target:${NC} ${TARGET}" + Console::verbose "${YELLOW}Tags:${NC}" + Console::verbose "${DGRAY}${TAGS[@]}${NC}" + Console::verbose "${YELLOW}Hash:${NC}" + Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}" + + done + + docker rmi -f "${targetImage}" >/dev/null 2>&1 || true +} + +function Images::build::_runBuild { + + docker build \ + -f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \ + "${@}" \ + --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \ + --label "spryker.project=${SPRYKER_DOCKER_PREFIX}" \ + --label "spryker.revision=${SPRYKER_BUILD_HASH}" \ + --label "spryker.sdk.revision=${SPRYKER_SDK_REVISION}" \ + --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \ + --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \ + --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \ + --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \ + --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ + --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \ + --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \ + --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \ + --build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \ + --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \ + --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \ + --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH}" \ + --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP}" \ + --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \ + --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \ + --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \ + --build-arg "USER_UID=${USER_UID}" \ + ./ >&2 +} diff --git a/bin/sdk/images/mount.sh b/bin/sdk/images/mount.sh index a9c047797..37e7f4613 100644 --- a/bin/sdk/images/mount.sh +++ b/bin/sdk/images/mount.sh @@ -2,7 +2,7 @@ import sdk/images/common.sh -function Images::buildApplication() { +function Images::build() { for arg in "${@}"; do case "${arg}" in '--force') @@ -12,31 +12,15 @@ function Images::buildApplication() { # TODO implement --no-cache for build images ;; *) - Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::buildApplication." + Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::build." ;; esac done - Images::_buildApp mount - Images::tagApplications "${SPRYKER_DOCKER_TAG}" + Images::_build mount } -function Images::buildFrontend() { - for arg in "${@}"; do - case "${arg}" in - '--force') - # it is always it. - ;; - '--no-cache') - # TODO implement --no-cache for build images - ;; - *) - Console::verbose "\nUnknown option ${INFO}${arg}${WARN} is acquired for Images::buildFrontend." - ;; - esac - done - - Images::_buildFrontend mount - Images::_buildGateway - Images::tagFrontend "${SPRYKER_DOCKER_TAG}" +function Images::export() { + Console::error "Export is not available in development mode." + exit 1 } diff --git a/generator/Dockerfile b/generator/Dockerfile index d44b258e6..4364f2a06 100644 --- a/generator/Dockerfile +++ b/generator/Dockerfile @@ -22,7 +22,7 @@ RUN chmod 755 ${srcRoot}/openssl/generate.sh USER root ARG USER_UID -RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true; +RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true; USER spryker diff --git a/generator/deploy-file-generator/Dockerfile b/generator/deploy-file-generator/Dockerfile index eba9890f7..b6ba01fb2 100644 --- a/generator/deploy-file-generator/Dockerfile +++ b/generator/deploy-file-generator/Dockerfile @@ -16,7 +16,7 @@ COPY --chown=spryker:spryker deploy-file-generator ${srcRoot}/deploy-file-genera USER root ARG USER_UID -RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true; +RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true; USER spryker diff --git a/generator/index.php b/generator/index.php index 7364c8bf5..cee4d40b2 100644 --- a/generator/index.php +++ b/generator/index.php @@ -22,6 +22,7 @@ $defaultDeploymentDir = getenv('SPRYKER_DOCKER_SDK_DEPLOYMENT_DIR') ?: './'; $platform = getenv('SPRYKER_DOCKER_SDK_PLATFORM') ?: 'linux'; // Possible values: linux windows macos +$sdkRevision = getenv('SPRYKER_SDK_REVISION') ?: ''; $loaders = new ChainLoader([ new FilesystemLoader(APPLICATION_SOURCE_DIR . DS . 'templates'), @@ -80,6 +81,7 @@ public function setIsActive(bool $isActive): void $projectData['_defaultDeploymentDir'] = $defaultDeploymentDir; $projectData['tag'] = $projectData['tag'] ?? uniqid(); $projectData['_platform'] = $platform; +$projectData['_sdk_revision'] = $sdkRevision; $mountMode = $projectData['_mountMode'] = retrieveMountMode($projectData, $platform); $projectData['_syncIgnore'] = buildSyncIgnore($deploymentDir); $projectData['_syncSessionName'] = preg_replace('/[^-a-zA-Z0-9]/', '-', $projectData['namespace'] . '-' . $projectData['tag'] . '-codebase'); @@ -575,11 +577,36 @@ static function ($endpoint) use ($projectData) { json_encode($environment, JSON_PRETTY_PRINT) ); +@mkdir($deploymentDir . DS . 'images' . DS . 'baked', 0777, true); file_put_contents( - $deploymentDir . DS . 'images' . DS . 'common' . DS . 'application' . DS . 'Dockerfile', - $twig->render('images' . DS . 'common' . DS . 'application' . DS . 'Dockerfile.twig', $projectData) + $deploymentDir . DS . 'images' . DS . 'baked' . DS . 'Dockerfile', + $twig->render('images' . DS . 'templates' . DS . 'baked' . DS . 'Dockerfile.twig', $projectData) ); -unlink($deploymentDir . DS . 'images' . DS . 'common' . DS . 'application' . DS . 'Dockerfile.twig'); +@mkdir($deploymentDir . DS . 'images' . DS . 'export', 0777, true); +file_put_contents( + $deploymentDir . DS . 'images' . DS . 'export' . DS . 'Dockerfile', + $twig->render('images' . DS . 'templates' . DS . 'export' . DS . 'Dockerfile.twig', $projectData) +); +@mkdir($deploymentDir . DS . 'images' . DS . 'mount', 0777, true); +file_put_contents( + $deploymentDir . DS . 'images' . DS . 'mount' . DS . 'Dockerfile', + $twig->render('images' . DS . 'templates' . DS . 'mount' . DS . 'Dockerfile.twig', $projectData) +); + +foreach (['baked' => 'runtime', 'mount' => 'runtime', 'export' => 'export'] as $buildMode => $tagMode) { + foreach (['bake' => 'docker-bake.hcl', 'build' => 'sh'] as $buildEngine => $fileExtension) { + foreach (['ecr', 'print'] as $buildDestination) { + if ($tagMode === 'runtime' && $buildDestination === 'ecr') continue; + file_put_contents( + $deploymentDir . DS . 'images' . DS . $buildMode . DS . "${buildDestination}.${fileExtension}", + $twig->render( + 'images' . DS . 'templates' . DS . 'plans' . DS . "${buildMode}.${fileExtension}", + ['_applications' => $projectData['_applications'], 'tagMode' => $tagMode, 'buildDestination' => $buildDestination] + ) + ); + } + } +} file_put_contents( $deploymentDir . DS . 'docker-compose.yml', diff --git a/generator/src/templates/deploy.bash.twig b/generator/src/templates/deploy.bash.twig index 18952361a..7872bd3a3 100644 --- a/generator/src/templates/deploy.bash.twig +++ b/generator/src/templates/deploy.bash.twig @@ -102,12 +102,13 @@ readonly PROJECT_DIR="$(pwd)" readonly DEPLOYMENT_DIR="$(cd "${BASH_SOURCE%/*}" >/dev/null 2>&1 && pwd)" readonly DEPLOYMENT_PATH="${DEPLOYMENT_DIR/$PROJECT_DIR/.}" readonly SECRETS_FILE_PATH="${DEPLOYMENT_DIR}/.secrets" +readonly SPRYKER_SDK_REVISION="{{ _sdk_revision | raw }}" readonly PROGRESS_TYPE=${PROGRESS_TYPE:-tty} VERBOSE=${VERBOSE:-0} # Predefined constants readonly APPLICATION_ENV="{{ environment | default('docker') }}" -declare -a -r SPRYKER_APPLICATIONS=({{ _applications | join(' ') | raw }}) +# deprecated with Image::export declare -a -r SPRYKER_ENTRYPOINTS=({{ _entryPoints | join(' ') | raw }}) readonly COMPOSE_PROJECT_NAME={{ namespace | default('spryker') }} readonly SPRYKER_PIPELINE=${SPRYKER_PIPELINE:-{{ pipeline | default('docker') }}} @@ -132,10 +133,10 @@ readonly SPRYKER_SYNC_VOLUME="${SPRYKER_DOCKER_PREFIX}_${SPRYKER_DOCKER_TAG}_dat readonly DOCKER_COMPOSE_FILES_EXTRA="{{ docker['compose']['yamls'] | default([]) | join(' ') }}" readonly SPRYKER_DASHBOARD_ENDPOINT="{{ _dashboardEndpoint }}" readonly KNOWN_HOSTS="{{ _knownHosts | default("") }}" -readonly SPRYKER_BUILD_HASH="${SPRYKER_BUILD_HASH}" -readonly SPRYKER_BUILD_DATE="${SPRYKER_BUILD_DATE}" +readonly SPRYKER_BUILD_HASH="${SPRYKER_BUILD_HASH:-current}" +readonly SPRYKER_BUILD_STAMP="${SPRYKER_BUILD_STAMP:-now}" readonly SPRYKER_REPOSITORY_HASH="${SPRYKER_BUILD_HASH:-"$(git rev-parse --verify HEAD 2>/dev/null || echo "head")"}" -readonly SPRYKER_MAINTENANCE_MODE_ENABLED="{{ docker['maintenance']['enabled'] | default('false') }}" +readonly SPRYKER_MAINTENANCE_MODE_ENABLED="${SPRYKER_MAINTENANCE_MODE_ENABLED:-{{ docker['maintenance']['enabled'] ? 1 : 0 | default(0) }}}" # PHP extensions {% for phpExtensionName in _phpExtensions %} @@ -172,6 +173,7 @@ readonly SSH_AUTH_SOCK_IN_CLI="$([ -n "${SSH_AUTH_SOCK}" ] && [ -z "${COMPOSER_A # Global variables readonly USER_FULL_ID=$(Environment::getFullUserId) +readonly USER_UID="${USER_FULL_ID%%:*}" readonly SPRYKER_XDEBUG_HOST_IP=$(Environment::getHostIp) readonly SECRETS_ENVIRONMENT=("COMPOSER_AUTH='${COMPOSER_AUTH}'") @@ -208,9 +210,11 @@ shift $((OPTIND - 1)) # Exporting for docker-compose export DEPLOYMENT_PATH +export SPRYKER_SDK_REVISION export SPRYKER_XDEBUG_HOST_IP export SSH_AUTH_SOCK_ON_HOST export COMPOSE_PROJECT_NAME +export SPRYKER_PROJECT_NAME export SPRYKER_DOCKER_PREFIX export SPRYKER_DOCKER_TAG export SPRYKER_TESTING_ENABLE @@ -218,7 +222,22 @@ export SPRYKER_XDEBUG_MODE_ENABLE export SPRYKER_XDEBUG_ENABLE export SPRYKER_LOG_DIRECTORY export SPRYKER_BUILD_HASH -export SPRYKER_BUILD_DATE +export SPRYKER_BUILD_STAMP +export SECRETS_FILE_PATH +export SPRYKER_PLATFORM_IMAGE +export SPRYKER_FRONTEND_IMAGE +export SPRYKER_PIPELINE +export APPLICATION_ENV +export SPRYKER_COMPOSER_MODE +export SPRYKER_COMPOSER_AUTOLOAD +export SPRYKER_ASSETS_MODE +export SPRYKER_DB_ENGINE +export KNOWN_HOSTS +export SPRYKER_NODE_IMAGE_VERSION +export SPRYKER_NODE_IMAGE_DISTRO +export SPRYKER_NPM_VERSION +export SPRYKER_MAINTENANCE_MODE_ENABLED +export USER_UID export BLACKFIRE_SERVER_ID=${BLACKFIRE_SERVER_ID} export BLACKFIRE_SERVER_TOKEN=${BLACKFIRE_SERVER_TOKEN} export BLACKFIRE_CLIENT_ID=${BLACKFIRE_CLIENT_ID} diff --git a/generator/src/templates/docker-compose.yml.twig b/generator/src/templates/docker-compose.yml.twig index d8ce809bb..a841ef130 100644 --- a/generator/src/templates/docker-compose.yml.twig +++ b/generator/src/templates/docker-compose.yml.twig @@ -73,6 +73,7 @@ services: ALLOWED_IP: "127.0.0.1" SPRYKER_TESTING_ENABLE: ${SPRYKER_TESTING_ENABLE} BLACKFIRE_APM_ENABLED: ${BLACKFIRE_APM_ENABLED} + SPRYKER_MAINTENANCE_MODE_ENABLED: ${SPRYKER_MAINTENANCE_MODE_ENABLED:-0} {% if docker['debug']['xdebug']['enabled'] is not defined or docker['debug']['xdebug']['enabled'] == true %} SPRYKER_XDEBUG_ENABLE: ${SPRYKER_XDEBUG_ENABLE} {% endif %} diff --git a/generator/src/templates/mount/nfs.yml.twig b/generator/src/templates/mount/nfs.yml.twig index 6bab37313..2b6412e3a 100644 --- a/generator/src/templates/mount/nfs.yml.twig +++ b/generator/src/templates/mount/nfs.yml.twig @@ -2,7 +2,7 @@ {% set port = docker['mount'][_mountMode]['anticache']['port'] | default(8998) %} {% if enabled is not empty %} anticache: - image: ${SPRYKER_DOCKER_PREFIX}_cli:${SPRYKER_DOCKER_TAG} + image: ${SPRYKER_DOCKER_PREFIX}_run_cli:${SPRYKER_DOCKER_TAG} ports: - "{{ port }}:8998" healthcheck: diff --git a/generator/src/templates/service/dashboard/default/dashboard.yml.twig b/generator/src/templates/service/dashboard/default/dashboard.yml.twig index 997151cfa..35f300feb 100644 --- a/generator/src/templates/service/dashboard/default/dashboard.yml.twig +++ b/generator/src/templates/service/dashboard/default/dashboard.yml.twig @@ -1,7 +1,13 @@ {{ serviceName }}: build: context: ./${DEPLOYMENT_PATH}/ - dockerfile: images/common/dashboard/Dockerfile + dockerfile: images/services/dashboard/Dockerfile + tags: + - "${SPRYKER_DOCKER_PREFIX}_dashboard:${SPRYKER_DOCKER_TAG}" + labels: + "spryker.image": "none" + "spryker.project": "${SPRYKER_DOCKER_PREFIX}" + "spryker.sdk.revision": "${SPRYKER_SDK_REVISION}" networks: - public - private diff --git a/generator/src/templates/service/tideways/default/tideways.yml.twig b/generator/src/templates/service/tideways/default/tideways.yml.twig index ac6c075ff..42d0e39f2 100644 --- a/generator/src/templates/service/tideways/default/tideways.yml.twig +++ b/generator/src/templates/service/tideways/default/tideways.yml.twig @@ -1,9 +1,15 @@ {{ serviceName }}: build: - context: ./${DEPLOYMENT_PATH}/images/common/services/tideways/ + context: ./${DEPLOYMENT_PATH}/images/services/tideways/ dockerfile: Dockerfile args: TIDEWAYS_ENVIRONMENT_DEFAULT: "{{ serviceData.environment | default('production') }}" + tags: + - "${SPRYKER_DOCKER_PREFIX}_tideways:${SPRYKER_DOCKER_TAG}" + labels: + "spryker.image": "none" + "spryker.project": "${SPRYKER_DOCKER_PREFIX}" + "spryker.sdk.revision": "${SPRYKER_SDK_REVISION}" networks: - private labels: diff --git a/images/baked/.gitkeep b/images/baked/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/images/baked/application/Dockerfile b/images/baked/application.Dockerfile similarity index 71% rename from images/baked/application/Dockerfile rename to images/baked/application.Dockerfile index aeeceae8e..11f9ee09e 100644 --- a/images/baked/application/Dockerfile +++ b/images/baked/application.Dockerfile @@ -1,7 +1,5 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE - -FROM ${SPRYKER_PARENT_IMAGE} AS application-production-dependencies +FROM application-basic AS application-codebase +LABEL "spryker.image" "none" USER spryker @@ -11,9 +9,8 @@ ARG SPRYKER_COMPOSER_MODE RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} -vvv - -FROM application-production-dependencies AS application-production-codebase + && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \ + && find ./vendor -type d -name \.git* -prune -exec rm -rf {} + COPY --chown=spryker:spryker src ${srcRoot}/src COPY --chown=spryker:spryker config ${srcRoot}/config @@ -22,24 +19,28 @@ COPY --chown=spryker:spryker *.php ${srcRoot}/ COPY --chown=spryker:spryker data ${srcRoot}/data RUN chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true -RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ - composer dump-autoload -o +ENV PATH=${srcRoot}/vendor/bin:$PATH + +ARG APPLICATION_ENV +ENV APPLICATION_ENV=${APPLICATION_ENV} + +FROM application-codebase AS application-before-stamp +LABEL "spryker.image" "none" + +USER spryker + +RUN composer dump-autoload -o ARG SPRYKER_PIPELINE ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} -ARG APPLICATION_ENV ARG SPRYKER_DB_ENGINE -ENV APPLICATION_ENV=${APPLICATION_ENV} ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} COPY --chown=spryker:spryker data ${srcRoot}/data -RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production -vvv +RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production ARG SPRYKER_COMPOSER_AUTOLOAD -RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ - composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} - -FROM application-production-codebase AS application-production +RUN composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} COPY --chown=spryker:spryker public ${srcRoot}/public COPY --chown=spryker:spryker frontend ${srcRoot}/frontend @@ -50,10 +51,13 @@ USER root RUN rm -rf /var/run/opcache/* RUN chown -R spryker:spryker /home/spryker -ARG SPRYKER_BUILD_HASH='current' -ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} -ARG SPRYKER_BUILD_STAMP='' -ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} - CMD [ "php-fpm", "--nodaemonize" ] EXPOSE 9000 + +FROM application-before-stamp AS application +LABEL "spryker.image" "application" + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/baked/assets/Dockerfile b/images/baked/assets/Dockerfile index 5e990fe16..25612de09 100644 --- a/images/baked/assets/Dockerfile +++ b/images/baked/assets/Dockerfile @@ -1,21 +1,60 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE +FROM application-codebase AS assets-builder +LABEL "spryker.image" "none" -FROM ${SPRYKER_PARENT_IMAGE} AS assets-builder +USER root + +COPY --from=node-distributive /usr/lib /usr/lib +COPY --from=node-distributive /usr/local/share /usr/local/share +COPY --from=node-distributive /usr/local/lib /usr/local/lib +COPY --from=node-distributive /usr/local/include /usr/local/include +COPY --from=node-distributive /usr/local/bin /usr/local/bin + +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ + python3 \ + g++ \ + make \ + ; fi' + +# Debian contains outdated Yarn package +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then \ + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ + echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ + apt update -y && apt install -y \ + yarn \ + ; fi' + +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ + bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ + coreutils \ + ncurses \ + yarn \ + python3 \ + g++ \ + make \ + ; fi' USER spryker -COPY --chown=spryker:spryker package.json package-lock.json ${srcRoot}/ +COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .* *.* ${srcRoot}/ COPY --chown=spryker:spryker frontend* ${srcRoot}/frontend -COPY --chown=spryker:spryker tsconfig*.json ${srcRoot}/ +COPY --chown=spryker:spryker public* ${srcRoot}/public +COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn COPY --chown=spryker:spryker config/Yves ${srcRoot}/config/Yves -ARG SPRYKER_ASSETS_MODE='development' +ARG SPRYKER_ASSETS_MODE ENV SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE} ARG SPRYKER_PIPELINE ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} -RUN --mount=type=cache,id=npm,sharing=locked,target=/root/.npm \ - echo "BUILD HASH: ${SPRYKER_BUILD_HASH}" \ - && echo "MODE: ${SPRYKER_ASSETS_MODE}" \ +# This instruction is necessary to ouline dependency on precacher to make sure assets are built after +COPY --from=npm-precacher /tmp/.dependency* /tmp/ + +RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 \ + --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules \ + echo "MODE: ${SPRYKER_ASSETS_MODE}" \ + && vendor/bin/console transfer:generate \ && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} -vvv diff --git a/images/baked/frontend/Dockerfile b/images/baked/frontend/Dockerfile index 1b3416b2f..f7e8fc72c 100644 --- a/images/baked/frontend/Dockerfile +++ b/images/baked/frontend/Dockerfile @@ -1,10 +1,17 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -ARG SPRYKER_ASSETS_BUILDER_IMAGE +FROM frontend-basic as frontend-before-stamp +LABEL "spryker.image" "none" -FROM ${SPRYKER_ASSETS_BUILDER_IMAGE} as assets-builder +RUN mkdir -p ${srcRoot}/public && chmod 0777 ${srcRoot}/public +COPY --from=assets-builder --chown=root:root ${srcRoot}/public ${srcRoot}/public -FROM ${SPRYKER_PARENT_IMAGE} as frontend-production +FROM frontend-before-stamp as frontend +LABEL "spryker.image" "frontend" -RUN mkdir -p /data/public && chmod 0777 /data/public -COPY --from=assets-builder --chown=root:root /data/public /data/public +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} + +RUN mkdir -p /usr/share/nginx/ \ + && envsubst '${SPRYKER_BUILD_HASH} ${SPRYKER_BUILD_STAMP}' < /tmp/build.json > /usr/share/nginx/build.json \ + && rm -f /tmp/build.json diff --git a/images/baked/cli/Dockerfile b/images/baked/pipeline.Dockerfile similarity index 53% rename from images/baked/cli/Dockerfile rename to images/baked/pipeline.Dockerfile index a019d0e4b..b90a428d3 100644 --- a/images/baked/cli/Dockerfile +++ b/images/baked/pipeline.Dockerfile @@ -1,24 +1,29 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE - -FROM ${SPRYKER_PARENT_IMAGE} as cli-production +FROM pipeline-basic as pipeline-before-stamp +LABEL "spryker.image" "none" USER spryker -# Install composer modules for Spryker +# Install dev modules for Spryker COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/ -ARG SPRYKER_COMPOSER_MODE +ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && composer install --no-interaction ${SPRYKER_COMPOSER_MODE} - -ARG SPRYKER_COMPOSER_AUTOLOAD -RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ - composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} + && composer install --no-interaction \ + && find ./vendor -type d -name \.git -prune -exec rm -rf {} + # Tests contain transfer declaration COPY --chown=spryker:spryker tests ${srcRoot}/tests ENV DEVELOPMENT_CONSOLE_COMMANDS=1 RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-development + +RUN composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} + +FROM pipeline-before-stamp as pipeline +LABEL "spryker.image" "pipeline" + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/common/application-local/Dockerfile b/images/common/application-local/Dockerfile index 1dd3fc224..79468c380 100644 --- a/images/common/application-local/Dockerfile +++ b/images/common/application-local/Dockerfile @@ -1,7 +1,6 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -FROM ${SPRYKER_PARENT_IMAGE} AS application-local +FROM application as application-dev +LABEL "spryker.image" "none" # Make self-signed certificate to be trusted locally -COPY nginx/ssl/ca.crt /usr/local/share/ca-certificates +COPY ${DEPLOYMENT_PATH}/context/nginx/ssl/ca.crt /usr/local/share/ca-certificates RUN update-ca-certificates diff --git a/images/common/application/Dockerfile.twig b/images/common/application.Dockerfile similarity index 61% rename from images/common/application/Dockerfile.twig rename to images/common/application.Dockerfile index be4ad5d3d..f6fed8d69 100644 --- a/images/common/application/Dockerfile.twig +++ b/images/common/application.Dockerfile @@ -1,15 +1,5 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PLATFORM_IMAGE=spryker/php:7.3 -ARG SPRYKER_NODE_IMAGE_VERSION -ARG SPRYKER_NODE_IMAGE_DISTRO - -FROM node:${SPRYKER_NODE_IMAGE_VERSION}-${SPRYKER_NODE_IMAGE_DISTRO} AS node - -ARG SPRYKER_NPM_VERSION - -RUN npm install -g npm@${SPRYKER_NPM_VERSION} - FROM ${SPRYKER_PLATFORM_IMAGE} AS application-basic +LABEL "spryker.image" "none" ENV SPRYKER_IN_DOCKER=1 ENV COMPOSER_IGNORE_CHROMEDRIVER=1 @@ -41,10 +31,11 @@ ENV PHP_FPM_PM_MAX_REQUESTS=500 ENV PHP_FPM_REQUEST_TERMINATE_TIMEOUT=1m # PHP configuration -COPY php/php-fpm.d/worker.conf /usr/local/etc/php-fpm.d/worker.conf +ARG DEPLOYMENT_PATH +COPY ${DEPLOYMENT_PATH}/context/php/php-fpm.d/worker.conf /usr/local/etc/php-fpm.d/worker.conf RUN bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/php-fpm.d/worker.conf || true" -COPY php/php.ini /usr/local/etc/php/ -COPY php/conf.d/90-opcache.ini /usr/local/etc/php/conf.d +COPY ${DEPLOYMENT_PATH}/context/php/php.ini /usr/local/etc/php/ +COPY ${DEPLOYMENT_PATH}/context/php/conf.d/90-opcache.ini /usr/local/etc/php/conf.d # removing default opcache.ini RUN rm -f /usr/local/etc/php/conf.d/opcache.ini @@ -54,17 +45,10 @@ RUN mv /usr/local/etc/php/disabled/{{phpExtention}}.ini /usr/local/etc/php/conf. {% endfor %} {% endif %} -COPY php/conf.d/99-from-deploy-yaml-php.ini /usr/local/etc/php/conf.d/ +COPY ${DEPLOYMENT_PATH}/context/php/conf.d/99-from-deploy-yaml-php.ini /usr/local/etc/php/conf.d/ # Jenkins -COPY --chown=spryker:spryker jenkins/jenkins.docker.xml.twig /home/spryker/jenkins.docker.xml.twig +COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/jenkins/jenkins.docker.xml.twig /home/spryker/jenkins.docker.xml.twig # Build info -COPY --chown=spryker:spryker php/build.php /home/spryker/build.php - -# NodeJS + NPM -COPY --from=node /usr/lib /usr/lib -COPY --from=node /usr/local/share /usr/local/share -COPY --from=node /usr/local/lib /usr/local/lib -COPY --from=node /usr/local/include /usr/local/include -COPY --from=node /usr/local/bin /usr/local/bin +COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/php/build.php /home/spryker/build.php diff --git a/images/common/args.Dockerfile b/images/common/args.Dockerfile new file mode 100644 index 000000000..9c799b84e --- /dev/null +++ b/images/common/args.Dockerfile @@ -0,0 +1,4 @@ +ARG SPRYKER_PLATFORM_IMAGE +ARG SPRYKER_FRONTEND_IMAGE +ARG SPRYKER_NODE_IMAGE_VERSION +ARG SPRYKER_NODE_IMAGE_DISTRO diff --git a/images/common/cli/Dockerfile b/images/common/cli.Dockerfile similarity index 53% rename from images/common/cli/Dockerfile rename to images/common/cli.Dockerfile index 661117ecf..cc350b448 100644 --- a/images/common/cli/Dockerfile +++ b/images/common/cli.Dockerfile @@ -1,7 +1,7 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE +FROM pipeline-before-stamp as cli +LABEL "spryker.image" "cli" -FROM ${SPRYKER_PARENT_IMAGE} as cli-basic +USER root # Blackfire client RUN mkdir -p /tmp/blackfire \ @@ -10,62 +10,28 @@ RUN mkdir -p /tmp/blackfire \ && mv /tmp/blackfire/blackfire /usr/bin/blackfire \ && rm -Rf /tmp/blackfire -ENV PATH=/data/vendor/bin:$PATH - RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ - inotify-tools \ netcat-openbsd \ - git \ redis-tools \ - jq \ - python3 \ g++ \ make \ ; fi' -# Debian contains outdated Yarn package -RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ - --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ - bash -c 'if [ ! -z "$(which apt)" ]; then \ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ - apt update -y && apt install -y \ - yarn \ - ; fi' - RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ - inotify-tools \ netcat-openbsd \ - coreutils \ - ncurses \ - git \ redis \ - yarn \ - jq \ - python3 \ g++ \ make \ ; fi' -# TODO Not-available feature: autoload-cache. Should be switchable -#RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache \ -# chown spryker:spryker /home/spryker/.composer/cache && chmod 0777 /home/spryker/.composer/cache -#USER spryker -#RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache \ -# composer global require sprymiker/autoload-cache:0.3.4 -#USER root -#RUN rm -rf /home/spryker/.composer/cache -# -#RUN --mount=type=cache,id=npm,sharing=locked,target=/root/.npm \ -# npm install autoload-cache@0.3.4 -g - USER spryker RUN mkdir -p /home/spryker/env -COPY --chown=spryker:spryker cli /home/spryker/bin +ARG DEPLOYMENT_PATH +COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/cli /home/spryker/bin RUN find /home/spryker/bin -type f -exec chmod +x {} \; ENV PATH=/home/spryker/bin:$PATH @@ -76,3 +42,8 @@ RUN mkdir -p /home/spryker/history && touch /home/spryker/history/.bash_history ENV HISTFILE=/home/spryker/history/.bash_history ENV NEWRELIC_ENABLED=0 + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/common/frontend.dev.Dockerfile b/images/common/frontend.dev.Dockerfile new file mode 100644 index 000000000..eb12ebbf2 --- /dev/null +++ b/images/common/frontend.dev.Dockerfile @@ -0,0 +1,2 @@ +FROM frontend as frontend-dev +LABEL "spryker.image" "frontend" diff --git a/images/common/frontend/Dockerfile b/images/common/frontend/Dockerfile index 1a9cab66d..2f00ccd26 100644 --- a/images/common/frontend/Dockerfile +++ b/images/common/frontend/Dockerfile @@ -1,32 +1,23 @@ -# syntax = docker/dockerfile:experimental # For brotli support you can use something like fholzer/nginx-brotli:v1.18.0 -ARG SPRYKER_FRONTEND_IMAGE=nginx:alpine - FROM ${SPRYKER_FRONTEND_IMAGE} as frontend-basic +LABEL "spryker.image" "frontend-basic" + +ENV srcRoot /data RUN mkdir -p /etc/nginx/template/ && chmod 0777 /etc/nginx/template/ -COPY --chown=root:root nginx/nginx.original.conf /etc/nginx/nginx.conf -COPY --chown=root:root nginx/conf.d/frontend.default.conf.tmpl /etc/nginx/template/default.conf.tmpl -COPY --chown=root:root nginx/conf.d/resolver.conf.tmpl /etc/nginx/template/resolver.conf.tmpl -COPY --chown=root:root nginx/auth /etc/nginx/auth -COPY --chown=root:root nginx/entrypoint.sh / +ARG DEPLOYMENT_PATH +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/nginx.original.conf /etc/nginx/nginx.conf +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/frontend.default.conf.tmpl /etc/nginx/template/default.conf.tmpl +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/resolver.conf.tmpl /etc/nginx/template/resolver.conf.tmpl +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/auth /etc/nginx/auth +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/entrypoint.sh / RUN chmod +x /entrypoint.sh ENV SPRYKER_DNS_RESOLVER_FLAGS="valid=10s ipv6=off" ENV SPRYKER_DNS_RESOLVER_IP="" +ENV SPRYKER_MAINTENANCE_MODE_ENABLED="0" -ARG SPRYKER_MAINTENANCE_MODE_ENABLED -ENV SPRYKER_MAINTENANCE_MODE_ENABLED=${SPRYKER_MAINTENANCE_MODE_ENABLED} - -# Build info -ARG SPRYKER_BUILD_HASH='current' -ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} -ARG SPRYKER_BUILD_STAMP='' -ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} -COPY --chown=root:root nginx/build.json /tmp/build.json -RUN mkdir -p /usr/share/nginx/ \ - && envsubst '${SPRYKER_BUILD_HASH} ${SPRYKER_BUILD_STAMP}' < /tmp/build.json > /usr/share/nginx/build.json \ - && rm -f /tmp/build.json +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/build.json /tmp/build.json ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/images/common/gateway/Dockerfile b/images/common/gateway/Dockerfile index 5fc928891..be9122b3a 100644 --- a/images/common/gateway/Dockerfile +++ b/images/common/gateway/Dockerfile @@ -1,13 +1,12 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_GATEWAY_IMAGE=nginx:alpine - -FROM ${SPRYKER_GATEWAY_IMAGE} as gateway - -COPY --chown=root:root nginx/nginx.with.stream.conf /etc/nginx/nginx.conf -COPY --chown=root:root nginx/conf.d/gateway.default.conf /etc/nginx/templates/default.conf.template -COPY --chown=root:root nginx/stream.d/gateway.default.conf /etc/nginx/stream.d/default.conf -COPY --chown=root:root nginx/vhost.d/ssl.default.conf /etc/nginx/vhost.d/ssl.default.conf -COPY --chown=root:root nginx/ssl /etc/nginx/ssl +FROM nginx:alpine as gateway +LABEL "spryker.image" "gateway" + +ARG DEPLOYMENT_PATH +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/nginx.with.stream.conf /etc/nginx/nginx.conf +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/gateway.default.conf /etc/nginx/templates/default.conf.template +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/stream.d/gateway.default.conf /etc/nginx/stream.d/default.conf +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/vhost.d/ssl.default.conf /etc/nginx/vhost.d/ssl.default.conf +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/ssl /etc/nginx/ssl ENV SPRYKER_XDEBUG_ENABLE=0 diff --git a/images/common/node.Dockerfile b/images/common/node.Dockerfile new file mode 100644 index 000000000..da0d10329 --- /dev/null +++ b/images/common/node.Dockerfile @@ -0,0 +1,5 @@ +FROM node:${SPRYKER_NODE_IMAGE_VERSION}-${SPRYKER_NODE_IMAGE_DISTRO} AS node-distributive +LABEL "spryker.image" "none" + +ARG SPRYKER_NPM_VERSION +RUN npm install -g npm@${SPRYKER_NPM_VERSION} diff --git a/images/common/pipeline.Dockerfile b/images/common/pipeline.Dockerfile new file mode 100644 index 000000000..46d51b938 --- /dev/null +++ b/images/common/pipeline.Dockerfile @@ -0,0 +1,41 @@ +FROM application-before-stamp as pipeline-basic +LABEL "spryker.image" "none" + +ENV DEVELOPMENT_CONSOLE_COMMANDS=1 + +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ + git \ + python3 \ + jq \ + ; fi' + +# Debian contains outdated Yarn package +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then \ + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ + echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ + apt update -y && apt install -y \ + yarn \ + ; fi' + +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ + bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ + coreutils \ + ncurses \ + git \ + yarn \ + jq \ + python3 \ + ; fi' + +# NodeJS + NPM +COPY --from=node-distributive /usr/lib /usr/lib +COPY --from=node-distributive /usr/local/share /usr/local/share +COPY --from=node-distributive /usr/local/lib /usr/local/lib +COPY --from=node-distributive /usr/local/include /usr/local/include +COPY --from=node-distributive /usr/local/bin /usr/local/bin + +USER spryker diff --git a/images/common/dashboard/Dockerfile b/images/common/services/dashboard/Dockerfile similarity index 93% rename from images/common/dashboard/Dockerfile rename to images/common/services/dashboard/Dockerfile index d7aaab8d4..95e0369b3 100644 --- a/images/common/dashboard/Dockerfile +++ b/images/common/services/dashboard/Dockerfile @@ -1,4 +1,5 @@ FROM node:alpine AS dashboard +LABEL "spryker.image" "dashboard" RUN mkdir -p /dashboard WORKDIR /dashboard diff --git a/images/common/services/jenkins/export/Dockerfile b/images/common/services/jenkins/export/Dockerfile index 611f5d024..bcf7ff16f 100644 --- a/images/common/services/jenkins/export/Dockerfile +++ b/images/common/services/jenkins/export/Dockerfile @@ -1,18 +1,20 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE +FROM spryker/jenkins-boilerplate:2.361.1 as jenkins-boilerplate +LABEL "spryker.image" "none" + +FROM application-before-stamp as jenkins +LABEL "spryker.image" "jenkins" -FROM spryker/jenkins-boilerplate:2.361.1 as spryker-jenkins-boilerplate -FROM ${SPRYKER_PARENT_IMAGE} as spryker_jenkins EXPOSE 8080 -COPY context/jenkins/export/jenkins.docker.xml.twig ./config/Zed/cronjobs/jenkins.docker.xml.twig +COPY ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.docker.xml.twig ./config/Zed/cronjobs/jenkins.docker.xml.twig -COPY --from=spryker-jenkins-boilerplate /usr/share/jenkins/ref/plugins /usr/share/jenkins/ref/plugins -COPY --from=spryker-jenkins-boilerplate /usr/share/jenkins/jenkins.war /usr/share/jenkins/jenkins.war -COPY --from=spryker-jenkins-boilerplate /usr/share/jenkins/jenkins-cli.jar /usr/share/jenkins/jenkins-cli.jar +COPY --from=jenkins-boilerplate /usr/share/jenkins/ref/plugins /usr/share/jenkins/ref/plugins +COPY --from=jenkins-boilerplate /usr/share/jenkins/jenkins.war /usr/share/jenkins/jenkins.war +COPY --from=jenkins-boilerplate /usr/share/jenkins/jenkins-cli.jar /usr/share/jenkins/jenkins-cli.jar # Install packages on Alpine -RUN bash -c 'if [ ! -z "$(which apk)" ]; then apk --no-cache add \ - curl \ +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ + bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ + curl \ bash \ openjdk11 \ ttf-dejavu \ @@ -22,25 +24,32 @@ RUN bash -c 'if [ ! -z "$(which apk)" ]; then apk --no-cache add \ ; fi' # Install packages on Debian -RUN bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && \ - apt-get install -y software-properties-common && \ - apt-add-repository "deb http://security.debian.org/debian-security bullseye-security main" && \ - apt-add-repository "deb http://ftp.de.debian.org/debian bullseye main" && \ - apt update -y && apt install -y \ - curl \ - bash \ - openjdk-11-jdk \ - fonts-dejavu \ - gettext \ - jq \ - && \ - mkdir -p /envs \ - ; fi' +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && \ + apt-get install -y software-properties-common && \ + apt-add-repository "deb http://security.debian.org/debian-security bullseye-security main" && \ + apt-add-repository "deb http://ftp.de.debian.org/debian bullseye main" && \ + apt update -y && apt install -y \ + curl \ + bash \ + openjdk-11-jdk \ + fonts-dejavu \ + gettext \ + jq \ + && \ + mkdir -p /envs \ + ; fi' -COPY terraform/cli /envs/ -COPY context/jenkins/export/entrypoint.sh /entrypoint.sh -COPY context/jenkins/export/jenkins.model.JenkinsLocationConfiguration.xml /opt/jenkins.model.JenkinsLocationConfiguration.xml +COPY ${DEPLOYMENT_PATH}/terraform/cli /envs/ +COPY ${DEPLOYMENT_PATH}/context/jenkins/export/entrypoint.sh /entrypoint.sh +COPY ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.model.JenkinsLocationConfiguration.xml /opt/jenkins.model.JenkinsLocationConfiguration.xml COPY context/jenkins/export/nr-credentials.xml /opt/nr-credentials.xml RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/common/services/tideways/Dockerfile b/images/common/services/tideways/Dockerfile index 541050438..a3a636d82 100644 --- a/images/common/services/tideways/Dockerfile +++ b/images/common/services/tideways/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:stable-slim AS tideways-daemon +FROM debian:stable-slim AS tidewaysDaemon +LABEL "spryker.image" "tideways-deamon" ARG TIDEWAYS_ENVIRONMENT_DEFAULT=production ENV TIDEWAYS_ENVIRONMENT=$TIDEWAYS_ENVIRONMENT_DEFAULT diff --git a/images/debug/application/Dockerfile b/images/debug/application.Dockerfile similarity index 79% rename from images/debug/application/Dockerfile rename to images/debug/application.Dockerfile index ee1fb2f08..ef02b03a9 100644 --- a/images/debug/application/Dockerfile +++ b/images/debug/application.Dockerfile @@ -1,6 +1,4 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -FROM ${SPRYKER_PARENT_IMAGE} AS application-debug +USER root RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ @@ -14,9 +12,10 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc ; fi' RUN /usr/bin/install -d -m 777 /var/run/opcache/debug -COPY php/debug/etc/ /usr/local/etc/ +ARG DEPLOYMENT_PATH +COPY ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ RUN bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/debug.php-fpm.conf/worker.conf || true" -COPY php/debug/supervisord.conf /etc/supervisor/supervisord.conf +COPY ${DEPLOYMENT_PATH}/context/php/debug/supervisord.conf /etc/supervisor/supervisord.conf RUN mkdir -p /var/log/supervisor CMD [ "/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ] diff --git a/images/debug/cli/Dockerfile b/images/debug/cli/Dockerfile index f06c6cf40..b78708d32 100644 --- a/images/debug/cli/Dockerfile +++ b/images/debug/cli/Dockerfile @@ -1,8 +1,8 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -FROM ${SPRYKER_PARENT_IMAGE} AS cli-debug - USER root + RUN /usr/bin/install -d -m 777 /var/run/opcache/debug + USER spryker -COPY php/debug/etc/ /usr/local/etc/ + +ARG DEPLOYMENT_PATH +COPY ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ diff --git a/images/debug/frontend/Dockerfile b/images/debug/frontend/Dockerfile index a0b2f8fe7..683acb0f8 100644 --- a/images/debug/frontend/Dockerfile +++ b/images/debug/frontend/Dockerfile @@ -1,8 +1,4 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -FROM ${SPRYKER_PARENT_IMAGE} AS frontend-debug +ENV SPRYKER_XDEBUG_MODE_ENABLE=1 -ARG SPRYKER_XDEBUG_MODE_ENABLE -ENV SPRYKER_XDEBUG_MODE_ENABLE=${SPRYKER_XDEBUG_MODE_ENABLE} - -COPY --chown=root:root nginx/conf.d/debug.default.conf /etc/nginx/template/debug.default.conf +ARG DEPLOYMENT_PATH +COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/debug.default.conf /etc/nginx/template/debug.default.conf diff --git a/images/export/.gitkeep b/images/export/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/images/mount/.gitkeep b/images/mount/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/images/mount/cli/Dockerfile b/images/mount/cli/Dockerfile index e6c0d8589..d27f17ecf 100644 --- a/images/mount/cli/Dockerfile +++ b/images/mount/cli/Dockerfile @@ -1,7 +1,12 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE +FROM pipeline-basic as pipeline-before-stamp +LABEL "spryker.image" "none" -FROM ${SPRYKER_PARENT_IMAGE} as cli-development +USER spryker -ENV DEVELOPMENT_CONSOLE_COMMANDS=1 +FROM pipeline-before-stamp as pipeline +LABEL "spryker.image" "pipeline" +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/mount/frontend/Dockerfile b/images/mount/frontend/Dockerfile index 59dcd1076..7fd1f802b 100644 --- a/images/mount/frontend/Dockerfile +++ b/images/mount/frontend/Dockerfile @@ -1,4 +1,13 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE +FROM frontend-basic as frontend-before-stamp +LABEL "spryker.image" "none" -FROM ${SPRYKER_PARENT_IMAGE} as frontend-development +FROM frontend-before-stamp as frontend +LABEL "spryker.image" "frontend" + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} +RUN mkdir -p /usr/share/nginx/ \ + && envsubst '${SPRYKER_BUILD_HASH} ${SPRYKER_BUILD_STAMP}' < /tmp/build.json > /usr/share/nginx/build.json \ + && rm -f /tmp/build.json diff --git a/images/templates/baked/Dockerfile.twig b/images/templates/baked/Dockerfile.twig new file mode 100644 index 000000000..20c41d111 --- /dev/null +++ b/images/templates/baked/Dockerfile.twig @@ -0,0 +1,73 @@ +# syntax = docker/dockerfile:experimental + +{% include "images/templates/common/args.Dockerfile" with _context %} + +{% include "images/templates/common/node.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/npm.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/frontend.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/gateway.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/application.dev.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/application.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +{% include "images/templates/baked/assets.builder.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/cli.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/cli.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +{% include "images/templates/baked/frontend.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/frontend.dev.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/frontend.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +# To build all necessary stages at once this fake stage has dependency to all of them (for `docker build`) +FROM busybox as target +LABEL "spryker.image" "none" + +COPY --from=application-dev /tmp/.dependency* /tmp/ +COPY --from=cli /tmp/.dependency* /tmp/ +COPY --from=frontend-dev /tmp/.dependency* /tmp/ +COPY --from=gateway /tmp/.dependency* /tmp/ diff --git a/images/templates/baked/npm.Dockerfile b/images/templates/baked/npm.Dockerfile new file mode 100644 index 000000000..aa587d23d --- /dev/null +++ b/images/templates/baked/npm.Dockerfile @@ -0,0 +1,9 @@ +FROM node-distributive AS npm-precacher +LABEL "spryker.image" "none" + +COPY --chown=spryker:spryker package.json* package-lock.json* /root/ + +RUN --mount=type=cache,id=npm-cache,sharing=locked,target=/root/.npm \ + --mount=type=cache,id=npm-modules,sharing=locked,target=/root/node_modules \ + cd /root \ + && sh -c 'if [ -f ${srcRoot}/package.json ]; then npm install --prefer-offline || npm ci --prefer-offline || true; fi' diff --git a/images/templates/export/Dockerfile.twig b/images/templates/export/Dockerfile.twig new file mode 100644 index 000000000..1467606e0 --- /dev/null +++ b/images/templates/export/Dockerfile.twig @@ -0,0 +1,52 @@ +# syntax = docker/dockerfile:experimental + +{% include "images/templates/common/args.Dockerfile" with _context %} + +{% include "images/templates/common/node.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/npm.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/frontend.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/export/jenkins.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/assets.builder.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/frontend.Dockerfile" with _context %} + +# ----------------------------- + +# To build all necessary stages at once this fake stage has dependency to all of them (for `docker build`) +FROM busybox as target +LABEL "spryker.image" "none" + +COPY --from=application /tmp/.dependency* /tmp/ +COPY --from=pipeline /tmp/.dependency* /tmp/ +COPY --from=jenkins /tmp/.dependency* /tmp/ +COPY --from=frontend /tmp/.dependency* /tmp/ diff --git a/images/templates/mount/Dockerfile.twig b/images/templates/mount/Dockerfile.twig new file mode 100644 index 000000000..682f764d3 --- /dev/null +++ b/images/templates/mount/Dockerfile.twig @@ -0,0 +1,65 @@ +# syntax = docker/dockerfile:experimental + +{% include "images/templates/common/args.Dockerfile" with _context %} + +{% include "images/templates/common/node.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/frontend.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/gateway.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/mount/application.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/application.dev.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/application.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +{% include "images/templates/common/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/mount/pipeline.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/cli.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/cli.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +{% include "images/templates/mount/frontend.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/common/frontend.dev.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/frontend.Dockerfile" with _context %} +{%endif%} + +# ----------------------------- + +# To build all necessary stages at once this fake stage has dependency to all of them (for `docker build`) +FROM busybox as target +LABEL "spryker.image" "none" + +COPY --from=application-dev /tmp/.dependency* /tmp/ +COPY --from=cli /tmp/.dependency* /tmp/ +COPY --from=frontend-dev /tmp/.dependency* /tmp/ +COPY --from=gateway /tmp/.dependency* /tmp/ diff --git a/images/mount/application/Dockerfile b/images/templates/mount/application.Dockerfile similarity index 66% rename from images/mount/application/Dockerfile rename to images/templates/mount/application.Dockerfile index 064db1efd..fc017c7d1 100644 --- a/images/mount/application/Dockerfile +++ b/images/templates/mount/application.Dockerfile @@ -1,6 +1,5 @@ -# syntax = docker/dockerfile:experimental -ARG SPRYKER_PARENT_IMAGE -FROM ${SPRYKER_PARENT_IMAGE} as application-development +FROM application-basic as application-before-stamp +LABEL "spryker.image" "none" ARG APPLICATION_ENV ENV APPLICATION_ENV=${APPLICATION_ENV} @@ -8,20 +7,23 @@ ARG SPRYKER_DB_ENGINE ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} ARG DEPLOYMENT_PATH ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} +ARG SPRYKER_PIPELINE +ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} -ENV PATH=/data/vendor/bin:$PATH +ENV PATH=${srcRoot}/vendor/bin:$PATH ARG USER_UID -RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true; +RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true; COPY ${DEPLOYMENT_PATH}/context/php/conf.d/91-opcache-dev.ini /usr/local/etc/php/conf.d -ARG SPRYKER_PIPELINE -ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} -ARG SPRYKER_BUILD_HASH='current' -ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} -ARG SPRYKER_BUILD_STAMP='' -ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} - CMD [ "php-fpm", "--nodaemonize" ] EXPOSE 9000 + +FROM application-before-stamp as application +LABEL "spryker.image" "application" + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/templates/plans/baked.docker-bake.hcl b/images/templates/plans/baked.docker-bake.hcl new file mode 100644 index 000000000..1258fe461 --- /dev/null +++ b/images/templates/plans/baked.docker-bake.hcl @@ -0,0 +1 @@ +{% include "images/templates/plans/runtime.docker-bake.hcl" with { _applications, folder: 'baked' } %} diff --git a/images/templates/plans/baked.sh b/images/templates/plans/baked.sh new file mode 100644 index 000000000..25caf756b --- /dev/null +++ b/images/templates/plans/baked.sh @@ -0,0 +1,17 @@ +{% import "images/templates/plans/tag." ~ buildDestination ~ "." ~ tagMode ~ ".twig" as tagMacro %} +# expects TARGET_TAG to be set +local -a IMAGE_TAGS=( \ +{% for application in _applications %} +'local IMAGE="{{ application }}"; local -a TAGS=({{ tagMacro.tagApplication(application, " ") }})' \ +{% endfor %} +'local IMAGE="frontend"; local -a TAGS=({{ tagMacro.tagService("run_frontend", " ") }})' \ +'local IMAGE="cli"; local -a TAGS=({{ tagMacro.tagService("run_cli", " ") }})' \ +'local IMAGE="gateway"; local -a TAGS=({{ tagMacro.tagService("gateway", " ") }})' \ +) + +local -a TARGET_TAGS=( \ +'local TARGET="application-dev"; local -a TAGS=({% for application in _applications %}{{ tagMacro.tagApplication(application, " ") }}{{ ' ' }}{% endfor %})' \ +'local TARGET="frontend-dev"; local -a TAGS=({{ tagMacro.tagService("run_frontend", " ") }})' \ +'local TARGET="cli"; local -a TAGS=({{ tagMacro.tagService("run_cli", " ") }})' \ +'local TARGET="gateway"; local -a TAGS=({{ tagMacro.tagService("gateway", " ") }})' \ +) diff --git a/images/templates/plans/common.docker-bake.hcl b/images/templates/plans/common.docker-bake.hcl new file mode 100644 index 000000000..bb2b22817 --- /dev/null +++ b/images/templates/plans/common.docker-bake.hcl @@ -0,0 +1,59 @@ +variable DEPLOYMENT_PATH { default = "" } +variable SPRYKER_PROJECT_NAME { default = "" } +variable SPRYKER_DOCKER_PREFIX { default = "" } +variable SPRYKER_DOCKER_TAG { default = "" } +variable SECRETS_FILE_PATH { default = "" } +variable SPRYKER_SDK_REVISION { default = "" } +variable SPRYKER_PLATFORM_IMAGE { default = "" } +variable SPRYKER_FRONTEND_IMAGE { default = "" } +variable SPRYKER_LOG_DIRECTORY { default = "" } +variable SPRYKER_PIPELINE { default = "" } +variable APPLICATION_ENV { default = "" } +variable SPRYKER_COMPOSER_MODE { default = "" } +variable SPRYKER_COMPOSER_AUTOLOAD { default = "" } +variable SPRYKER_ASSETS_MODE { default = "" } +variable SPRYKER_DB_ENGINE { default = "" } +variable KNOWN_HOSTS { default = "" } +variable SPRYKER_BUILD_HASH { default = "" } +variable SPRYKER_BUILD_STAMP { default = "" } +variable SPRYKER_NODE_IMAGE_VERSION { default = "" } +variable SPRYKER_NODE_IMAGE_DISTRO { default = "" } +variable SPRYKER_NPM_VERSION { default = "" } +variable USER_UID { default = "" } + +variable AWS_ACCOUNT_ID { default = "" } +variable AWS_REGION { default = "" } + +variable SPRYKER_BUILD_SSH { default = "" } +variable TARGET_TAG { default = "" } + +target "_common" { + dockerfile = "${DEPLOYMENT_PATH}/images/{{ folder }}/Dockerfile" + context = "." + labels = { + "spryker.revision" = "${SPRYKER_BUILD_HASH}" + "spryker.sdk.revision" = "${SPRYKER_SDK_REVISION}" + "spryker.project" = "${SPRYKER_DOCKER_PREFIX}" + } + args = { + DEPLOYMENT_PATH = "${DEPLOYMENT_PATH}" + SPRYKER_PLATFORM_IMAGE = "${SPRYKER_PLATFORM_IMAGE}" + SPRYKER_FRONTEND_IMAGE = "${SPRYKER_FRONTEND_IMAGE}" + SPRYKER_LOG_DIRECTORY = "${SPRYKER_LOG_DIRECTORY}" + SPRYKER_PIPELINE = "${SPRYKER_PIPELINE}" + APPLICATION_ENV = "${APPLICATION_ENV}" + SPRYKER_COMPOSER_MODE = "${SPRYKER_COMPOSER_MODE}" + SPRYKER_COMPOSER_AUTOLOAD = "${SPRYKER_COMPOSER_AUTOLOAD}" + SPRYKER_ASSETS_MODE = "${SPRYKER_ASSETS_MODE}" + SPRYKER_DB_ENGINE = "${SPRYKER_DB_ENGINE}" + KNOWN_HOSTS = "${KNOWN_HOSTS}" + SPRYKER_BUILD_HASH = "${SPRYKER_BUILD_HASH}" + SPRYKER_BUILD_STAMP = "${SPRYKER_BUILD_STAMP}" + SPRYKER_NODE_IMAGE_VERSION = "${SPRYKER_NODE_IMAGE_VERSION}" + SPRYKER_NODE_IMAGE_DISTRO = "${SPRYKER_NODE_IMAGE_DISTRO}" + SPRYKER_NPM_VERSION = "${SPRYKER_NPM_VERSION}" + USER_UID = "${USER_UID}" + } + secret = ["type=file,id=secrets-env,src=${SECRETS_FILE_PATH}"] + ssh = ["${SPRYKER_BUILD_SSH}"] +} diff --git a/images/templates/plans/export.docker-bake.hcl b/images/templates/plans/export.docker-bake.hcl new file mode 100644 index 000000000..a4b5fd85c --- /dev/null +++ b/images/templates/plans/export.docker-bake.hcl @@ -0,0 +1,45 @@ +{% import "images/templates/plans/tag." ~ buildDestination ~ "." ~ tagMode ~ ".twig" as tagMacro %} +group "default" { + targets = [ + "application", + "frontend", + "pipeline", + "jenkins", + ] +} + +{% include "images/templates/plans/common.docker-bake.hcl" with { folder: 'export'} %} + +target "application" { + inherits = ["_common"] + target = "application" + tags = [ +{% for application in _applications %} + {{ tagMacro.tagApplication(application, ",\n ") | spaceless }} +{% endfor %} + ] +} + +target "frontend" { + inherits = ["_common"] + target = "frontend" + tags = [ + {{ tagMacro.tagService("frontend", ",\n ") | spaceless }} + ] +} + +target "pipeline" { + inherits = ["_common"] + target = "pipeline" + tags = [ + {{ tagMacro.tagService("pipeline", ",\n ") | spaceless }} + ] +} + +target "jenkins" { + inherits = ["_common"] + target = "jenkins" + tags = [ + {{ tagMacro.tagService("jenkins", ",\n ") | spaceless }} + ] +} diff --git a/images/templates/plans/export.sh b/images/templates/plans/export.sh new file mode 100644 index 000000000..f01ac44d2 --- /dev/null +++ b/images/templates/plans/export.sh @@ -0,0 +1,17 @@ +{% import "images/templates/plans/tag." ~ buildDestination ~ "." ~ tagMode ~ ".twig" as tagMacro %} +# expects TARGET_TAG to be set +local -a IMAGE_TAGS=( \ +{% for application in _applications %} +'local IMAGE="{{ application }}"; local -a TAGS=({{ tagMacro.tagApplication(application, " ") }})' \ +{% endfor %} +'local IMAGE="frontend"; local -a TAGS=({{ tagMacro.tagService("frontend", " ") }})' \ +'local IMAGE="pipeline"; local -a TAGS=({{ tagMacro.tagService("pipeline", " ") }})' \ +'local IMAGE="jenkins"; local -a TAGS=({{ tagMacro.tagService("jenkins", " ") }})' \ +) + +local -a TARGET_TAGS=( \ +'local TARGET="application"; local -a TAGS=({% for application in _applications %}{{ tagMacro.tagApplication(application, " ") }}{{ ' ' }}{% endfor %})' \ +'local TARGET="frontend"; local -a TAGS=({{ tagMacro.tagService("frontend", " ") }})' \ +'local TARGET="pipeline"; local -a TAGS=({{ tagMacro.tagService("pipeline", " ") }})' \ +'local TARGET="jenkins"; local -a TAGS=({{ tagMacro.tagService("jenkins", " ") }})' \ +) diff --git a/images/templates/plans/mount.docker-bake.hcl b/images/templates/plans/mount.docker-bake.hcl new file mode 100644 index 000000000..943c4a130 --- /dev/null +++ b/images/templates/plans/mount.docker-bake.hcl @@ -0,0 +1 @@ +{% include "images/templates/plans/runtime.docker-bake.hcl" with { _applications, folder: 'mount' } %} diff --git a/images/templates/plans/mount.sh b/images/templates/plans/mount.sh new file mode 100644 index 000000000..10ac24f2d --- /dev/null +++ b/images/templates/plans/mount.sh @@ -0,0 +1 @@ +{% include "images/templates/plans/baked.sh" %} diff --git a/images/templates/plans/runtime.docker-bake.hcl b/images/templates/plans/runtime.docker-bake.hcl new file mode 100644 index 000000000..c4f365876 --- /dev/null +++ b/images/templates/plans/runtime.docker-bake.hcl @@ -0,0 +1,45 @@ +{% import "images/templates/plans/tag." ~ buildDestination ~ "." ~ tagMode ~ ".twig" as tagMacro %} +group "default" { + targets = [ + "application", + "frontend", + "cli", + "gateway", + ] +} + +{% include "images/templates/plans/common.docker-bake.hcl" with { folder } %} + +target "application" { + inherits = ["_common"] + target = "application-dev" + tags = [ +{% for application in _applications %} + {{ tagMacro.tagApplication(application, ",\n ") | spaceless }} +{% endfor %} + ] +} + +target "frontend" { + inherits = ["_common"] + target = "frontend-dev" + tags = [ + {{ tagMacro.tagService("run_frontend", ",\n ") | spaceless }} + ] +} + +target "cli" { + inherits = ["_common"] + target = "cli" + tags = [ + {{ tagMacro.tagService("run_cli", ",\n ") | spaceless }} + ] +} + +target "gateway" { + inherits = ["_common"] + target = "gateway" + tags = [ + {{ tagMacro.tagService("gateway", ",\n ") | spaceless }} + ] +} diff --git a/images/templates/plans/tag.ecr.export.twig b/images/templates/plans/tag.ecr.export.twig new file mode 100644 index 000000000..56b0020cc --- /dev/null +++ b/images/templates/plans/tag.ecr.export.twig @@ -0,0 +1,2 @@ +{% macro tagApplication(application, separator) %}"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-{{ application | lower }}:${TARGET_TAG}"{{ separator }}"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-{{ application | lower }}:latest"{{ separator }}{% endmacro %} +{% macro tagService(service, separator) %}"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-{{ service | lower }}:${TARGET_TAG}"{{ separator }}"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-{{ service | lower }}:latest"{{ separator }}{% endmacro %} diff --git a/images/templates/plans/tag.print.export.twig b/images/templates/plans/tag.print.export.twig new file mode 100644 index 000000000..f000caac8 --- /dev/null +++ b/images/templates/plans/tag.print.export.twig @@ -0,0 +1,2 @@ +{% macro tagApplication(application, separator) %}"${SPRYKER_DOCKER_PREFIX}_app:${TARGET_TAG}-{{ application | lower }}"{{ separator }}{% endmacro %} +{% macro tagService(service, separator) %}"${SPRYKER_DOCKER_PREFIX}-{{ service | lower }}:${TARGET_TAG}-{{ service | lower }}"{{ separator }}{% endmacro %} diff --git a/images/templates/plans/tag.print.runtime.twig b/images/templates/plans/tag.print.runtime.twig new file mode 100644 index 000000000..1590d36f9 --- /dev/null +++ b/images/templates/plans/tag.print.runtime.twig @@ -0,0 +1,2 @@ +{% macro tagApplication(application, separator) %}"${SPRYKER_DOCKER_PREFIX}_run_app:${TARGET_TAG}-{{ application | lower }}"{{ separator }}{% endmacro %} +{% macro tagService(service, separator) %}"${SPRYKER_DOCKER_PREFIX}_{{ service | lower }}:${TARGET_TAG}"{{ separator }}{% endmacro %} diff --git a/sdk b/sdk index fb373761a..978813a2a 100755 --- a/sdk +++ b/sdk @@ -48,6 +48,7 @@ shift $((OPTIND - 1)) System::Bootstrap +export SPRYKER_PROJECT_NAME readonly DESTINATION_DIR="${SOURCE_DIR}/deployment/${SPRYKER_PROJECT_NAME}" readonly DEPLOY_SCRIPT=${DESTINATION_DIR}/deploy From 252ff7389fa32754be38694422b5b8e3f834c824 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Wed, 28 Jun 2023 22:08:52 +0300 Subject: [PATCH 02/26] Moved dockerfiles to match current structure and keep git history as update instead of create/delete --- images/{common => }/services/dashboard/Dockerfile | 0 images/{common => }/services/tideways/Dockerfile | 0 images/{ => templates}/baked/application.Dockerfile | 0 .../Dockerfile => templates/baked/assets.builder.Dockerfile} | 0 .../frontend/Dockerfile => templates/baked/frontend.Dockerfile} | 0 images/{ => templates}/baked/pipeline.Dockerfile | 0 images/{ => templates}/common/application.Dockerfile | 0 .../Dockerfile => templates/common/application.dev.Dockerfile} | 0 images/{ => templates}/common/args.Dockerfile | 0 images/{ => templates}/common/cli.Dockerfile | 0 images/{ => templates/common}/debug/application.Dockerfile | 0 .../cli/Dockerfile => templates/common/debug/cli.Dockerfile} | 0 .../Dockerfile => templates/common/debug/frontend.Dockerfile} | 0 .../frontend/Dockerfile => templates/common/frontend.Dockerfile} | 0 images/{ => templates}/common/frontend.dev.Dockerfile | 0 .../gateway/Dockerfile => templates/common/gateway.Dockerfile} | 0 images/{ => templates}/common/node.Dockerfile | 0 images/{ => templates}/common/pipeline.Dockerfile | 0 .../export/Dockerfile => templates/export/jenkins.Dockerfile} | 0 .../frontend/Dockerfile => templates/mount/frontend.Dockerfile} | 0 .../{mount/cli/Dockerfile => templates/mount/pipeline.Dockerfile} | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename images/{common => }/services/dashboard/Dockerfile (100%) rename images/{common => }/services/tideways/Dockerfile (100%) rename images/{ => templates}/baked/application.Dockerfile (100%) rename images/{baked/assets/Dockerfile => templates/baked/assets.builder.Dockerfile} (100%) rename images/{baked/frontend/Dockerfile => templates/baked/frontend.Dockerfile} (100%) rename images/{ => templates}/baked/pipeline.Dockerfile (100%) rename images/{ => templates}/common/application.Dockerfile (100%) rename images/{common/application-local/Dockerfile => templates/common/application.dev.Dockerfile} (100%) rename images/{ => templates}/common/args.Dockerfile (100%) rename images/{ => templates}/common/cli.Dockerfile (100%) rename images/{ => templates/common}/debug/application.Dockerfile (100%) rename images/{debug/cli/Dockerfile => templates/common/debug/cli.Dockerfile} (100%) rename images/{debug/frontend/Dockerfile => templates/common/debug/frontend.Dockerfile} (100%) rename images/{common/frontend/Dockerfile => templates/common/frontend.Dockerfile} (100%) rename images/{ => templates}/common/frontend.dev.Dockerfile (100%) rename images/{common/gateway/Dockerfile => templates/common/gateway.Dockerfile} (100%) rename images/{ => templates}/common/node.Dockerfile (100%) rename images/{ => templates}/common/pipeline.Dockerfile (100%) rename images/{common/services/jenkins/export/Dockerfile => templates/export/jenkins.Dockerfile} (100%) rename images/{mount/frontend/Dockerfile => templates/mount/frontend.Dockerfile} (100%) rename images/{mount/cli/Dockerfile => templates/mount/pipeline.Dockerfile} (100%) diff --git a/images/common/services/dashboard/Dockerfile b/images/services/dashboard/Dockerfile similarity index 100% rename from images/common/services/dashboard/Dockerfile rename to images/services/dashboard/Dockerfile diff --git a/images/common/services/tideways/Dockerfile b/images/services/tideways/Dockerfile similarity index 100% rename from images/common/services/tideways/Dockerfile rename to images/services/tideways/Dockerfile diff --git a/images/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile similarity index 100% rename from images/baked/application.Dockerfile rename to images/templates/baked/application.Dockerfile diff --git a/images/baked/assets/Dockerfile b/images/templates/baked/assets.builder.Dockerfile similarity index 100% rename from images/baked/assets/Dockerfile rename to images/templates/baked/assets.builder.Dockerfile diff --git a/images/baked/frontend/Dockerfile b/images/templates/baked/frontend.Dockerfile similarity index 100% rename from images/baked/frontend/Dockerfile rename to images/templates/baked/frontend.Dockerfile diff --git a/images/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile similarity index 100% rename from images/baked/pipeline.Dockerfile rename to images/templates/baked/pipeline.Dockerfile diff --git a/images/common/application.Dockerfile b/images/templates/common/application.Dockerfile similarity index 100% rename from images/common/application.Dockerfile rename to images/templates/common/application.Dockerfile diff --git a/images/common/application-local/Dockerfile b/images/templates/common/application.dev.Dockerfile similarity index 100% rename from images/common/application-local/Dockerfile rename to images/templates/common/application.dev.Dockerfile diff --git a/images/common/args.Dockerfile b/images/templates/common/args.Dockerfile similarity index 100% rename from images/common/args.Dockerfile rename to images/templates/common/args.Dockerfile diff --git a/images/common/cli.Dockerfile b/images/templates/common/cli.Dockerfile similarity index 100% rename from images/common/cli.Dockerfile rename to images/templates/common/cli.Dockerfile diff --git a/images/debug/application.Dockerfile b/images/templates/common/debug/application.Dockerfile similarity index 100% rename from images/debug/application.Dockerfile rename to images/templates/common/debug/application.Dockerfile diff --git a/images/debug/cli/Dockerfile b/images/templates/common/debug/cli.Dockerfile similarity index 100% rename from images/debug/cli/Dockerfile rename to images/templates/common/debug/cli.Dockerfile diff --git a/images/debug/frontend/Dockerfile b/images/templates/common/debug/frontend.Dockerfile similarity index 100% rename from images/debug/frontend/Dockerfile rename to images/templates/common/debug/frontend.Dockerfile diff --git a/images/common/frontend/Dockerfile b/images/templates/common/frontend.Dockerfile similarity index 100% rename from images/common/frontend/Dockerfile rename to images/templates/common/frontend.Dockerfile diff --git a/images/common/frontend.dev.Dockerfile b/images/templates/common/frontend.dev.Dockerfile similarity index 100% rename from images/common/frontend.dev.Dockerfile rename to images/templates/common/frontend.dev.Dockerfile diff --git a/images/common/gateway/Dockerfile b/images/templates/common/gateway.Dockerfile similarity index 100% rename from images/common/gateway/Dockerfile rename to images/templates/common/gateway.Dockerfile diff --git a/images/common/node.Dockerfile b/images/templates/common/node.Dockerfile similarity index 100% rename from images/common/node.Dockerfile rename to images/templates/common/node.Dockerfile diff --git a/images/common/pipeline.Dockerfile b/images/templates/common/pipeline.Dockerfile similarity index 100% rename from images/common/pipeline.Dockerfile rename to images/templates/common/pipeline.Dockerfile diff --git a/images/common/services/jenkins/export/Dockerfile b/images/templates/export/jenkins.Dockerfile similarity index 100% rename from images/common/services/jenkins/export/Dockerfile rename to images/templates/export/jenkins.Dockerfile diff --git a/images/mount/frontend/Dockerfile b/images/templates/mount/frontend.Dockerfile similarity index 100% rename from images/mount/frontend/Dockerfile rename to images/templates/mount/frontend.Dockerfile diff --git a/images/mount/cli/Dockerfile b/images/templates/mount/pipeline.Dockerfile similarity index 100% rename from images/mount/cli/Dockerfile rename to images/templates/mount/pipeline.Dockerfile From 6e1a6e0268eebe7b908a8bc43dd4a6c2e61fb719 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Wed, 28 Jun 2023 22:12:50 +0300 Subject: [PATCH 03/26] Adjusted default progress type to auto for more compatibility --- generator/src/templates/deploy.bash.twig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/src/templates/deploy.bash.twig b/generator/src/templates/deploy.bash.twig index 7872bd3a3..d9d1977e7 100644 --- a/generator/src/templates/deploy.bash.twig +++ b/generator/src/templates/deploy.bash.twig @@ -103,7 +103,7 @@ readonly DEPLOYMENT_DIR="$(cd "${BASH_SOURCE%/*}" >/dev/null 2>&1 && pwd)" readonly DEPLOYMENT_PATH="${DEPLOYMENT_DIR/$PROJECT_DIR/.}" readonly SECRETS_FILE_PATH="${DEPLOYMENT_DIR}/.secrets" readonly SPRYKER_SDK_REVISION="{{ _sdk_revision | raw }}" -readonly PROGRESS_TYPE=${PROGRESS_TYPE:-tty} +readonly PROGRESS_TYPE=${PROGRESS_TYPE:-auto} VERBOSE=${VERBOSE:-0} # Predefined constants From 1c9721ca90fdbc4b832ed50f13cc256f89373143 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Wed, 28 Jun 2023 22:49:37 +0300 Subject: [PATCH 04/26] Replaced dependency on `ip` package in linux to use more common packages --- bin/environment/docker.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/environment/docker.sh b/bin/environment/docker.sh index 020e064a0..c0343f2ad 100644 --- a/bin/environment/docker.sh +++ b/bin/environment/docker.sh @@ -3,7 +3,7 @@ export DOCKER_BUILDKIT=1 require docker tr -require:linux ip grep awk +require:linux grep awk require:macos ipconfig require:windows tail cut @@ -87,7 +87,11 @@ function Environment::getHostIp() { case ${_PLATFORM} in linux) if ! Environment::isWSL; then - myIp=$(ip route get 1 | sed 's/^.*src \([^ ]*\).*$/\1/;q') + if command -v ip &> /dev/null; then + myIp=$(ip route get 1 | sed 's/^.*src \([^ ]*\).*$/\1/;q') + else + myIp=$(hostname -i | awk '{ print $1 }') + fi fi ;; macos) From fcecee29006d9b307031730d2da691164e61a7f1 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Wed, 28 Jun 2023 23:01:10 +0300 Subject: [PATCH 05/26] Fixed wrong path appeared after merging with master --- images/templates/export/jenkins.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/templates/export/jenkins.Dockerfile b/images/templates/export/jenkins.Dockerfile index bcf7ff16f..63ba0600d 100644 --- a/images/templates/export/jenkins.Dockerfile +++ b/images/templates/export/jenkins.Dockerfile @@ -44,7 +44,7 @@ RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ COPY ${DEPLOYMENT_PATH}/terraform/cli /envs/ COPY ${DEPLOYMENT_PATH}/context/jenkins/export/entrypoint.sh /entrypoint.sh COPY ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.model.JenkinsLocationConfiguration.xml /opt/jenkins.model.JenkinsLocationConfiguration.xml -COPY context/jenkins/export/nr-credentials.xml /opt/nr-credentials.xml +COPY ${DEPLOYMENT_PATH}/context/jenkins/export/nr-credentials.xml /opt/nr-credentials.xml RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] From 59d5a9f94866ed9a12f4efe91cfee73cd99b5b77 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Thu, 29 Jun 2023 12:08:38 +0300 Subject: [PATCH 06/26] Adjusted bake configuration by separating targets in export process allowing parallel pushes of application images --- images/templates/plans/export.docker-bake.hcl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/images/templates/plans/export.docker-bake.hcl b/images/templates/plans/export.docker-bake.hcl index a4b5fd85c..9bb45797e 100644 --- a/images/templates/plans/export.docker-bake.hcl +++ b/images/templates/plans/export.docker-bake.hcl @@ -1,7 +1,9 @@ {% import "images/templates/plans/tag." ~ buildDestination ~ "." ~ tagMode ~ ".twig" as tagMacro %} group "default" { targets = [ - "application", +{% for application in _applications %} + "app-{{ application | lower }}", +{% endfor %} "frontend", "pipeline", "jenkins", @@ -10,16 +12,17 @@ group "default" { {% include "images/templates/plans/common.docker-bake.hcl" with { folder: 'export'} %} -target "application" { +{% for application in _applications %} +target "app-{{ application | lower }}" { inherits = ["_common"] target = "application" tags = [ -{% for application in _applications %} {{ tagMacro.tagApplication(application, ",\n ") | spaceless }} -{% endfor %} ] } +{% endfor %} + target "frontend" { inherits = ["_common"] target = "frontend" From c0bc1baad21f3f469b7048bb845cd59d118a41ee Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Fri, 30 Jun 2023 01:17:56 +0300 Subject: [PATCH 07/26] - Added fallback for cases when buildx docker plugin is not installed - Removed intermediate dump-autoload as it probably takes more time than benifits for the following build --- bin/sdk/images/common.sh | 22 ++++- bin/sdk/images/engine/build.common.sh | 91 +++++++++++++++++++ bin/sdk/images/engine/build.sh | 83 ++--------------- bin/sdk/images/engine/buildx.sh | 13 +++ images/templates/baked/application.Dockerfile | 2 - 5 files changed, 133 insertions(+), 78 deletions(-) create mode 100644 bin/sdk/images/engine/build.common.sh create mode 100644 bin/sdk/images/engine/buildx.sh diff --git a/bin/sdk/images/common.sh b/bin/sdk/images/common.sh index 53a7dab56..335f39d21 100644 --- a/bin/sdk/images/common.sh +++ b/bin/sdk/images/common.sh @@ -18,13 +18,29 @@ function Images::destroy() { docker rmi -f "${SPRYKER_PLATFORM_IMAGE}" 2>/dev/null || true } +function Images::_checkBuildxVersion() { + min_version=${1} + version_regex="v([0-9]+\.[0-9]+\.[0-9]+)" + actual_version=$([[ $(docker buildx version) =~ $version_regex ]] && echo "${BASH_REMATCH[1]}") + greater_version=$(printf "%s\n%s\n" "${actual_version}" "${min_version}" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | tail -n 1) + if [ "$min_version" == "$greater_version" ]; then + return "${FALSE}" + fi +} + function Images::_build() { - # Checking availability of docker bake - if docker buildx --help | grep bake >/dev/null 2>&1; then - import sdk/images/engine/bake.sh + # Checking availability of docker bake or buildx + if docker buildx >/dev/null 2>&1; then + if Images::_checkBuildxVersion "0.8.99"; then + import sdk/images/engine/bake.sh + else + import sdk/images/engine/buildx.sh + Console::warn 'Warning! Upgrade `buildx` docker plugin to the latest version for better performance' + fi else import sdk/images/engine/build.sh + Console::warn 'Warning! Install `buildx` docker plugin for better performance' fi # Using temporary file for secrets as `docker secret` is only available for swarm mode. diff --git a/bin/sdk/images/engine/build.common.sh b/bin/sdk/images/engine/build.common.sh new file mode 100644 index 000000000..56e069f65 --- /dev/null +++ b/bin/sdk/images/engine/build.common.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +require docker + +function Images::_build::perform() { + + local folder=${1} + local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}} + local destination=${3:-print} + + local targetImage="${SPRYKER_DOCKER_PREFIX}_target:${SPRYKER_DOCKER_TAG}" + + Console::verbose "${INFO}Building images${NC}" + + # Primal build including all stages to fully benefit from multistage features + Images::build::_runBuild --target "target" -t "${targetImage}" + + # Tagging all the images by running the same build targeting different stages + source ${DEPLOYMENT_PATH}/images/${folder}/${destination}.sh + + local -a arguments=('--quiet' '--progress' 'auto') + + if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then + arguments+=('--ssh' 'default') + fi + + if Images::needPush; then + Console::verbose "${INFO}Tagging and pushing images${NC}" + else + Console::verbose "${INFO}Tagging images${NC}" + fi + + arguments+=($(Images::_build::prepareArguments)) + + local -a pids + local targetData + for targetData in "${TARGET_TAGS[@]}"; do + eval "${targetData}" + + local -a tagArguments=() + local tag + for tag in "${TAGS[@]}"; do + tagArguments+=('-t' "${tag}") + done + + Console::info "${YELLOW}Target:${NC} ${TARGET}" + Console::verbose "${YELLOW}Tags:${NC}" + Console::verbose "${DGRAY}${TAGS[@]}${NC}" + Console::verbose "${YELLOW}Hash:${NC}" + Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}" + + Images::_build::afterTaggingAnImage "${TAGS[@]}" & + pids+=($!) + done + + docker rmi -f "${targetImage}" >/dev/null 2>&1 || true + + local pid + for pid in "$pids[@]"; do + wait $pid || exit 1 + done +} + +function Images::build::_runBuild { + + docker build \ + -f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \ + "${@}" \ + --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \ + --label "spryker.project=${SPRYKER_DOCKER_PREFIX}" \ + --label "spryker.revision=${SPRYKER_BUILD_HASH}" \ + --label "spryker.sdk.revision=${SPRYKER_SDK_REVISION}" \ + --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \ + --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \ + --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \ + --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \ + --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ + --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \ + --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \ + --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \ + --build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \ + --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \ + --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \ + --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH}" \ + --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP}" \ + --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \ + --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \ + --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \ + --build-arg "USER_UID=${USER_UID}" \ + ./ >&2 +} diff --git a/bin/sdk/images/engine/build.sh b/bin/sdk/images/engine/build.sh index 0e96c115b..04df5442d 100644 --- a/bin/sdk/images/engine/build.sh +++ b/bin/sdk/images/engine/build.sh @@ -1,82 +1,19 @@ #!/bin/bash -require docker +import sdk/images/engine/build.common.sh -function Images::_build::perform() { - - local folder=${1} - local TARGET_TAG=${2:-${SPRYKER_DOCKER_TAG}} - local destination=${3:-print} - - local targetImage="${SPRYKER_DOCKER_PREFIX}_target:${SPRYKER_DOCKER_TAG}" - - Console::verbose "${INFO}Building images${NC}" - - # Primal build including all stages to fully benefit from multistage features - Images::build::_runBuild --target "target" -t "${targetImage}" - - # Tagging all the images by running the same build targeting different stages - source ${DEPLOYMENT_PATH}/images/${folder}/${destination}.sh - - local -a arguments=('--quiet' '--progress' 'auto') +function Images::_build::prepareArguments() { + : +} - if [ -n "${SSH_AUTH_SOCK_IN_CLI}" ]; then - arguments+=('--ssh' 'default') - fi +function Images::_build::afterTaggingAnImage() { +set -x + local -a tagsToPush=("${@}") if Images::needPush; then - arguments+=('--push') - Console::verbose "${INFO}Tagging and pushing images${NC}" - else - Console::verbose "${INFO}Tagging images${NC}" - fi - - local targetData - for targetData in "${TARGET_TAGS[@]}"; do - eval "${targetData}" - - local -a tagArguments=() local tag - for tag in "${TAGS[@]}"; do - tagArguments+=('-t' "${tag}") + for tag in "${tagsToPush[@]}"; do + docker push ${tag} done - - Console::info "${YELLOW}Target:${NC} ${TARGET}" - Console::verbose "${YELLOW}Tags:${NC}" - Console::verbose "${DGRAY}${TAGS[@]}${NC}" - Console::verbose "${YELLOW}Hash:${NC}" - Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}" - - done - - docker rmi -f "${targetImage}" >/dev/null 2>&1 || true -} - -function Images::build::_runBuild { - - docker build \ - -f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \ - "${@}" \ - --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \ - --label "spryker.project=${SPRYKER_DOCKER_PREFIX}" \ - --label "spryker.revision=${SPRYKER_BUILD_HASH}" \ - --label "spryker.sdk.revision=${SPRYKER_SDK_REVISION}" \ - --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \ - --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \ - --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \ - --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \ - --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \ - --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \ - --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \ - --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \ - --build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \ - --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \ - --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \ - --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH}" \ - --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP}" \ - --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \ - --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \ - --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \ - --build-arg "USER_UID=${USER_UID}" \ - ./ >&2 + fi } diff --git a/bin/sdk/images/engine/buildx.sh b/bin/sdk/images/engine/buildx.sh new file mode 100644 index 000000000..1f04111c4 --- /dev/null +++ b/bin/sdk/images/engine/buildx.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +import sdk/images/engine/build.common.sh + +function Images::_build::prepareArguments() { + if Images::needPush; then + echo '--push' + fi +} + +function Images::_build::afterTaggingAnImage() { + : +} diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 11f9ee09e..c010fadd8 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -29,8 +29,6 @@ LABEL "spryker.image" "none" USER spryker -RUN composer dump-autoload -o - ARG SPRYKER_PIPELINE ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} ARG SPRYKER_DB_ENGINE From 8c4fe3c527c609bd7be84c9158d10660a954026a Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Fri, 30 Jun 2023 02:50:11 +0300 Subject: [PATCH 08/26] - Reduced requirement for buildx version as it is confirmed working on 0.7.1 - Upgraded syntax docker version - Removed parallelisation for fallback build (without buildx) --- bin/sdk/images/common.sh | 2 +- bin/sdk/images/engine/build.common.sh | 9 +-------- generator/Dockerfile | 2 +- generator/deploy-file-generator/Dockerfile | 2 +- images/templates/baked/Dockerfile.twig | 2 +- images/templates/export/Dockerfile.twig | 2 +- images/templates/mount/Dockerfile.twig | 2 +- 7 files changed, 7 insertions(+), 14 deletions(-) diff --git a/bin/sdk/images/common.sh b/bin/sdk/images/common.sh index 335f39d21..6e4736b08 100644 --- a/bin/sdk/images/common.sh +++ b/bin/sdk/images/common.sh @@ -32,7 +32,7 @@ function Images::_build() { # Checking availability of docker bake or buildx if docker buildx >/dev/null 2>&1; then - if Images::_checkBuildxVersion "0.8.99"; then + if Images::_checkBuildxVersion "0.6.99"; then import sdk/images/engine/bake.sh else import sdk/images/engine/buildx.sh diff --git a/bin/sdk/images/engine/build.common.sh b/bin/sdk/images/engine/build.common.sh index 56e069f65..ee2529c19 100644 --- a/bin/sdk/images/engine/build.common.sh +++ b/bin/sdk/images/engine/build.common.sh @@ -32,7 +32,6 @@ function Images::_build::perform() { arguments+=($(Images::_build::prepareArguments)) - local -a pids local targetData for targetData in "${TARGET_TAGS[@]}"; do eval "${targetData}" @@ -49,16 +48,10 @@ function Images::_build::perform() { Console::verbose "${YELLOW}Hash:${NC}" Images::build::_runBuild --target "${TARGET}" "${arguments[@]}" "${tagArguments[@]}" - Images::_build::afterTaggingAnImage "${TAGS[@]}" & - pids+=($!) + Images::_build::afterTaggingAnImage "${TAGS[@]}" done docker rmi -f "${targetImage}" >/dev/null 2>&1 || true - - local pid - for pid in "$pids[@]"; do - wait $pid || exit 1 - done } function Images::build::_runBuild { diff --git a/generator/Dockerfile b/generator/Dockerfile index 4364f2a06..a0c5f4805 100644 --- a/generator/Dockerfile +++ b/generator/Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.5 ARG SPRYKER_PHP_VERSION=7.3 FROM spryker/php:${SPRYKER_PHP_VERSION} diff --git a/generator/deploy-file-generator/Dockerfile b/generator/deploy-file-generator/Dockerfile index b6ba01fb2..e9d5e7878 100644 --- a/generator/deploy-file-generator/Dockerfile +++ b/generator/deploy-file-generator/Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.5 ARG SPRYKER_PHP_VERSION=7.3 FROM spryker/php:${SPRYKER_PHP_VERSION} diff --git a/images/templates/baked/Dockerfile.twig b/images/templates/baked/Dockerfile.twig index 20c41d111..d92e9291d 100644 --- a/images/templates/baked/Dockerfile.twig +++ b/images/templates/baked/Dockerfile.twig @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.5 {% include "images/templates/common/args.Dockerfile" with _context %} diff --git a/images/templates/export/Dockerfile.twig b/images/templates/export/Dockerfile.twig index 1467606e0..c80376047 100644 --- a/images/templates/export/Dockerfile.twig +++ b/images/templates/export/Dockerfile.twig @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.5 {% include "images/templates/common/args.Dockerfile" with _context %} diff --git a/images/templates/mount/Dockerfile.twig b/images/templates/mount/Dockerfile.twig index 682f764d3..bf8000022 100644 --- a/images/templates/mount/Dockerfile.twig +++ b/images/templates/mount/Dockerfile.twig @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:experimental +# syntax = docker/dockerfile:1.5 {% include "images/templates/common/args.Dockerfile" with _context %} From ba082cbc0e13b69d2baf91611bbebcba79c1a3d2 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Fri, 30 Jun 2023 03:07:11 +0300 Subject: [PATCH 09/26] - Fixed mount cache permissions and wrong id --- images/templates/baked/assets.builder.Dockerfile | 2 +- images/templates/baked/npm.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 25612de09..9be068939 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -54,7 +54,7 @@ ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} COPY --from=npm-precacher /tmp/.dependency* /tmp/ RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 \ - --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules \ + --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules,uid=1000 \ echo "MODE: ${SPRYKER_ASSETS_MODE}" \ && vendor/bin/console transfer:generate \ && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} -vvv diff --git a/images/templates/baked/npm.Dockerfile b/images/templates/baked/npm.Dockerfile index aa587d23d..a64dfe3ce 100644 --- a/images/templates/baked/npm.Dockerfile +++ b/images/templates/baked/npm.Dockerfile @@ -3,7 +3,7 @@ LABEL "spryker.image" "none" COPY --chown=spryker:spryker package.json* package-lock.json* /root/ -RUN --mount=type=cache,id=npm-cache,sharing=locked,target=/root/.npm \ +RUN --mount=type=cache,id=npm,sharing=locked,target=/root/.npm \ --mount=type=cache,id=npm-modules,sharing=locked,target=/root/node_modules \ cd /root \ && sh -c 'if [ -f ${srcRoot}/package.json ]; then npm install --prefer-offline || npm ci --prefer-offline || true; fi' From a69258f84177c48abb45be83997b169cbc8a7ae0 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Fri, 30 Jun 2023 10:04:25 +0300 Subject: [PATCH 10/26] Fixed git cleanup command (mostly for nonsplit) --- images/templates/baked/application.Dockerfile | 2 +- images/templates/baked/pipeline.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index c010fadd8..7919ac100 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -10,7 +10,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \ - && find ./vendor -type d -name \.git* -prune -exec rm -rf {} + + && find ./vendor -type d -name "\.git*" -prune -exec rm -rf {}/** + COPY --chown=spryker:spryker src ${srcRoot}/src COPY --chown=spryker:spryker config ${srcRoot}/config diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index b90a428d3..d4486a5b8 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -10,7 +10,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ && composer install --no-interaction \ - && find ./vendor -type d -name \.git -prune -exec rm -rf {} + + && find ./vendor -type d -name "\.git*" -prune -exec rm -rf {}/** + # Tests contain transfer declaration COPY --chown=spryker:spryker tests ${srcRoot}/tests From 31715fbc208ed8febe98a766ae0267223701700f Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Fri, 30 Jun 2023 12:46:55 +0300 Subject: [PATCH 11/26] Fixed git cleanup command (mostly for nonsplit). Part 2 --- images/templates/baked/application.Dockerfile | 2 +- images/templates/baked/pipeline.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 7919ac100..51986f5b0 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -10,7 +10,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \ - && find ./vendor -type d -name "\.git*" -prune -exec rm -rf {}/** + + && find ./vendor -type d -name ".git*" -prune -exec sh -c 'cd {} && rm -rf -- ./*' \; COPY --chown=spryker:spryker src ${srcRoot}/src COPY --chown=spryker:spryker config ${srcRoot}/config diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index d4486a5b8..806cd2c51 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -10,7 +10,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ && composer install --no-interaction \ - && find ./vendor -type d -name "\.git*" -prune -exec rm -rf {}/** + + && find ./vendor -type d -name ".git*" -prune -exec sh -c 'cd {} && rm -rf -- ./*' \; # Tests contain transfer declaration COPY --chown=spryker:spryker tests ${srcRoot}/tests From 1787de8de6a5172edbe29a412cc480e4b031d45d Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Sun, 2 Jul 2023 03:42:52 +0300 Subject: [PATCH 12/26] Deep speed, number of layers and image size optimizations. Restructuring dependencies. --- images/templates/baked/application.Dockerfile | 102 ++++++++++++------ .../templates/baked/assets.builder.Dockerfile | 35 +++--- images/templates/baked/frontend.Dockerfile | 3 +- images/templates/baked/npm.Dockerfile | 2 +- images/templates/baked/pipeline.Dockerfile | 45 ++++++-- .../templates/common/application.Dockerfile | 53 +++++---- .../common/application.dev.Dockerfile | 2 +- images/templates/common/cli.Dockerfile | 32 +++--- .../common/debug/application.Dockerfile | 11 +- images/templates/common/debug/cli.Dockerfile | 4 +- .../common/debug/frontend.Dockerfile | 2 +- images/templates/common/frontend.Dockerfile | 14 ++- images/templates/common/gateway.Dockerfile | 10 +- images/templates/common/node.Dockerfile | 8 ++ images/templates/common/pipeline.Dockerfile | 17 +-- images/templates/export/jenkins.Dockerfile | 18 ++-- images/templates/mount/application.Dockerfile | 14 +-- images/templates/mount/frontend.Dockerfile | 1 + images/templates/mount/pipeline.Dockerfile | 2 +- 19 files changed, 213 insertions(+), 162 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 51986f5b0..7ab3144d8 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -1,53 +1,95 @@ -FROM application-basic AS application-codebase +FROM busybox AS stash-data LABEL "spryker.image" "none" -USER spryker +COPY --chown=spryker:spryker data /data-without-import -# Install composer modules for Spryker -COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/ +RUN rm -rf /data-without-import/import + +# ----------------------------- + +FROM busybox AS stash-src + +COPY --chown=spryker:spryker data /data-without-import + +RUN mkdir /data + +COPY src /data/src +COPY config /data/config +COPY resource[s] /data/resources +COPY LICENSE /data + +# ----------------------------- + +FROM ${SPRYKER_PLATFORM_IMAGE} AS stash-rsync +LABEL "spryker.image" "none" + +RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ + rsync \ + ; fi' + +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ + bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ + rsync \ + ; fi' + +RUN --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + cp -fp /usr/bin/rsync /rsync/ \ + && ldd /usr/bin/rsync | awk '/=>/ { print $3 }' | xargs -I '{}' cp -fp '{}' /rsync + +# ----------------------------- + +FROM ${SPRYKER_PLATFORM_IMAGE} AS application-codebase +LABEL "spryker.image" "none" + +USER spryker:spryker + +COPY --chown=spryker:spryker composer.json composer.lock *.php ${srcRoot}/ ARG SPRYKER_COMPOSER_MODE RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ + --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \ - && find ./vendor -type d -name ".git*" -prune -exec sh -c 'cd {} && rm -rf -- ./*' \; - -COPY --chown=spryker:spryker src ${srcRoot}/src -COPY --chown=spryker:spryker config ${srcRoot}/config -COPY --chown=spryker:spryker *.php ${srcRoot}/ -# TODO: Move codebase related stuff from data folder to src. In ideal: translations in /data are needed for build below -COPY --chown=spryker:spryker data ${srcRoot}/data -RUN chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true + && rm -rf vendor/composer \ + && rm -rf vendor/bin \ + && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} -ENV PATH=${srcRoot}/vendor/bin:$PATH +# Dependency: rsync is needed for next steps +COPY --from=stash-rsync /tmp/.dependency* /tmp/ -ARG APPLICATION_ENV -ENV APPLICATION_ENV=${APPLICATION_ENV} +# ----------------------------- -FROM application-codebase AS application-before-stamp +FROM application-basic AS application-before-stamp LABEL "spryker.image" "none" -USER spryker +USER spryker:spryker -ARG SPRYKER_PIPELINE -ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} -ARG SPRYKER_DB_ENGINE -ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} +# Dependency: Run ONLY after vendor folder is +COPY --from=application-codebase --chown=spryker:spryker ${srcRoot}/composer.* ${srcRoot}/*.php ${srcRoot}/ + +# Install composer modules for Spryker +RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ + --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + --mount=type=tmpfs,target=/var/run/opcache/ \ + LD_LIBRARY_PATH=/rsync time /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + --exclude 'tests/' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' --exclude '*.md' \ + --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' -COPY --chown=spryker:spryker data ${srcRoot}/data -RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production +COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} +COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/data ARG SPRYKER_COMPOSER_AUTOLOAD -RUN composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} +RUN --mount=type=tmpfs,target=/var/run/opcache/ \ + bash -c 'chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true' \ + && ls -al \ + && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production \ + && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} COPY --chown=spryker:spryker public ${srcRoot}/public -COPY --chown=spryker:spryker frontend ${srcRoot}/frontend -COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn -COPY --chown=spryker:spryker .* *.* LICENSE ${srcRoot}/ USER root -RUN rm -rf /var/run/opcache/* -RUN chown -R spryker:spryker /home/spryker +# RUN chown -R spryker:spryker /home/spryker CMD [ "php-fpm", "--nodaemonize" ] EXPOSE 9000 diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 9be068939..8d02d7695 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -1,13 +1,9 @@ -FROM application-codebase AS assets-builder +FROM application-basic AS assets-builder LABEL "spryker.image" "none" USER root -COPY --from=node-distributive /usr/lib /usr/lib -COPY --from=node-distributive /usr/local/share /usr/local/share -COPY --from=node-distributive /usr/local/lib /usr/local/lib -COPY --from=node-distributive /usr/local/include /usr/local/include -COPY --from=node-distributive /usr/local/bin /usr/local/bin +COPY --from=node-distributive /node/usr /usr/ RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ @@ -15,12 +11,7 @@ RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ python3 \ g++ \ make \ - ; fi' - -# Debian contains outdated Yarn package -RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ - --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ - bash -c 'if [ ! -z "$(which apt)" ]; then \ + ; \ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ apt update -y && apt install -y \ @@ -37,24 +28,22 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc make \ ; fi' -USER spryker +USER spryker:spryker COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .* *.* ${srcRoot}/ COPY --chown=spryker:spryker frontend* ${srcRoot}/frontend COPY --chown=spryker:spryker public* ${srcRoot}/public COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn -COPY --chown=spryker:spryker config/Yves ${srcRoot}/config/Yves - -ARG SPRYKER_ASSETS_MODE -ENV SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE} -ARG SPRYKER_PIPELINE -ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} +COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} +COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/data # This instruction is necessary to ouline dependency on precacher to make sure assets are built after COPY --from=npm-precacher /tmp/.dependency* /tmp/ +COPY --from=application-codebase /tmp/.dependency* /tmp/ RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 \ - --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules,uid=1000 \ - echo "MODE: ${SPRYKER_ASSETS_MODE}" \ - && vendor/bin/console transfer:generate \ - && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} -vvv + --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules,uid=1000 \ + --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ + echo "MODE: ${SPRYKER_ASSETS_MODE}" \ + && vendor/bin/console transfer:generate \ + && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} -vvv diff --git a/images/templates/baked/frontend.Dockerfile b/images/templates/baked/frontend.Dockerfile index f7e8fc72c..910406ba1 100644 --- a/images/templates/baked/frontend.Dockerfile +++ b/images/templates/baked/frontend.Dockerfile @@ -1,8 +1,7 @@ FROM frontend-basic as frontend-before-stamp LABEL "spryker.image" "none" -RUN mkdir -p ${srcRoot}/public && chmod 0777 ${srcRoot}/public -COPY --from=assets-builder --chown=root:root ${srcRoot}/public ${srcRoot}/public +COPY --from=assets-builder --link --chown=root:root --chmod=755 ${srcRoot}/public ${srcRoot}/public FROM frontend-before-stamp as frontend LABEL "spryker.image" "frontend" diff --git a/images/templates/baked/npm.Dockerfile b/images/templates/baked/npm.Dockerfile index a64dfe3ce..1b9a0c60e 100644 --- a/images/templates/baked/npm.Dockerfile +++ b/images/templates/baked/npm.Dockerfile @@ -1,7 +1,7 @@ FROM node-distributive AS npm-precacher LABEL "spryker.image" "none" -COPY --chown=spryker:spryker package.json* package-lock.json* /root/ +COPY --chown=spryker:spryker package.jso[n] package-lock.jso[n] /root/ RUN --mount=type=cache,id=npm,sharing=locked,target=/root/.npm \ --mount=type=cache,id=npm-modules,sharing=locked,target=/root/node_modules \ diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 806cd2c51..b2ada5095 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -1,24 +1,53 @@ +FROM application-codebase AS application-codebase-dev +LABEL "spryker.image" "none" + +RUN --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ + --mount=type=cache,id=vendor-dev,target=/data/vendor.dev,uid=1000 \ + LD_LIBRARY_PATH=/rsync /rsync/rsync -ap ./vendor/ ./vendor.dev + +RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ + --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ + --mount=type=cache,id=vendor-dev,target=/data/vendor,uid=1000 \ + set -o allexport && . /run/secrets/secrets-env && set +o allexport \ + && rm -rf vendor/composer \ + && composer install --no-scripts --no-interaction + +# ----------------------------- + FROM pipeline-basic as pipeline-before-stamp LABEL "spryker.image" "none" -USER spryker +USER spryker:spryker +COPY --from=application-codebase-dev --chown=spryker:spryker ${srcRoot}/composer.* ${srcRoot}/*.php ${srcRoot}/ # Install dev modules for Spryker -COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/ -ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ + --mount=type=cache,id=vendor-dev,target=/vendor,uid=1000 \ + --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + --mount=type=tmpfs,target=/var/run/opcache/ \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && composer install --no-interaction \ - && find ./vendor -type d -name ".git*" -prune -exec sh -c 'cd {} && rm -rf -- ./*' \; + && LD_LIBRARY_PATH=/rsync time /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ \ + && time composer install --no-interaction \ + && find ./vendor -type d -name ".git*" -prune -exec rm -rf {} \; +# ^ Running install again to run all scripts +COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} +COPY --chown=spryker:spryker data ${srcRoot}/data # Tests contain transfer declaration -COPY --chown=spryker:spryker tests ${srcRoot}/tests +COPY --chown=spryker:spryker test[s] /${srcRoot}/tests ENV DEVELOPMENT_CONSOLE_COMMANDS=1 -RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-development -RUN composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} +ARG SPRYKER_COMPOSER_AUTOLOAD +RUN --mount=type=tmpfs,target=/var/run/opcache/ \ + vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development \ + && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} + +COPY --link --chown=spryker:spryker fronten[d] ${srcRoot}/frontend +COPY --link --chown=spryker:spryker .yar[n] ${srcRoot}/.yarn +COPY --link --chown=spryker:spryker .* *.* LICENSE ${srcRoot} FROM pipeline-before-stamp as pipeline LABEL "spryker.image" "pipeline" diff --git a/images/templates/common/application.Dockerfile b/images/templates/common/application.Dockerfile index f6fed8d69..c43bcda35 100644 --- a/images/templates/common/application.Dockerfile +++ b/images/templates/common/application.Dockerfile @@ -7,20 +7,6 @@ ENV COMPOSER_IGNORE_CHROMEDRIVER=1 ENV {{ envName }}='{{ envValue }}' {% endfor %} -WORKDIR /data - -# Create log directory -ARG SPRYKER_LOG_DIRECTORY -ENV SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY} -RUN mkdir -p ${SPRYKER_LOG_DIRECTORY} && \ -chown spryker:spryker ${SPRYKER_LOG_DIRECTORY} - -# Creates the list of known hosts -ARG KNOWN_HOSTS -RUN mkdir -p /home/spryker/.ssh && chmod 0700 /home/spryker/.ssh -RUN bash -c '[ ! -z "${KNOWN_HOSTS}" ] && ssh-keyscan -t rsa ${KNOWN_HOSTS} >> /home/spryker/.ssh/known_hosts || true' -RUN chown spryker:spryker -R /home/spryker/.ssh - # PHP-FPM environment variables ENV PHP_FPM_PM=dynamic ENV PHP_FPM_PM_MAX_CHILDREN=4 @@ -30,25 +16,38 @@ ENV PHP_FPM_PM_MAX_SPARE_SERVERS=2 ENV PHP_FPM_PM_MAX_REQUESTS=500 ENV PHP_FPM_REQUEST_TERMINATE_TIMEOUT=1m -# PHP configuration +WORKDIR /data + ARG DEPLOYMENT_PATH COPY ${DEPLOYMENT_PATH}/context/php/php-fpm.d/worker.conf /usr/local/etc/php-fpm.d/worker.conf -RUN bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/php-fpm.d/worker.conf || true" COPY ${DEPLOYMENT_PATH}/context/php/php.ini /usr/local/etc/php/ COPY ${DEPLOYMENT_PATH}/context/php/conf.d/90-opcache.ini /usr/local/etc/php/conf.d -# removing default opcache.ini -RUN rm -f /usr/local/etc/php/conf.d/opcache.ini +COPY ${DEPLOYMENT_PATH}/context/php/conf.d/99-from-deploy-yaml-php.ini /usr/local/etc/php/conf.d/ +COPY --link --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/jenkins/jenkins.docker.xml.twig /home/spryker/jenkins.docker.xml.twig +COPY --link --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/php/build.php /home/spryker/build.php +ARG SPRYKER_LOG_DIRECTORY +ARG KNOWN_HOSTS +ENV SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY} +RUN mkdir -p ${SPRYKER_LOG_DIRECTORY} \ + && chown spryker:spryker ${SPRYKER_LOG_DIRECTORY} \ + && mkdir -p /home/spryker/.ssh && chmod 0700 /home/spryker/.ssh \ + && bash -c '[ ! -z "${KNOWN_HOSTS}" ] && ssh-keyscan -t rsa ${KNOWN_HOSTS} >> /home/spryker/.ssh/known_hosts || true' \ + && chown spryker:spryker -R /home/spryker/.ssh \ + && rm -f /usr/local/etc/php/conf.d/opcache.ini \ {% if _phpExtensions is defined and _phpExtensions is not empty %} {% for phpExtention in _phpExtensions %} -RUN mv /usr/local/etc/php/disabled/{{phpExtention}}.ini /usr/local/etc/php/conf.d/90-{{phpExtention}}.ini + && mv /usr/local/etc/php/disabled/{{phpExtention}}.ini /usr/local/etc/php/conf.d/90-{{phpExtention}}.ini \ {% endfor %} {% endif %} - -COPY ${DEPLOYMENT_PATH}/context/php/conf.d/99-from-deploy-yaml-php.ini /usr/local/etc/php/conf.d/ - -# Jenkins -COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/jenkins/jenkins.docker.xml.twig /home/spryker/jenkins.docker.xml.twig - -# Build info -COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/php/build.php /home/spryker/build.php + && rm -rf /var/run \ + && /usr/bin/install -d -m 777 /var/run/opcache \ + && bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/php-fpm.d/worker.conf || true" + +ARG SPRYKER_PIPELINE +ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} +ARG SPRYKER_DB_ENGINE +ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} +ARG APPLICATION_ENV +ENV APPLICATION_ENV=${APPLICATION_ENV} +ENV PATH=${srcRoot}/vendor/bin:$PATH diff --git a/images/templates/common/application.dev.Dockerfile b/images/templates/common/application.dev.Dockerfile index 79468c380..770409fbb 100644 --- a/images/templates/common/application.dev.Dockerfile +++ b/images/templates/common/application.dev.Dockerfile @@ -2,5 +2,5 @@ FROM application as application-dev LABEL "spryker.image" "none" # Make self-signed certificate to be trusted locally -COPY ${DEPLOYMENT_PATH}/context/nginx/ssl/ca.crt /usr/local/share/ca-certificates +COPY --link ${DEPLOYMENT_PATH}/context/nginx/ssl/ca.crt /usr/local/share/ca-certificates RUN update-ca-certificates diff --git a/images/templates/common/cli.Dockerfile b/images/templates/common/cli.Dockerfile index cc350b448..edbf2e40f 100644 --- a/images/templates/common/cli.Dockerfile +++ b/images/templates/common/cli.Dockerfile @@ -1,5 +1,5 @@ -FROM pipeline-before-stamp as cli -LABEL "spryker.image" "cli" +FROM ${SPRYKER_PLATFORM_IMAGE} as cli-dependencies +LABEL "spryker.image" "none" USER root @@ -10,6 +10,13 @@ RUN mkdir -p /tmp/blackfire \ && mv /tmp/blackfire/blackfire /usr/bin/blackfire \ && rm -Rf /tmp/blackfire +FROM pipeline-before-stamp as cli +LABEL "spryker.image" "cli" + +USER root + +COPY --from=cli-dependencies --link /usr/bin/blackfire /usr/bin/blackfire + RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ @@ -27,22 +34,19 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc make \ ; fi' -USER spryker +USER spryker:spryker -RUN mkdir -p /home/spryker/env -ARG DEPLOYMENT_PATH -COPY --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/cli /home/spryker/bin -RUN find /home/spryker/bin -type f -exec chmod +x {} \; -ENV PATH=/home/spryker/bin:$PATH - -RUN mkdir -p /home/spryker/ssh-relay/ && chmod 777 /home/spryker/ssh-relay && touch /home/spryker/ssh-relay/ssh-auth.sock && chmod 666 /home/spryker/ssh-relay/ssh-auth.sock \ - && touch /tmp/stdout && touch /tmp/stderr && chmod 666 /tmp/stdout && chmod 666 /tmp/stderr - -RUN mkdir -p /home/spryker/history && touch /home/spryker/history/.bash_history && chmod 0600 /home/spryker/history/.bash_history ENV HISTFILE=/home/spryker/history/.bash_history - ENV NEWRELIC_ENABLED=0 +ARG DEPLOYMENT_PATH +COPY --chown=spryker:spryker --link --chmod=755 ${DEPLOYMENT_PATH}/context/cli /home/spryker/bin + +RUN mkdir -p /home/spryker/env \ + && mkdir -p /home/spryker/ssh-relay/ && chmod 777 /home/spryker/ssh-relay && touch /home/spryker/ssh-relay/ssh-auth.sock && chmod 666 /home/spryker/ssh-relay/ssh-auth.sock \ + && touch /tmp/stdout && touch /tmp/stderr && chmod 666 /tmp/stdout && chmod 666 /tmp/stderr \ + && mkdir -p /home/spryker/history && touch /home/spryker/history/.bash_history && chmod 0600 /home/spryker/history/.bash_history + ARG SPRYKER_BUILD_HASH ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} ARG SPRYKER_BUILD_STAMP diff --git a/images/templates/common/debug/application.Dockerfile b/images/templates/common/debug/application.Dockerfile index ef02b03a9..cf41d62c0 100644 --- a/images/templates/common/debug/application.Dockerfile +++ b/images/templates/common/debug/application.Dockerfile @@ -11,12 +11,13 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc supervisor \ ; fi' -RUN /usr/bin/install -d -m 777 /var/run/opcache/debug ARG DEPLOYMENT_PATH -COPY ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ -RUN bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/debug.php-fpm.conf/worker.conf || true" -COPY ${DEPLOYMENT_PATH}/context/php/debug/supervisord.conf /etc/supervisor/supervisord.conf -RUN mkdir -p /var/log/supervisor +COPY --link ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ +COPY --link ${DEPLOYMENT_PATH}/context/php/debug/supervisord.conf /etc/supervisor/supervisord.conf + +RUN /usr/bin/install -d -m 777 /var/run/opcache/debug \ + && mkdir -p /var/log/supervisor \ + && bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/debug.php-fpm.conf/worker.conf || true" CMD [ "/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ] EXPOSE 9000 9001 diff --git a/images/templates/common/debug/cli.Dockerfile b/images/templates/common/debug/cli.Dockerfile index b78708d32..ed0af657a 100644 --- a/images/templates/common/debug/cli.Dockerfile +++ b/images/templates/common/debug/cli.Dockerfile @@ -2,7 +2,7 @@ USER root RUN /usr/bin/install -d -m 777 /var/run/opcache/debug -USER spryker +USER spryker:spryker ARG DEPLOYMENT_PATH -COPY ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ +COPY --link ${DEPLOYMENT_PATH}/context/php/debug/etc/ /usr/local/etc/ diff --git a/images/templates/common/debug/frontend.Dockerfile b/images/templates/common/debug/frontend.Dockerfile index 683acb0f8..1e1bd969f 100644 --- a/images/templates/common/debug/frontend.Dockerfile +++ b/images/templates/common/debug/frontend.Dockerfile @@ -1,4 +1,4 @@ ENV SPRYKER_XDEBUG_MODE_ENABLE=1 ARG DEPLOYMENT_PATH -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/debug.default.conf /etc/nginx/template/debug.default.conf +COPY --link --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/debug.default.conf /etc/nginx/template/debug.default.conf diff --git a/images/templates/common/frontend.Dockerfile b/images/templates/common/frontend.Dockerfile index 2f00ccd26..3ed852b6b 100644 --- a/images/templates/common/frontend.Dockerfile +++ b/images/templates/common/frontend.Dockerfile @@ -4,20 +4,18 @@ LABEL "spryker.image" "frontend-basic" ENV srcRoot /data -RUN mkdir -p /etc/nginx/template/ && chmod 0777 /etc/nginx/template/ ARG DEPLOYMENT_PATH -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/nginx.original.conf /etc/nginx/nginx.conf -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/frontend.default.conf.tmpl /etc/nginx/template/default.conf.tmpl -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/resolver.conf.tmpl /etc/nginx/template/resolver.conf.tmpl -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/auth /etc/nginx/auth -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/entrypoint.sh / -RUN chmod +x /entrypoint.sh +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/nginx.original.conf /etc/nginx/nginx.conf +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/conf.d/frontend.default.conf.tmpl /etc/nginx/template/default.conf.tmpl +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/conf.d/resolver.conf.tmpl /etc/nginx/template/resolver.conf.tmpl +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/auth /etc/nginx/auth +COPY --chown=root:root --link --chmod=755 ${DEPLOYMENT_PATH}/context/nginx/entrypoint.sh / ENV SPRYKER_DNS_RESOLVER_FLAGS="valid=10s ipv6=off" ENV SPRYKER_DNS_RESOLVER_IP="" ENV SPRYKER_MAINTENANCE_MODE_ENABLED="0" -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/build.json /tmp/build.json +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/build.json /tmp/build.json ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/images/templates/common/gateway.Dockerfile b/images/templates/common/gateway.Dockerfile index be9122b3a..8eaa4388c 100644 --- a/images/templates/common/gateway.Dockerfile +++ b/images/templates/common/gateway.Dockerfile @@ -2,11 +2,11 @@ FROM nginx:alpine as gateway LABEL "spryker.image" "gateway" ARG DEPLOYMENT_PATH -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/nginx.with.stream.conf /etc/nginx/nginx.conf -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/conf.d/gateway.default.conf /etc/nginx/templates/default.conf.template -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/stream.d/gateway.default.conf /etc/nginx/stream.d/default.conf -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/vhost.d/ssl.default.conf /etc/nginx/vhost.d/ssl.default.conf -COPY --chown=root:root ${DEPLOYMENT_PATH}/context/nginx/ssl /etc/nginx/ssl +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/nginx.with.stream.conf /etc/nginx/nginx.conf +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/conf.d/gateway.default.conf /etc/nginx/templates/default.conf.template +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/stream.d/gateway.default.conf /etc/nginx/stream.d/default.conf +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/vhost.d/ssl.default.conf /etc/nginx/vhost.d/ssl.default.conf +COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/ssl /etc/nginx/ssl ENV SPRYKER_XDEBUG_ENABLE=0 diff --git a/images/templates/common/node.Dockerfile b/images/templates/common/node.Dockerfile index da0d10329..397c0b2f9 100644 --- a/images/templates/common/node.Dockerfile +++ b/images/templates/common/node.Dockerfile @@ -3,3 +3,11 @@ LABEL "spryker.image" "none" ARG SPRYKER_NPM_VERSION RUN npm install -g npm@${SPRYKER_NPM_VERSION} + +# TODO get the difference what need to be copied only to reduce the layer size +RUN mkdir -p /node/usr/lib && mkdir -p /node/usr/local \ + && cp -rp /usr/lib/ /node/usr \ + && cp -rp /usr/local/share/ /node/usr/local \ + && cp -rp /usr/local/lib/ /node/usr/local \ + && cp -rp /usr/local/include/ /node/usr/local \ + && cp -rp /usr/local/bin/ /node/usr/local diff --git a/images/templates/common/pipeline.Dockerfile b/images/templates/common/pipeline.Dockerfile index 46d51b938..fc11712ce 100644 --- a/images/templates/common/pipeline.Dockerfile +++ b/images/templates/common/pipeline.Dockerfile @@ -1,4 +1,4 @@ -FROM application-before-stamp as pipeline-basic +FROM application-basic as pipeline-basic LABEL "spryker.image" "none" ENV DEVELOPMENT_CONSOLE_COMMANDS=1 @@ -9,12 +9,7 @@ RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ git \ python3 \ jq \ - ; fi' - -# Debian contains outdated Yarn package -RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ - --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ - bash -c 'if [ ! -z "$(which apt)" ]; then \ + ; \ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ apt update -y && apt install -y \ @@ -32,10 +27,6 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc ; fi' # NodeJS + NPM -COPY --from=node-distributive /usr/lib /usr/lib -COPY --from=node-distributive /usr/local/share /usr/local/share -COPY --from=node-distributive /usr/local/lib /usr/local/lib -COPY --from=node-distributive /usr/local/include /usr/local/include -COPY --from=node-distributive /usr/local/bin /usr/local/bin +COPY --link --from=node-distributive /node/usr /usr/ -USER spryker +USER spryker:spryker diff --git a/images/templates/export/jenkins.Dockerfile b/images/templates/export/jenkins.Dockerfile index 63ba0600d..6cc4d5737 100644 --- a/images/templates/export/jenkins.Dockerfile +++ b/images/templates/export/jenkins.Dockerfile @@ -4,12 +4,16 @@ LABEL "spryker.image" "none" FROM application-before-stamp as jenkins LABEL "spryker.image" "jenkins" -EXPOSE 8080 COPY ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.docker.xml.twig ./config/Zed/cronjobs/jenkins.docker.xml.twig -COPY --from=jenkins-boilerplate /usr/share/jenkins/ref/plugins /usr/share/jenkins/ref/plugins -COPY --from=jenkins-boilerplate /usr/share/jenkins/jenkins.war /usr/share/jenkins/jenkins.war -COPY --from=jenkins-boilerplate /usr/share/jenkins/jenkins-cli.jar /usr/share/jenkins/jenkins-cli.jar +COPY --link --from=jenkins-boilerplate /usr/share/jenkins/ref/plugins /usr/share/jenkins/ref/plugins +COPY --link --from=jenkins-boilerplate /usr/share/jenkins/jenkins.war /usr/share/jenkins/jenkins.war +COPY --link --from=jenkins-boilerplate /usr/share/jenkins/jenkins-cli.jar /usr/share/jenkins/jenkins-cli.jar + +COPY --link ${DEPLOYMENT_PATH}/terraform/cli /envs/ +COPY --link --chmod=755 ${DEPLOYMENT_PATH}/context/jenkins/export/entrypoint.sh /entrypoint.sh +COPY --link ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.model.JenkinsLocationConfiguration.xml /opt/jenkins.model.JenkinsLocationConfiguration.xml +COPY --link ${DEPLOYMENT_PATH}/context/jenkins/export/nr-credentials.xml /opt/nr-credentials.xml # Install packages on Alpine RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ @@ -41,11 +45,7 @@ RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ mkdir -p /envs \ ; fi' -COPY ${DEPLOYMENT_PATH}/terraform/cli /envs/ -COPY ${DEPLOYMENT_PATH}/context/jenkins/export/entrypoint.sh /entrypoint.sh -COPY ${DEPLOYMENT_PATH}/context/jenkins/export/jenkins.model.JenkinsLocationConfiguration.xml /opt/jenkins.model.JenkinsLocationConfiguration.xml -COPY ${DEPLOYMENT_PATH}/context/jenkins/export/nr-credentials.xml /opt/nr-credentials.xml -RUN chmod +x /entrypoint.sh +EXPOSE 8080 ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/templates/mount/application.Dockerfile b/images/templates/mount/application.Dockerfile index fc017c7d1..bfd1fe2f6 100644 --- a/images/templates/mount/application.Dockerfile +++ b/images/templates/mount/application.Dockerfile @@ -1,21 +1,11 @@ FROM application-basic as application-before-stamp LABEL "spryker.image" "none" -ARG APPLICATION_ENV -ENV APPLICATION_ENV=${APPLICATION_ENV} -ARG SPRYKER_DB_ENGINE -ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} -ARG DEPLOYMENT_PATH -ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE} -ARG SPRYKER_PIPELINE -ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} - -ENV PATH=${srcRoot}/vendor/bin:$PATH - ARG USER_UID RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true; -COPY ${DEPLOYMENT_PATH}/context/php/conf.d/91-opcache-dev.ini /usr/local/etc/php/conf.d +ARG DEPLOYMENT_PATH +COPY --link ${DEPLOYMENT_PATH}/context/php/conf.d/91-opcache-dev.ini /usr/local/etc/php/conf.d CMD [ "php-fpm", "--nodaemonize" ] EXPOSE 9000 diff --git a/images/templates/mount/frontend.Dockerfile b/images/templates/mount/frontend.Dockerfile index 7fd1f802b..d754298fd 100644 --- a/images/templates/mount/frontend.Dockerfile +++ b/images/templates/mount/frontend.Dockerfile @@ -8,6 +8,7 @@ ARG SPRYKER_BUILD_HASH ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} ARG SPRYKER_BUILD_STAMP ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} + RUN mkdir -p /usr/share/nginx/ \ && envsubst '${SPRYKER_BUILD_HASH} ${SPRYKER_BUILD_STAMP}' < /tmp/build.json > /usr/share/nginx/build.json \ && rm -f /tmp/build.json diff --git a/images/templates/mount/pipeline.Dockerfile b/images/templates/mount/pipeline.Dockerfile index d27f17ecf..53b2e9a58 100644 --- a/images/templates/mount/pipeline.Dockerfile +++ b/images/templates/mount/pipeline.Dockerfile @@ -1,7 +1,7 @@ FROM pipeline-basic as pipeline-before-stamp LABEL "spryker.image" "none" -USER spryker +USER spryker:spryker FROM pipeline-before-stamp as pipeline LABEL "spryker.image" "pipeline" From f833e16a9488a45a31904912efff6cfc44ef909e Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Sun, 2 Jul 2023 04:13:52 +0300 Subject: [PATCH 13/26] Speeding up dev composer run --- images/templates/baked/application.Dockerfile | 5 ++--- images/templates/baked/pipeline.Dockerfile | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 7ab3144d8..1e34ae6c2 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -51,8 +51,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && rm -rf vendor/composer \ - && rm -rf vendor/bin \ + && rm -rf vendor/** \ && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} # Dependency: rsync is needed for next steps @@ -72,7 +71,7 @@ COPY --from=application-codebase --chown=spryker:spryker ${srcRoot}/composer.* $ RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ - LD_LIBRARY_PATH=/rsync time /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ --exclude 'tests/' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' --exclude '*.md' \ --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index b2ada5095..ac5599c00 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -10,7 +10,6 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ --mount=type=cache,id=vendor-dev,target=/data/vendor,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && rm -rf vendor/composer \ && composer install --no-scripts --no-interaction # ----------------------------- @@ -28,10 +27,8 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && LD_LIBRARY_PATH=/rsync time /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ \ - && time composer install --no-interaction \ - && find ./vendor -type d -name ".git*" -prune -exec rm -rf {} \; -# ^ Running install again to run all scripts + && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} COPY --chown=spryker:spryker data ${srcRoot}/data From a418643b80549909730220205ed13783d050a174 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Sun, 2 Jul 2023 04:34:15 +0300 Subject: [PATCH 14/26] Fixed assets build process that does not build --- images/templates/baked/assets.builder.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 8d02d7695..9389c333b 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -41,6 +41,7 @@ COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/d COPY --from=npm-precacher /tmp/.dependency* /tmp/ COPY --from=application-codebase /tmp/.dependency* /tmp/ +ARG SPRYKER_ASSETS_MODE RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 \ --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules,uid=1000 \ --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ From 728d8dd39414b24dd9cf0a37d6b49f0faaceeafd Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Sun, 2 Jul 2023 20:38:32 +0300 Subject: [PATCH 15/26] Excluding tests/dd.php to be skipped for web app images --- images/templates/baked/application.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 1e34ae6c2..4fe479ad3 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -72,8 +72,8 @@ RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ - --exclude 'tests/' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' --exclude '*.md' \ - --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' + --include 'tests/dd.php' --exclude 'tests/*' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' \ + --exclude '*.md' --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/data From c8dcc4be7d233d0b5c0ade4b44c1df191b6b54be Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 12:36:21 +0300 Subject: [PATCH 16/26] Reduced number of layers in web service images --- images/templates/baked/application.Dockerfile | 22 ++++++++----------- .../templates/baked/assets.builder.Dockerfile | 4 +--- images/templates/baked/pipeline.Dockerfile | 1 + 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 4fe479ad3..494d585fa 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -1,10 +1,3 @@ -FROM busybox AS stash-data -LABEL "spryker.image" "none" - -COPY --chown=spryker:spryker data /data-without-import - -RUN rm -rf /data-without-import/import - # ----------------------------- FROM busybox AS stash-src @@ -14,10 +7,18 @@ COPY --chown=spryker:spryker data /data-without-import RUN mkdir /data COPY src /data/src +COPY public /data/public COPY config /data/config COPY resource[s] /data/resources COPY LICENSE /data +FROM stash-src AS stash-src-with-data-excluding-import +LABEL "spryker.image" "none" + +COPY data /data/data + +RUN rm -rf /data/data/import + # ----------------------------- FROM ${SPRYKER_PLATFORM_IMAGE} AS stash-rsync @@ -75,20 +76,15 @@ RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ --include 'tests/dd.php' --exclude 'tests/*' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' \ --exclude '*.md' --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' -COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} -COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/data +COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data ${srcRoot} ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=tmpfs,target=/var/run/opcache/ \ bash -c 'chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true' \ - && ls -al \ && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production \ && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} -COPY --chown=spryker:spryker public ${srcRoot}/public - USER root -# RUN chown -R spryker:spryker /home/spryker CMD [ "php-fpm", "--nodaemonize" ] EXPOSE 9000 diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 9389c333b..ff9d1668a 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -32,10 +32,8 @@ USER spryker:spryker COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .* *.* ${srcRoot}/ COPY --chown=spryker:spryker frontend* ${srcRoot}/frontend -COPY --chown=spryker:spryker public* ${srcRoot}/public COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn -COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} -COPY --from=stash-data --chown=spryker:spryker /data-without-import ${srcRoot}/data +COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data ${srcRoot} # This instruction is necessary to ouline dependency on precacher to make sure assets are built after COPY --from=npm-precacher /tmp/.dependency* /tmp/ diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index ac5599c00..06d48bb3b 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -31,6 +31,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} +# Data with import COPY --chown=spryker:spryker data ${srcRoot}/data # Tests contain transfer declaration COPY --chown=spryker:spryker test[s] /${srcRoot}/tests From ff820d6a8ac9e675b67753f148bc9c54d7f0e364 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 17:46:19 +0300 Subject: [PATCH 17/26] Fixed autoload issue with spryker/spryker --- images/templates/baked/application.Dockerfile | 6 ++++-- images/templates/baked/pipeline.Dockerfile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 494d585fa..8a212c291 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -73,8 +73,10 @@ RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ - --include 'tests/dd.php' --exclude 'tests/*' --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' \ - --exclude '*.md' --exclude 'composer.json' --exclude 'composer.lock' --exclude 'codeception.yml' --exclude '.scrutinizer.yml' + --include 'tests/dd.php' --exclude 'tests/*' --exclude 'codeception.yml' \ + --exclude '*.md --exclude 'composer.lock' --exclude '.scrutinizer.yml' \ + --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' \ + --exclude 'package.json' --exclude 'package-lock.json' COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data ${srcRoot} diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 06d48bb3b..877c7250d 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -27,7 +27,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ \ && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} From 1ec23b458687dc66f521ade277ce24dc2b767bec Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 18:11:40 +0300 Subject: [PATCH 18/26] Reduced pipeline image by excluding git folders in vendor, but keeping them in cli image --- images/templates/baked/Dockerfile.twig | 4 ++++ images/templates/baked/application.Dockerfile | 2 +- images/templates/baked/cli.Dockerfile | 14 ++++++++++++++ images/templates/baked/pipeline.Dockerfile | 2 +- images/templates/common/cli.Dockerfile | 9 ++------- images/templates/mount/Dockerfile.twig | 4 ++++ images/templates/mount/cli.Dockerfile | 9 +++++++++ 7 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 images/templates/baked/cli.Dockerfile create mode 100644 images/templates/mount/cli.Dockerfile diff --git a/images/templates/baked/Dockerfile.twig b/images/templates/baked/Dockerfile.twig index d92e9291d..9bd848e0c 100644 --- a/images/templates/baked/Dockerfile.twig +++ b/images/templates/baked/Dockerfile.twig @@ -46,6 +46,10 @@ # ----------------------------- {% include "images/templates/common/cli.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/baked/cli.Dockerfile" with _context %} {%if _context['docker']['debug']['enabled'] %} {% include "images/templates/common/debug/cli.Dockerfile" with _context %} {%endif%} diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 8a212c291..3c5263b8e 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -74,7 +74,7 @@ RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ --include 'tests/dd.php' --exclude 'tests/*' --exclude 'codeception.yml' \ - --exclude '*.md --exclude 'composer.lock' --exclude '.scrutinizer.yml' \ + --exclude '*.md' --exclude 'composer.lock' --exclude '.scrutinizer.yml' \ --exclude 'assets/' --exclude '*.ts' --exclude '*.scss' --exclude '*.js' \ --exclude 'package.json' --exclude 'package-lock.json' diff --git a/images/templates/baked/cli.Dockerfile b/images/templates/baked/cli.Dockerfile new file mode 100644 index 000000000..55ea8887d --- /dev/null +++ b/images/templates/baked/cli.Dockerfile @@ -0,0 +1,14 @@ +FROM cli-basic as cli +LABEL "spryker.image" "cli" + +USER spryker:spryker + +# Copying .git folders that was skipped in pipeline +RUN --mount=type=cache,id=vendor-dev,target=/vendor,uid=1000 \ + --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --include '.git*/' --exclude '*' + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 877c7250d..06d48bb3b 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -27,7 +27,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ --mount=type=tmpfs,target=/var/run/opcache/ \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ \ + && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} diff --git a/images/templates/common/cli.Dockerfile b/images/templates/common/cli.Dockerfile index edbf2e40f..2cbb7d93c 100644 --- a/images/templates/common/cli.Dockerfile +++ b/images/templates/common/cli.Dockerfile @@ -10,8 +10,8 @@ RUN mkdir -p /tmp/blackfire \ && mv /tmp/blackfire/blackfire /usr/bin/blackfire \ && rm -Rf /tmp/blackfire -FROM pipeline-before-stamp as cli -LABEL "spryker.image" "cli" +FROM pipeline-before-stamp as cli-basic +LABEL "spryker.image" "none" USER root @@ -46,8 +46,3 @@ RUN mkdir -p /home/spryker/env \ && mkdir -p /home/spryker/ssh-relay/ && chmod 777 /home/spryker/ssh-relay && touch /home/spryker/ssh-relay/ssh-auth.sock && chmod 666 /home/spryker/ssh-relay/ssh-auth.sock \ && touch /tmp/stdout && touch /tmp/stderr && chmod 666 /tmp/stdout && chmod 666 /tmp/stderr \ && mkdir -p /home/spryker/history && touch /home/spryker/history/.bash_history && chmod 0600 /home/spryker/history/.bash_history - -ARG SPRYKER_BUILD_HASH -ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} -ARG SPRYKER_BUILD_STAMP -ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} diff --git a/images/templates/mount/Dockerfile.twig b/images/templates/mount/Dockerfile.twig index bf8000022..7634b649d 100644 --- a/images/templates/mount/Dockerfile.twig +++ b/images/templates/mount/Dockerfile.twig @@ -38,6 +38,10 @@ # ----------------------------- {% include "images/templates/common/cli.Dockerfile" with _context %} + +# ----------------------------- + +{% include "images/templates/mount/cli.Dockerfile" with _context %} {%if _context['docker']['debug']['enabled'] %} {% include "images/templates/common/debug/cli.Dockerfile" with _context %} {%endif%} diff --git a/images/templates/mount/cli.Dockerfile b/images/templates/mount/cli.Dockerfile new file mode 100644 index 000000000..c0b0a46a0 --- /dev/null +++ b/images/templates/mount/cli.Dockerfile @@ -0,0 +1,9 @@ +FROM cli-basic as cli +LABEL "spryker.image" "cli" + +USER spryker:spryker + +ARG SPRYKER_BUILD_HASH +ENV SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH} +ARG SPRYKER_BUILD_STAMP +ENV SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP} From 5e7cb8557167d9261bffcedd3148a5751e8d88bb Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 20:37:49 +0300 Subject: [PATCH 19/26] Fixed resolver cache for cli/pipeline --- images/templates/baked/pipeline.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 06d48bb3b..74987b056 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -40,7 +40,7 @@ ENV DEVELOPMENT_CONSOLE_COMMANDS=1 ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=tmpfs,target=/var/run/opcache/ \ - vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development \ + vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development -s build-production \ && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} COPY --link --chown=spryker:spryker fronten[d] ${srcRoot}/frontend From 293a8fde8d1a594fd96e72acd024c2a254ad50ce Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 21:51:18 +0300 Subject: [PATCH 20/26] Fixed cli and scheduler does not work properly --- images/templates/baked/assets.builder.Dockerfile | 2 +- images/templates/common/cli.Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index ff9d1668a..c07a35ef9 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -30,7 +30,7 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc USER spryker:spryker -COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .* *.* ${srcRoot}/ +COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .es*.* .npm* ${srcRoot}/ COPY --chown=spryker:spryker frontend* ${srcRoot}/frontend COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data ${srcRoot} diff --git a/images/templates/common/cli.Dockerfile b/images/templates/common/cli.Dockerfile index 2cbb7d93c..7962e6c21 100644 --- a/images/templates/common/cli.Dockerfile +++ b/images/templates/common/cli.Dockerfile @@ -36,6 +36,7 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc USER spryker:spryker +ENV PATH=/home/spryker/bin:$PATH ENV HISTFILE=/home/spryker/history/.bash_history ENV NEWRELIC_ENABLED=0 From a91406dd56ed5d83485bf1264cbbc825a08a4c65 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 22:20:18 +0300 Subject: [PATCH 21/26] Fixed permission issue in cli --- images/templates/baked/pipeline.Dockerfile | 3 ++- images/templates/mount/cli.Dockerfile | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 74987b056..579af0f95 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -40,7 +40,8 @@ ENV DEVELOPMENT_CONSOLE_COMMANDS=1 ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=tmpfs,target=/var/run/opcache/ \ - vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development -s build-production \ + bash -c 'chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true' \ + && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development -s build-production \ && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} COPY --link --chown=spryker:spryker fronten[d] ${srcRoot}/frontend diff --git a/images/templates/mount/cli.Dockerfile b/images/templates/mount/cli.Dockerfile index c0b0a46a0..0fe237d52 100644 --- a/images/templates/mount/cli.Dockerfile +++ b/images/templates/mount/cli.Dockerfile @@ -1,6 +1,14 @@ FROM cli-basic as cli LABEL "spryker.image" "cli" +USER root + +ARG USER_UID +RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true; + +ARG DEPLOYMENT_PATH +COPY --link ${DEPLOYMENT_PATH}/context/php/conf.d/91-opcache-dev.ini /usr/local/etc/php/conf.d + USER spryker:spryker ARG SPRYKER_BUILD_HASH From d321fb024515430681e49cab89bfeca974d6074a Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 22:40:03 +0300 Subject: [PATCH 22/26] Optimized pipeline build speed by reusing application build results --- images/templates/baked/application.Dockerfile | 4 ---- images/templates/baked/assets.builder.Dockerfile | 2 +- images/templates/baked/pipeline.Dockerfile | 10 ++++++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 3c5263b8e..54c5a768c 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -2,10 +2,6 @@ FROM busybox AS stash-src -COPY --chown=spryker:spryker data /data-without-import - -RUN mkdir /data - COPY src /data/src COPY public /data/public COPY config /data/config diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index c07a35ef9..58312bac3 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -45,4 +45,4 @@ RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ echo "MODE: ${SPRYKER_ASSETS_MODE}" \ && vendor/bin/console transfer:generate \ - && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} -vvv + && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index 579af0f95..c966427b8 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -1,3 +1,9 @@ +FROM stash-src AS stash-src-after-app + +COPY --from=application-before-stamp ${srcRoot}/src /data/src + +# ----------------------------- + FROM application-codebase AS application-codebase-dev LABEL "spryker.image" "none" @@ -30,7 +36,7 @@ RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' -COPY --from=stash-src --chown=spryker:spryker /data ${srcRoot} +COPY --from=stash-src-after-app --chown=spryker:spryker /data ${srcRoot} # Data with import COPY --chown=spryker:spryker data ${srcRoot}/data # Tests contain transfer declaration @@ -41,7 +47,7 @@ ENV DEVELOPMENT_CONSOLE_COMMANDS=1 ARG SPRYKER_COMPOSER_AUTOLOAD RUN --mount=type=tmpfs,target=/var/run/opcache/ \ bash -c 'chmod 600 ${srcRoot}/config/Zed/*.key 2>/dev/null || true' \ - && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development -s build-production \ + && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development \ && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} COPY --link --chown=spryker:spryker fronten[d] ${srcRoot}/frontend From 2f22727fe9782bc322702ddcc0800e33be9af85f Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Mon, 3 Jul 2023 22:45:19 +0300 Subject: [PATCH 23/26] Optimized pipeline build speed by reusing application build results [Part 2] --- images/templates/baked/pipeline.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index c966427b8..b6759dabc 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -1,6 +1,6 @@ FROM stash-src AS stash-src-after-app -COPY --from=application-before-stamp ${srcRoot}/src /data/src +COPY --from=application-before-stamp /data/src /data/src # ----------------------------- From 8d7a792b43ec6f95b8d3245ae8b71b137344f2a2 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Tue, 4 Jul 2023 13:32:35 +0300 Subject: [PATCH 24/26] Fixed lack of files in root folder for FE build in suite-nonsplit (angular) --- images/templates/baked/assets.builder.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 58312bac3..dfebb5b46 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -30,7 +30,8 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc USER spryker:spryker -COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .es*.* .npm* ${srcRoot}/ +# TODO exclude test-related and deploy.yml files from the scope. HOW? +COPY --chown=spryker:spryker package.json* package-lock.json* tsconfig*.json .* *.* ${srcRoot}/ COPY --chown=spryker:spryker frontend* ${srcRoot}/frontend COPY --chown=spryker:spryker .yarn* ${srcRoot}/.yarn COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data ${srcRoot} From dd734cede2283fc51ffc60e333ab99563038a872 Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Tue, 4 Jul 2023 15:04:12 +0000 Subject: [PATCH 25/26] Improved docker build structure and hopefully speed be using --mount=type=bind --- images/templates/baked/application.Dockerfile | 22 ++++++------------- .../templates/baked/assets.builder.Dockerfile | 8 ++++--- images/templates/baked/cli.Dockerfile | 4 ++-- images/templates/baked/pipeline.Dockerfile | 15 ++++--------- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 54c5a768c..97caabae7 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -6,7 +6,7 @@ COPY src /data/src COPY public /data/public COPY config /data/config COPY resource[s] /data/resources -COPY LICENSE /data +COPY composer.json composer.lock *.php LICENSE /data FROM stash-src AS stash-src-with-data-excluding-import LABEL "spryker.image" "none" @@ -31,8 +31,8 @@ RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc rsync \ ; fi' -RUN --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ - cp -fp /usr/bin/rsync /rsync/ \ +RUN mkdir -p /rsync/ \ + && cp -fp /usr/bin/rsync /rsync/ \ && ldd /usr/bin/rsync | awk '/=>/ { print $3 }' | xargs -I '{}' cp -fp '{}' /rsync # ----------------------------- @@ -46,13 +46,8 @@ COPY --chown=spryker:spryker composer.json composer.lock *.php ${srcRoot}/ ARG SPRYKER_COMPOSER_MODE RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ - --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ - set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && rm -rf vendor/** \ - && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} - -# Dependency: rsync is needed for next steps -COPY --from=stash-rsync /tmp/.dependency* /tmp/ + set -o allexport && . /run/secrets/secrets-env && set +o allexport && \ + composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} # ----------------------------- @@ -61,12 +56,9 @@ LABEL "spryker.image" "none" USER spryker:spryker -# Dependency: Run ONLY after vendor folder is -COPY --from=application-codebase --chown=spryker:spryker ${srcRoot}/composer.* ${srcRoot}/*.php ${srcRoot}/ - # Install composer modules for Spryker -RUN --mount=type=cache,id=vendor,target=/vendor,uid=1000 \ - --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ +RUN --mount=type=bind,from=application-codebase,source=/data/vendor,target=/vendor \ + --mount=type=bind,from=stash-rsync,source=/rsync,target=/rsync \ --mount=type=tmpfs,target=/var/run/opcache/ \ LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ --include 'tests/dd.php' --exclude 'tests/*' --exclude 'codeception.yml' \ diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index dfebb5b46..9cc9478b1 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -38,12 +38,14 @@ COPY --from=stash-src-with-data-excluding-import --chown=spryker:spryker /data $ # This instruction is necessary to ouline dependency on precacher to make sure assets are built after COPY --from=npm-precacher /tmp/.dependency* /tmp/ -COPY --from=application-codebase /tmp/.dependency* /tmp/ ARG SPRYKER_ASSETS_MODE RUN --mount=type=cache,id=npm,sharing=locked,target=/home/spryker/.npm,uid=1000 \ --mount=type=cache,id=node_modules,sharing=locked,target=${srcRoot}/node_modules,uid=1000 \ - --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ - echo "MODE: ${SPRYKER_ASSETS_MODE}" \ + --mount=type=bind,from=application-codebase,source=/data/vendor,target=/vendor \ + --mount=type=bind,from=stash-rsync,source=/rsync,target=/rsync \ + --mount=type=tmpfs,target=/var/run/opcache/ \ + LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + && echo "MODE: ${SPRYKER_ASSETS_MODE}" \ && vendor/bin/console transfer:generate \ && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build-static -s build-static-${SPRYKER_ASSETS_MODE} diff --git a/images/templates/baked/cli.Dockerfile b/images/templates/baked/cli.Dockerfile index 55ea8887d..410950676 100644 --- a/images/templates/baked/cli.Dockerfile +++ b/images/templates/baked/cli.Dockerfile @@ -4,8 +4,8 @@ LABEL "spryker.image" "cli" USER spryker:spryker # Copying .git folders that was skipped in pipeline -RUN --mount=type=cache,id=vendor-dev,target=/vendor,uid=1000 \ - --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ +RUN --mount=type=bind,from=application-codebase-dev,source=/data/vendor,target=/vendor \ + --mount=type=bind,from=stash-rsync,source=/rsync,target=/rsync \ LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --include '.git*/' --exclude '*' ARG SPRYKER_BUILD_HASH diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile index b6759dabc..f250a2166 100644 --- a/images/templates/baked/pipeline.Dockerfile +++ b/images/templates/baked/pipeline.Dockerfile @@ -7,14 +7,8 @@ COPY --from=application-before-stamp /data/src /data/src FROM application-codebase AS application-codebase-dev LABEL "spryker.image" "none" -RUN --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ - --mount=type=cache,id=vendor,target=/data/vendor,uid=1000 \ - --mount=type=cache,id=vendor-dev,target=/data/vendor.dev,uid=1000 \ - LD_LIBRARY_PATH=/rsync /rsync/rsync -ap ./vendor/ ./vendor.dev - RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ - --mount=type=cache,id=vendor-dev,target=/data/vendor,uid=1000 \ set -o allexport && . /run/secrets/secrets-env && set +o allexport \ && composer install --no-scripts --no-interaction @@ -25,15 +19,14 @@ LABEL "spryker.image" "none" USER spryker:spryker -COPY --from=application-codebase-dev --chown=spryker:spryker ${srcRoot}/composer.* ${srcRoot}/*.php ${srcRoot}/ # Install dev modules for Spryker RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ - --mount=type=cache,id=vendor-dev,target=/vendor,uid=1000 \ - --mount=type=cache,id=rsync,target=/rsync,uid=1000 \ + --mount=type=bind,from=application-codebase-dev,source=/data/vendor,target=/vendor \ + --mount=type=bind,from=stash-rsync,source=/rsync,target=/rsync \ --mount=type=tmpfs,target=/var/run/opcache/ \ - set -o allexport && . /run/secrets/secrets-env && set +o allexport \ - && LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ + set -o allexport && . /run/secrets/secrets-env && set +o allexport && \ + LD_LIBRARY_PATH=/rsync /rsync/rsync -ap --chown=spryker:spryker /vendor/ ./vendor/ --exclude '.git*/' \ && bash -c 'if composer run --list | grep post-install-cmd; then composer run post-install-cmd; fi' COPY --from=stash-src-after-app --chown=spryker:spryker /data ${srcRoot} From a60b3fdc3c428115976de166f714a8097e32506c Mon Sep 17 00:00:00 2001 From: Mike Kalinin Date: Thu, 3 Aug 2023 11:06:19 +0000 Subject: [PATCH 26/26] Refactored Dockerfiles to use newest build features Optimized pipeline and jenkins images by removing unnecessary binaries (e.g. PGSQL client if database is MYSQL) --- generator/Dockerfile | 2 +- images/services/dashboard/Dockerfile | 16 ++-- images/services/tideways/Dockerfile | 20 +++-- images/templates/baked/Dockerfile.twig | 18 ++--- images/templates/baked/application.Dockerfile | 68 +++++++++++------ .../templates/baked/assets.builder.Dockerfile | 58 +++++++------- images/templates/baked/cli.Dockerfile | 5 +- images/templates/baked/frontend.Dockerfile | 6 +- images/templates/baked/npm.Dockerfile | 9 ++- images/templates/baked/pipeline.Dockerfile | 29 ++++--- .../templates/common/application.Dockerfile | 26 ++++--- .../templates/common}/build.json | 0 images/templates/common/cli.Dockerfile | 76 ++++++++++++------- .../common/debug/application.Dockerfile | 34 +++++---- images/templates/common/frontend.Dockerfile | 2 - images/templates/common/node.Dockerfile | 17 +++-- images/templates/common/pipeline.Dockerfile | 51 +++++++------ images/templates/export/Dockerfile.twig | 11 ++- images/templates/export/jenkins.Dockerfile | 71 ++++++++--------- images/templates/mount/application.Dockerfile | 5 +- images/templates/mount/cli.Dockerfile | 5 +- images/templates/mount/frontend.Dockerfile | 6 +- images/templates/mount/pipeline.Dockerfile | 2 + 23 files changed, 315 insertions(+), 222 deletions(-) rename {context/nginx => images/templates/common}/build.json (100%) diff --git a/generator/Dockerfile b/generator/Dockerfile index a0c5f4805..c3390b68e 100644 --- a/generator/Dockerfile +++ b/generator/Dockerfile @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.5 -ARG SPRYKER_PHP_VERSION=7.3 +ARG SPRYKER_PHP_VERSION=8.2 FROM spryker/php:${SPRYKER_PHP_VERSION} diff --git a/images/services/dashboard/Dockerfile b/images/services/dashboard/Dockerfile index 95e0369b3..b20e416bd 100644 --- a/images/services/dashboard/Dockerfile +++ b/images/services/dashboard/Dockerfile @@ -1,20 +1,18 @@ FROM node:alpine AS dashboard LABEL "spryker.image" "dashboard" -RUN mkdir -p /dashboard WORKDIR /dashboard ENV HOME=/dashboard -RUN npm install log.io pm2 -g - COPY context/dashboard/package.json context/dashboard/package-lock.json /dashboard/ -RUN npm install -COPY context/dashboard/.log.io /dashboard/.log.io/ -COPY context/dashboard/assets /dashboard/assets/ -COPY context/dashboard/src /dashboard/src/ -COPY context/dashboard/views /dashboard/views/ -COPY context/dashboard/process.yml /dashboard/ +RUN < /etc/apt/sources.list.d/tideways.list && \ - wget -qO - 'https://packages.tideways.com/key.gpg' | apt-key add - -RUN DEBIAN_FRONTEND=noninteractive apt update -y && apt install -yq tideways-daemon && \ - apt autoremove --assume-yes && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN < /etc/apt/sources.list.d/tideways.list + wget -qO - 'https://packages.tideways.com/key.gpg' | apt-key add - + apt update -y + apt install -yq tideways-daemon + apt autoremove --assume-yes + apt clean + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +EOT ENTRYPOINT ["tideways-daemon","--hostname=tideways","--address=0.0.0.0:9135"] diff --git a/images/templates/baked/Dockerfile.twig b/images/templates/baked/Dockerfile.twig index 9bd848e0c..f0c5d6b49 100644 --- a/images/templates/baked/Dockerfile.twig +++ b/images/templates/baked/Dockerfile.twig @@ -14,30 +14,30 @@ # ----------------------------- -{% include "images/templates/common/frontend.Dockerfile" with _context %} +{% include "images/templates/baked/application.Dockerfile" with _context %} # ----------------------------- -{% include "images/templates/common/gateway.Dockerfile" with _context %} +{% include "images/templates/common/pipeline.Dockerfile" with _context %} # ----------------------------- -{% include "images/templates/baked/application.Dockerfile" with _context %} +{% include "images/templates/baked/assets.builder.Dockerfile" with _context %} # ----------------------------- -{% include "images/templates/common/application.dev.Dockerfile" with _context %} -{%if _context['docker']['debug']['enabled'] %} -{% include "images/templates/common/debug/application.Dockerfile" with _context %} -{%endif%} +{% include "images/templates/common/frontend.Dockerfile" with _context %} # ----------------------------- -{% include "images/templates/baked/assets.builder.Dockerfile" with _context %} +{% include "images/templates/common/gateway.Dockerfile" with _context %} # ----------------------------- -{% include "images/templates/common/pipeline.Dockerfile" with _context %} +{% include "images/templates/common/application.dev.Dockerfile" with _context %} +{%if _context['docker']['debug']['enabled'] %} +{% include "images/templates/common/debug/application.Dockerfile" with _context %} +{%endif%} # ----------------------------- diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile index 97caabae7..eb747727f 100644 --- a/images/templates/baked/application.Dockerfile +++ b/images/templates/baked/application.Dockerfile @@ -20,20 +20,30 @@ RUN rm -rf /data/data/import FROM ${SPRYKER_PLATFORM_IMAGE} AS stash-rsync LABEL "spryker.image" "none" -RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk \ + --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ - bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ - rsync \ - ; fi' - -RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ - bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ - rsync \ - ; fi' - -RUN mkdir -p /rsync/ \ - && cp -fp /usr/bin/rsync /rsync/ \ - && ldd /usr/bin/rsync | awk '/=>/ { print $3 }' | xargs -I '{}' cp -fp '{}' /rsync + </ { print $3 }' | xargs -I '{}' cp -fp '{}' /rsync +EOT # ----------------------------- @@ -46,8 +56,10 @@ COPY --chown=spryker:spryker composer.json composer.lock *.php ${srcRoot}/ ARG SPRYKER_COMPOSER_MODE RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \ --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \ - set -o allexport && . /run/secrets/secrets-env && set +o allexport && \ - composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} + </dev/null || true' \ - && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production \ - && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} + </dev/null || true + vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production + composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} +EOT USER root diff --git a/images/templates/baked/assets.builder.Dockerfile b/images/templates/baked/assets.builder.Dockerfile index 9cc9478b1..f28e2b8fa 100644 --- a/images/templates/baked/assets.builder.Dockerfile +++ b/images/templates/baked/assets.builder.Dockerfile @@ -1,32 +1,29 @@ -FROM application-basic AS assets-builder +FROM pipeline-basic AS assets-builder LABEL "spryker.image" "none" USER root -COPY --from=node-distributive /node/usr /usr/ - -RUN --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ - --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ - bash -c 'if [ ! -z "$(which apt)" ]; then apt update -y && apt install -y \ - python3 \ - g++ \ - make \ - ; \ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ - apt update -y && apt install -y \ - yarn \ - ; fi' - -RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk mkdir -p /etc/apk && ln -vsf /var/cache/apk /etc/apk/cache && \ - bash -c 'if [ ! -z "$(which apk)" ]; then apk update && apk add \ - coreutils \ - ncurses \ - yarn \ - python3 \ - g++ \ - make \ - ; fi' +RUN --mount=type=cache,id=apk,sharing=locked,target=/var/cache/apk \ + --mount=type=cache,id=aptlib,sharing=locked,target=/var/lib/apt \ + --mount=type=cache,id=aptcache,sharing=locked,target=/var/cache/apt \ + < /usr/share/nginx/build.json \ - && rm -f /tmp/build.json +COPY --link <<-EOT /usr/share/nginx/build.json +{% include "images/templates/common/build.json" with _context %} +EOT diff --git a/images/templates/baked/npm.Dockerfile b/images/templates/baked/npm.Dockerfile index 1b9a0c60e..468e151b7 100644 --- a/images/templates/baked/npm.Dockerfile +++ b/images/templates/baked/npm.Dockerfile @@ -1,9 +1,14 @@ FROM node-distributive AS npm-precacher LABEL "spryker.image" "none" +WORKDIR /root + COPY --chown=spryker:spryker package.jso[n] package-lock.jso[n] /root/ RUN --mount=type=cache,id=npm,sharing=locked,target=/root/.npm \ --mount=type=cache,id=npm-modules,sharing=locked,target=/root/node_modules \ - cd /root \ - && sh -c 'if [ -f ${srcRoot}/package.json ]; then npm install --prefer-offline || npm ci --prefer-offline || true; fi' + </dev/null || true' \ - && vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development \ - && composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} + </dev/null || true + vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-development + composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD} +EOT COPY --link --chown=spryker:spryker fronten[d] ${srcRoot}/frontend COPY --link --chown=spryker:spryker .yar[n] ${srcRoot}/.yarn -COPY --link --chown=spryker:spryker .* *.* LICENSE ${srcRoot} +COPY --link --chown=spryker:spryker .* *.* ${srcRoot} FROM pipeline-before-stamp as pipeline LABEL "spryker.image" "pipeline" diff --git a/images/templates/common/application.Dockerfile b/images/templates/common/application.Dockerfile index c43bcda35..bd9a59e71 100644 --- a/images/templates/common/application.Dockerfile +++ b/images/templates/common/application.Dockerfile @@ -3,6 +3,7 @@ LABEL "spryker.image" "none" ENV SPRYKER_IN_DOCKER=1 ENV COMPOSER_IGNORE_CHROMEDRIVER=1 +ENV SPRYKER_JENKINS_TEMPLATE_PATH=/home/spryker/jenkins.docker.xml.twig {% for envName, envValue in _envs %} ENV {{ envName }}='{{ envValue }}' {% endfor %} @@ -29,20 +30,25 @@ COPY --link --chown=spryker:spryker ${DEPLOYMENT_PATH}/context/php/build.php /ho ARG SPRYKER_LOG_DIRECTORY ARG KNOWN_HOSTS ENV SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY} -RUN mkdir -p ${SPRYKER_LOG_DIRECTORY} \ - && chown spryker:spryker ${SPRYKER_LOG_DIRECTORY} \ - && mkdir -p /home/spryker/.ssh && chmod 0700 /home/spryker/.ssh \ - && bash -c '[ ! -z "${KNOWN_HOSTS}" ] && ssh-keyscan -t rsa ${KNOWN_HOSTS} >> /home/spryker/.ssh/known_hosts || true' \ - && chown spryker:spryker -R /home/spryker/.ssh \ - && rm -f /usr/local/etc/php/conf.d/opcache.ini \ +RUN <> /home/spryker/.ssh/known_hosts + fi + chown spryker:spryker -R /home/spryker/.ssh + rm -f /usr/local/etc/php/conf.d/opcache.ini {% if _phpExtensions is defined and _phpExtensions is not empty %} {% for phpExtention in _phpExtensions %} - && mv /usr/local/etc/php/disabled/{{phpExtention}}.ini /usr/local/etc/php/conf.d/90-{{phpExtention}}.ini \ + mv /usr/local/etc/php/disabled/{{phpExtention}}.ini /usr/local/etc/php/conf.d/90-{{phpExtention}}.ini {% endfor %} {% endif %} - && rm -rf /var/run \ - && /usr/bin/install -d -m 777 /var/run/opcache \ - && bash -c "php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/php-fpm.d/worker.conf || true" + rm -rf /var/run + /usr/bin/install -d -m 777 /var/run/opcache + php -r 'exit(PHP_VERSION_ID > 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/php-fpm.d/worker.conf || true +EOT ARG SPRYKER_PIPELINE ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE} diff --git a/context/nginx/build.json b/images/templates/common/build.json similarity index 100% rename from context/nginx/build.json rename to images/templates/common/build.json diff --git a/images/templates/common/cli.Dockerfile b/images/templates/common/cli.Dockerfile index 7962e6c21..699f817a4 100644 --- a/images/templates/common/cli.Dockerfile +++ b/images/templates/common/cli.Dockerfile @@ -4,35 +4,44 @@ LABEL "spryker.image" "none" USER root # Blackfire client -RUN mkdir -p /tmp/blackfire \ - && architecture=$(case $(uname -m) in i386 | i686 | x86) echo "i386" ;; x86_64 | amd64) echo "amd64" ;; aarch64 | arm64 | armv8) echo "arm64" ;; *) echo "amd64" ;; esac) \ - && curl -A "Docker" -L https://blackfire.io/api/v1/releases/cli/linux/$architecture | tar zxp -C /tmp/blackfire \ - && mv /tmp/blackfire/blackfire /usr/bin/blackfire \ - && rm -Rf /tmp/blackfire +RUN < 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/debug.php-fpm.conf/worker.conf || true" +RUN < 70400 ? 1 : 0);' && sed -i '' -e 's/decorate_workers_output/;decorate_workers_output/g' /usr/local/etc/debug.php-fpm.conf/worker.conf || true +EOT CMD [ "/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ] EXPOSE 9000 9001 diff --git a/images/templates/common/frontend.Dockerfile b/images/templates/common/frontend.Dockerfile index 3ed852b6b..1745a2978 100644 --- a/images/templates/common/frontend.Dockerfile +++ b/images/templates/common/frontend.Dockerfile @@ -15,8 +15,6 @@ ENV SPRYKER_DNS_RESOLVER_FLAGS="valid=10s ipv6=off" ENV SPRYKER_DNS_RESOLVER_IP="" ENV SPRYKER_MAINTENANCE_MODE_ENABLED="0" -COPY --chown=root:root --link ${DEPLOYMENT_PATH}/context/nginx/build.json /tmp/build.json - ENTRYPOINT [ "/entrypoint.sh" ] CMD ["nginx", "-g", "daemon off;"] diff --git a/images/templates/common/node.Dockerfile b/images/templates/common/node.Dockerfile index 397c0b2f9..659e2f927 100644 --- a/images/templates/common/node.Dockerfile +++ b/images/templates/common/node.Dockerfile @@ -4,10 +4,13 @@ LABEL "spryker.image" "none" ARG SPRYKER_NPM_VERSION RUN npm install -g npm@${SPRYKER_NPM_VERSION} -# TODO get the difference what need to be copied only to reduce the layer size -RUN mkdir -p /node/usr/lib && mkdir -p /node/usr/local \ - && cp -rp /usr/lib/ /node/usr \ - && cp -rp /usr/local/share/ /node/usr/local \ - && cp -rp /usr/local/lib/ /node/usr/local \ - && cp -rp /usr/local/include/ /node/usr/local \ - && cp -rp /usr/local/bin/ /node/usr/local +RUN < /usr/share/nginx/build.json \ - && rm -f /tmp/build.json +COPY --link <<-EOT /usr/share/nginx/build.json +{% include "images/templates/common/build.json" with _context %} +EOT diff --git a/images/templates/mount/pipeline.Dockerfile b/images/templates/mount/pipeline.Dockerfile index 53b2e9a58..f2ee951a5 100644 --- a/images/templates/mount/pipeline.Dockerfile +++ b/images/templates/mount/pipeline.Dockerfile @@ -3,6 +3,8 @@ LABEL "spryker.image" "none" USER spryker:spryker +ENV DEVELOPMENT_CONSOLE_COMMANDS=1 + FROM pipeline-before-stamp as pipeline LABEL "spryker.image" "pipeline"