Skip to content

Commit

Permalink
fix: update logic for h2o-ecr
Browse files Browse the repository at this point in the history
  • Loading branch information
@SandevDewthilina
SandevDewthilina committed Mar 6, 2025
1 parent 65692a2 commit 5106c9c
Showing 1 changed file with 52 additions and 20 deletions.
72 changes: 52 additions & 20 deletions docker/Jenkinsfile-build-hadoop-docker
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ final String DOCKER_STASH = 'h2o-3-hadoop-docker-stash'
final String REGISTRY_PREFIX = "${params.dockerRegistry}"
final String IMAGE_NAME_PREFIX = "opsh2oai/h2o-3-hadoop"
final List INTERNAL_CLUSTER_HADOOPS = [
[ distribution: "hdp-2.2", krb: false ], // 0xd6 cluster
[ distribution: "hdp-2.4", krb: true ], // 0xg5 cluster
// [ distribution: "hdp-2.2", krb: false ], // 0xd6 cluster
// [ distribution: "hdp-2.4", krb: true ], // 0xg5 cluster
[ distribution: "cdh-6.3", krb: true ] // 0xg9 cluster
]

Expand All @@ -19,7 +19,7 @@ properties(
parameters(
[
string(defaultValue: 'master', description: 'Branch to checkout', name: 'gitBranch'),
string(name: 'dockerRegistry', defaultValue: 'harbor.h2o.ai'),
string(name: 'dockerRegistry', defaultValue: '353750902984.dkr.ecr.us-east-1.amazonaws.com'),
booleanParam(name: 'force', defaultValue: false, description: 'If false and image with version specified by BuildConfig exists in repository, then the build fails.'),
booleanParam(name: 'noCache', defaultValue: true, description: 'If set to true, the docker image is built from scratch, with no stages used from the cache.')
]
Expand Down Expand Up @@ -79,11 +79,11 @@ parallel(pipelineContext.getBuildConfig().getSupportedHadoopDistributions().coll
cleanWs()

final String imageName = "${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}"
final boolean conflict = pipelineContext.getUtils()
.dockerImageExistsInRegistry(this, params.dockerRegistry, imageName, version)
if (conflict && !params.force) {
error "Tag ${imageName}:${version} already exists in the repository"
}
// final boolean conflict = pipelineContext.getUtils()
// .dockerImageExistsInRegistry(this, params.dockerRegistry, imageName, version)
// if (conflict && !params.force) {
// error "Tag ${imageName}:${version} already exists in the repository"
// }
def repoVersion = "${distribution.version}.0"
if (distribution.name == "cdh" && distribution.version == "6.3") {
repoVersion = "6.3.2" // 6.3.0 is not available in cloudera repos
Expand Down Expand Up @@ -139,14 +139,20 @@ parallel(pipelineContext.getBuildConfig().getSupportedHadoopDistributions().coll
pipelineContext.getBuildSummary().addStageSummary(this, publishStageName, '')
pipelineContext.getBuildSummary().setStageDetails(this, publishStageName, env.NODE_NAME, env.WORKSPACE)

withCredentials([usernamePassword(credentialsId: "${params.dockerRegistry}", usernameVariable: 'REGISTRY_USERNAME', passwordVariable: 'REGISTRY_PASSWORD')]) {
sh """
docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD ${params.dockerRegistry}
docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}:${version}
docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}-krb:${version}
"""
// withCredentials([usernamePassword(credentialsId: "${params.dockerRegistry}", usernameVariable: 'REGISTRY_USERNAME', passwordVariable: 'REGISTRY_PASSWORD')]) {
// sh """
// docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD ${params.dockerRegistry}
// docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}:${version}
// docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}-krb:${version}
// """
// echo "###### Docker image ${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}:${version} built and pushed. ######"
// }
dockerLoginH2OECR()
sh """
docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}:${version}
docker push ${REGISTRY_PREFIX}/${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}-krb:${version}
echo "###### Docker image ${IMAGE_NAME_PREFIX}-${distribution.name}-${distribution.version}:${version} built and pushed. ######"
}
"""
pipelineContext.getBuildSummary().markStageSuccessful(this, publishStageName)
} catch (Exception e) {
pipelineContext.getBuildSummary().markStageFailed(this, publishStageName)
Expand Down Expand Up @@ -174,11 +180,11 @@ parallel(INTERNAL_CLUSTER_HADOOPS.collectEntries { hdp ->
pipelineContext.getBuildSummary().setStageDetails(this, buildStageName, env.NODE_NAME, env.WORKSPACE)
cleanWs()

final boolean conflict = pipelineContext.getUtils()
.dockerImageExistsInRegistry(this, params.dockerRegistry, imageName, version)
if (conflict && !params.force) {
error "Tag ${imageName}:${version} already exists in the repository"
}
// final boolean conflict = pipelineContext.getUtils()
// .dockerImageExistsInRegistry(this, params.dockerRegistry, imageName, version)
// if (conflict && !params.force) {
// error "Tag ${imageName}:${version} already exists in the repository"
// }

dir("${imageName}.${version}") {
pipelineContext.getUtils().unstashFiles(this, DOCKER_STASH)
Expand Down Expand Up @@ -215,6 +221,12 @@ parallel(INTERNAL_CLUSTER_HADOOPS.collectEntries { hdp ->
"""
echo "###### Docker image ${imageName}:${version} built and pushed. ######"
}

dockerLoginH2OECR()
sh """
docker push ${REGISTRY_PREFIX}/${imageName}:${version}
echo "###### Docker image ${imageName}:${version} built and pushed. ######"
"""
pipelineContext.getBuildSummary().markStageSuccessful(this, publishStageName)
} catch (Exception e) {
pipelineContext.getBuildSummary().markStageFailed(this, publishStageName)
Expand All @@ -225,3 +237,23 @@ parallel(INTERNAL_CLUSTER_HADOOPS.collectEntries { hdp ->
}
]
})

def imageExistsH2OECR(String repository, String tag) {
def result = "-1"
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: "H2O-AWS_CT-JENKINS-H2O-ECR"]]) {
result = sh(
script: "aws ecr batch-get-image --registry-id=353750902984 --repository-name='${repository}' --image-ids=imageTag='${tag}' --query 'images[].imageId.imageTag' --output text",
returnStdout: true
).trim()
}
return result == tag
}

def dockerLoginH2OECR(){
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: "H2O-AWS_CT-JENKINS-H2O-ECR"]]) {
H2O_ECR_PWD = sh(script: 'aws ecr get-login-password --region us-east-1', returnStdout: true).trim()
sh """
docker login --username AWS --password $H2O_ECR_PWD $H2O_ECR
"""
}
}

0 comments on commit 5106c9c

Please sign in to comment.