Skip to content

Commit 5b0c1df

Browse files
saheerbsaheerb
committed
github actions for mbed-os-env docker management
1 parent 862a942 commit 5b0c1df

File tree

10 files changed

+1050
-0
lines changed

10 files changed

+1050
-0
lines changed

‎.github/workflows/ci_scripts/ghcr_utils.py

+144
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
Copyright (c) 2017-2021 ARM Limited. All rights reserved.
5+
SPDX-License-Identifier: Apache-2.0
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations
15+
"""
16+
import click
17+
import requests
18+
import logging
19+
import sys
20+
import time
21+
import json
22+
import subprocess
23+
24+
"""
25+
This file contains ghcr utlity wrapper used for:
26+
- retrieving digest of docker image
27+
- deleting images in ghcr
28+
"""
29+
30+
31+
@click.command()
32+
@click.pass_context
33+
@click.option("-r", "--repository", required=True)
34+
@click.option("-t", "--tag", required=True)
35+
@click.option("-p", "--platform", required=False)
36+
def get_digest(ctx, repository, tag, platform=None):
37+
"""
38+
Prints docker digest of specific platform of multi architecture image
39+
:param ctx: click context
40+
:param repository: docker repository
41+
:param tag: docker tag
42+
:param platform: platform for e.g, linux/arm64
43+
"""
44+
command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw"
45+
output = subprocess.run(
46+
command.split(), stdout=subprocess.PIPE, check=True
47+
).stdout.decode("utf-8")
48+
output = json.loads(output)
49+
50+
images = output["manifests"]
51+
digest = ""
52+
if len(images) and platform is None:
53+
logging.error(
54+
"This tag has more than one platform associated to it, please input a platform"
55+
)
56+
sys.exit(1)
57+
58+
for image in images:
59+
if platform != None:
60+
if (platform.split("/")[0] == image["platform"]["os"]) and (
61+
platform.split("/")[1] == image["platform"]["architecture"]
62+
):
63+
digest = image["digest"]
64+
else:
65+
digest = image["digest"]
66+
67+
if digest == "":
68+
logging.error("Digest not found. image not in repo for the given platform")
69+
sys.exit(1)
70+
71+
print(digest)
72+
73+
74+
@click.command()
75+
@click.pass_context
76+
@click.option("-r", "--repository", required=True)
77+
@click.option(
78+
"-n",
79+
"--number_of_days",
80+
default="10",
81+
help="number of days since image was created",
82+
required=False,
83+
)
84+
def delete_old_images(ctx, repository, number_of_days):
85+
"""
86+
delete old images from docker repository
87+
88+
:param ctx: click context
89+
:param repository: docker repository
90+
:param number_of_days: delete older than these number of days
91+
"""
92+
with requests.Session() as s:
93+
github_api_accept = "application/vnd.github.v3+json"
94+
s.headers.update(
95+
{"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept}
96+
)
97+
r = s.get(
98+
f"https://api.github.com/user/packages/container/{repository}/versions"
99+
)
100+
versions = r.json()
101+
version_id = None
102+
pattern = "%d.%m.%Y %H:%M:%S"
103+
pattern = "%Y-%m-%dT%H:%M:%SZ"
104+
current_time = time.time()
105+
for version in versions:
106+
logging.info(version)
107+
epoch = int(time.mktime(time.strptime(version["updated_at"], pattern)))
108+
109+
if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days):
110+
version_id = version["id"]
111+
logging.debug(f"deleteing image with version id {version_id}")
112+
113+
url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}"
114+
resp = s.delete(url)
115+
resp.raise_for_status()
116+
117+
118+
@click.group()
119+
@click.pass_context
120+
@click.option("-u", "--username", required=False)
121+
@click.option("-p", "--passwd", required=False)
122+
@click.option("-v", "--verbose", is_flag=True, default=False)
123+
def main(ctx, username, passwd, verbose):
124+
ctx.obj = {"username": username, "passwd": passwd}
125+
126+
if verbose:
127+
logging.basicConfig(
128+
stream=sys.stdout,
129+
format="%(levelname)s %(asctime)s %(message)s",
130+
datefmt="%m/%d/%Y %I:%M:%S %p",
131+
)
132+
logging.getLogger().setLevel(logging.DEBUG)
133+
else:
134+
logging.basicConfig(
135+
format="%(levelname)s %(asctime)s %(message)s",
136+
datefmt="%m/%d/%Y %I:%M:%S %p",
137+
)
138+
logging.getLogger().setLevel(logging.INFO)
139+
140+
141+
if __name__ == "__main__":
142+
main.add_command(get_digest)
143+
main.add_command(delete_old_images)
144+
main()

‎.github/workflows/docker_management.branch.yml

+237
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
name: Publish or Update docker image for head of branch
2+
# Design details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
3+
4+
on:
5+
6+
# passive update once a week
7+
schedule:
8+
- cron: '15 4 * * 6'
9+
10+
# build on master branch when there is changes for active update
11+
push:
12+
branches:
13+
- master
14+
15+
paths:
16+
- requirements.txt
17+
- docker_images/mbed-os-env/**
18+
- .github/workflows/docker_management.branch.yml
19+
20+
21+
# manual trigger when needed
22+
workflow_dispatch:
23+
24+
25+
jobs:
26+
prepare-tags:
27+
runs-on: ubuntu-latest
28+
29+
steps:
30+
-
31+
name: Extract branch name
32+
shell: bash
33+
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
34+
id: extract_branch
35+
36+
-
37+
name: Checkout
38+
uses: actions/checkout@v2
39+
with:
40+
fetch-depth: 0
41+
42+
-
43+
name: Set UUID
44+
id: generate-uuid
45+
uses: filipstefansson/uuid-action@v1
46+
47+
# set docker tags we are building, and intending to publish
48+
# dev-tag is temporary for testing purpose. This should be considered as unstable.
49+
# dated-tag is created for versioning purpose
50+
# prod-tag-latest could be used by customers, CI etc for keeping up to date
51+
-
52+
name: Get build information
53+
shell: bash
54+
run: |
55+
mkdir -p build_info
56+
date=$(date +"%Y.%m.%dT%H.%M.%S")
57+
echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag
58+
echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated
59+
echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest
60+
echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version
61+
62+
-
63+
name: Archive information
64+
uses: actions/upload-artifact@v2
65+
with:
66+
name: build-info
67+
path: build_info
68+
69+
70+
build-container:
71+
runs-on: ubuntu-latest
72+
needs: prepare-tags
73+
outputs:
74+
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
75+
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}
76+
77+
steps:
78+
-
79+
name: unarchive artefacts
80+
uses: actions/download-artifact@v2
81+
with:
82+
name: build-info
83+
84+
-
85+
name: Get build info from archive
86+
shell: bash
87+
id: build_info
88+
run: |
89+
value=`cat dev_tag`
90+
echo "DEV TAG is $value"
91+
echo "::set-output name=DOCKER_DEV_TAG::$value"
92+
value=`cat prod_tag_dated`
93+
echo "PROD TAG DATED is $value"
94+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
95+
value=`cat prod_tag_latest`
96+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
97+
echo "PROD TAG is $value"
98+
99+
-
100+
name: Set up Docker Buildx
101+
uses: docker/setup-buildx-action@v1
102+
103+
-
104+
name: Set up QEMU
105+
uses: docker/setup-qemu-action@v1
106+
107+
-
108+
name: Login to DockerHub
109+
uses: docker/login-action@v1
110+
with:
111+
registry: ghcr.io
112+
username: ${{ github.actor }}
113+
password: ${{ secrets.GITHUB_TOKEN }}
114+
115+
-
116+
name: Checkout
117+
uses: actions/checkout@v2
118+
119+
-
120+
name: Build docker containers
121+
uses: docker/build-push-action@v2
122+
id: docker_build_dev
123+
with:
124+
context: .
125+
platforms: linux/amd64,linux/arm64
126+
push: true
127+
file: ./docker_images/mbed-os-env/Dockerfile
128+
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}
129+
130+
test-container:
131+
runs-on: ubuntu-latest
132+
needs: build-container
133+
strategy:
134+
matrix:
135+
platform: [linux/amd64, linux/arm64]
136+
137+
steps:
138+
-
139+
name: unarchive artefacts
140+
uses: actions/download-artifact@v2
141+
with:
142+
name: build-info
143+
144+
-
145+
name: Get build info from archive
146+
shell: bash
147+
id: build_info
148+
run: |
149+
value=`cat dev_tag`
150+
echo "TAG is $value"
151+
echo "::set-output name=DOCKER_DEV_TAG::$value"
152+
value=`cat prod_tag_dated`
153+
echo "TAG is $value"
154+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
155+
value=`cat prod_tag_latest`
156+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
157+
value=`cat mbed_os_version`
158+
echo "::set-output name=MBED_OS_VERSION::$value"
159+
160+
-
161+
name: Checkout
162+
uses: actions/checkout@v2
163+
164+
-
165+
name: Find DEV DOCKER DIGEST
166+
id: docker_info_dev
167+
run: |
168+
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
169+
echo "::set-output name=DIGEST::$DIGEST"
170+
echo "Docker DIGEST: $DIGEST"
171+
172+
# as the dev images are created only for master branch, run test against
173+
# development branch of blinky
174+
-
175+
name: Checkout
176+
uses: actions/checkout@v2
177+
with:
178+
repository: ARMmbed/mbed-os-example-blinky
179+
path: mbed-os-example-blinky
180+
ref: development
181+
-
182+
name: Set up QEMU
183+
uses: docker/setup-qemu-action@v1
184+
185+
-
186+
name: test the container
187+
id: test
188+
uses: addnab/docker-run-action@v3
189+
with:
190+
username: ${{ github.actor }}
191+
password: ${{ secrets.GITHUB_TOKEN }}
192+
registry: ghcr.io
193+
options: -v ${{ github.workspace }}:/work -w=/work
194+
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }}
195+
shell: bash
196+
197+
run: |
198+
uname -m
199+
cd mbed-os-example-blinky
200+
mbed deploy
201+
# build using CLI1
202+
mbed compile -m K64F -t GCC_ARM
203+
204+
# build using CLI2
205+
mbed-tools compile -m K64F -t GCC_ARM
206+
207+
208+
deploy-container:
209+
runs-on: ubuntu-latest
210+
needs: test-container
211+
212+
steps:
213+
-
214+
name: unarchive artefacts
215+
uses: actions/download-artifact@v2
216+
with:
217+
name: build-info
218+
219+
-
220+
name: Get build info from archive
221+
shell: bash
222+
id: build_info
223+
run: |
224+
value=`cat dev_tag`
225+
echo "TAG is $value"
226+
echo "::set-output name=DOCKER_DEV_TAG::$value"
227+
value=`cat prod_tag_dated`
228+
echo "TAG is $value"
229+
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
230+
value=`cat prod_tag_latest`
231+
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
232+
233+
-
234+
name: copy dev tag to prod
235+
run: |
236+
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }}
237+
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}

0 commit comments

Comments
 (0)