Skip to content

Commit fe43b80

Browse files
jiridanekjstourac
andauthored
RHOAIENG-18400: chore(konflux): create helper scripts to generate konflux configs (opendatahub-io#903)
* NO-JIRA: chore(konflux): create helper scripts to generate konflux configs * Update ci/cached-builds/gha_pr_changed_files.py Co-authored-by: Jan Stourac <[email protected]> --------- Co-authored-by: Jan Stourac <[email protected]>
1 parent 65aed92 commit fe43b80

7 files changed

+1013
-16
lines changed

.tekton/image-registry.yaml

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
# List of images referenced from the Python code generation scripts for Tekton pipelines.
3+
#
4+
# The structure of this file must be compatible with
5+
# https://docs.renovatebot.com/modules/manager/tekton/
6+
#
7+
# Specifically, see `function getDeps` and `function getBundleValue()` in
8+
# https://github.com/renovatebot/renovate/blob/main/lib/modules/manager/tekton/extract.ts
9+
#
10+
# This is using the 'older-style' bundle references (see ^^^), because they are a bit less verbose
11+
#
12+
# Konflux (MintMaker) will then update the hashes in this yaml together with the generated Tekton pipelines
13+
# because the default renovate.json config includes `.tekton/**.yaml` (and `.yml`) files
14+
# https://github.com/konflux-ci/mintmaker/blob/289fefb5c7ac18c978b96080c2628d55d0712e83/config/renovate/renovate.json#L62-L70
15+
items:
16+
- spec:
17+
taskRef:
18+
bundle: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.4@sha256:3e01d3870212cfa0a5f947abcd74348adb80591a2f0828c64bb29323dff7225d
19+
- spec:
20+
taskRef:
21+
bundle: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:04f15cbce548e1db7770eee3f155ccb2cc0140a6c371dc67e9a34d83673ea0c0
22+
- spec:
23+
taskRef:
24+
bundle: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:63eb4a4c0cfb491276bff86fdad1c96bf238506388848e79001058450a8e843a
25+
- spec:
26+
taskRef:
27+
bundle: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:aab5f0f4906ba2c2a64a67b591c7ecf57018d066f1206ebc56158476e29f2cf3
28+
- spec:
29+
taskRef:
30+
bundle: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:752230a646483aebd465a942aef4f35c08e67185609ac26e19a3b931de9b7b0a
31+
- spec:
32+
taskRef:
33+
bundle: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:24feb32a91fb9960aa0a2d3a982dd549bad2d40074e1e5e3f9ae9739a66174b8
34+
- spec:
35+
taskRef:
36+
bundle: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.2@sha256:183b28fc7c3ca8bc81b00d695517cd2e0b7c31e13365bcfd7e3c758ce13c489c
37+
- spec:
38+
taskRef:
39+
bundle: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:a0a5b05286e3df5045432b3da3cc11224a831e05bc77c927cbfd00381f7f6235
40+
- spec:
41+
taskRef:
42+
bundle: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:c45aae9e7d4449e1ea3ef0fc59dec84b77831329ae2b03c1578e02bd051a2863
43+
- spec:
44+
taskRef:
45+
bundle: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:df8a25a3431a70544172ed4844f9d0c6229d39130633960729f825a031a7dea9
46+
- spec:
47+
taskRef:
48+
bundle: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.3@sha256:6673cbd19e4f1872dd194c91d0b1fe14cacd3768050f6516d3888f660e0732de
49+
- spec:
50+
taskRef:
51+
bundle: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.2@sha256:7595ba07e6bf3737a7ce51e0d75e43bd2658a9b9c5b59e161c005029ac758b3d
52+
- spec:
53+
taskRef:
54+
bundle: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.2@sha256:61a4b5afd0c49cd999fbe60b36e2d92750c7fb337942015929b3d3f393e3bde5
55+
- spec:
56+
taskRef:
57+
bundle: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:d6b15fa9874cceb1e68f564942507939499971d17108b5540990de035d1a8266
58+
- spec:
59+
taskRef:
60+
bundle: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:b1b78cb0b9eb6b6e333b35f90db182d4e86ef8e93acb4f3450dd1ad88ea3fab2
61+
- spec:
62+
taskRef:
63+
bundle: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.1@sha256:a8e5bec59687de42b77c579e931827de755623244a2c006701b4f9e08a3713b1
64+
- spec:
65+
taskRef:
66+
bundle: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:5e5f290359fd34ae4cc77cbbba6ef8c9907d752572d6dc2a00f5a4c504eb48bb
67+
- spec:
68+
taskRef:
69+
bundle: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:443e665458bd44f029c8e44e8d4c44e4faa8c533f129014ccb3c4c51fd89bbfc
70+
- spec:
71+
taskRef:
72+
bundle: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:5bc61290c6d56cb3d61409efdf522574e7d08a497f362d7456ed33d56189c4f9

.tekton/jupyter-minimal-ubi9-python-3-11-pull-request.yaml

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
# yamllint disable-file
2+
# This file is autogenerated by ci/cached-builds/konflux_generate_component_build_pipelines.py
23
apiVersion: tekton.dev/v1
34
kind: PipelineRun
45
metadata:
56
annotations:
6-
build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/notebooks?rev={{revision}}
7+
build.appstudio.openshift.io/repo: 'https://github.com/opendatahub-io/notebooks?rev={{revision}}'
78
build.appstudio.redhat.com/commit_sha: '{{revision}}'
89
build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}'
910
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
1011
pipelinesascode.tekton.dev/cancel-in-progress: "true"
11-
pipelinesascode.tekton.dev/max-keep-runs: "3"
12+
pipelinesascode.tekton.dev/max-keep-runs: '3'
1213
pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
1314
== "main" && has(body.repository) && body.repository.full_name == "opendatahub-io/notebooks"
1415
creationTimestamp: null
@@ -25,7 +26,7 @@ spec:
2526
- name: revision
2627
value: '{{revision}}'
2728
- name: output-image
28-
value: quay.io/redhat-user-workloads/rhoai-ide-konflux-tenant/jupyter-minimal-ubi9-python-3-11:on-pr-{{revision}}
29+
value: 'quay.io/redhat-user-workloads/rhoai-ide-konflux-tenant/jupyter-minimal-ubi9-python-3-11:on-pr-{{revision}}'
2930
- name: image-expires-after
3031
value: 5d
3132
- name: build-platforms

.tekton/jupyter-minimal-ubi9-python-3-11-push.yaml

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
# yamllint disable-file
2+
# This file is autogenerated by ci/cached-builds/konflux_generate_component_build_pipelines.py
23
apiVersion: tekton.dev/v1
34
kind: PipelineRun
45
metadata:
56
annotations:
6-
build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/notebooks?rev={{revision}}
7+
build.appstudio.openshift.io/repo: 'https://github.com/opendatahub-io/notebooks?rev={{revision}}'
78
build.appstudio.redhat.com/commit_sha: '{{revision}}'
89
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
910
pipelinesascode.tekton.dev/cancel-in-progress: "false"
10-
pipelinesascode.tekton.dev/max-keep-runs: "3"
11+
pipelinesascode.tekton.dev/max-keep-runs: '3'
1112
pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
1213
== "main" && has(body.repository) && body.repository.full_name == "opendatahub-io/notebooks"
1314
creationTimestamp: null
@@ -24,7 +25,7 @@ spec:
2425
- name: revision
2526
value: '{{revision}}'
2627
- name: output-image
27-
value: quay.io/redhat-user-workloads/rhoai-ide-konflux-tenant/jupyter-minimal-ubi9-python-3-11:{{revision}}
28+
value: 'quay.io/redhat-user-workloads/rhoai-ide-konflux-tenant/jupyter-minimal-ubi9-python-3-11:{{revision}}'
2829
- name: image-expires-after
2930
value: 28d
3031
- name: build-platforms

Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ endef
9494
#
9595
# PUSH_IMAGES: allows skipping podman push
9696
define image
97+
$(info #*# Image build Dockerfile: <$(2)> #(MACHINE-PARSED LINE)#*#...)
9798
$(eval BUILD_DIRECTORY := $(shell echo $(2) | sed 's/\/Dockerfile.*//'))
9899
$(info #*# Image build directory: <$(BUILD_DIRECTORY)> #(MACHINE-PARSED LINE)#*#...)
99100

ci/cached-builds/gha_pr_changed_files.py

+22-10
Original file line numberDiff line numberDiff line change
@@ -28,25 +28,34 @@ def list_changed_files(from_ref: str, to_ref: str) -> list[str]:
2828
return files
2929

3030

31-
def get_build_directory(make_target) -> str:
32-
directories = []
31+
def _query_build(make_target: str, query: str) -> str:
32+
results = []
3333

34-
pattern = re.compile(r"#\*# Image build directory: <(?P<dir>[^>]+)> #\(MACHINE-PARSED LINE\)#\*#\.\.\.")
34+
pattern = re.compile(r"#\*# " + query + r": <(?P<result>[^>]+)> #\(MACHINE-PARSED LINE\)#\*#\.\.\.")
3535
try:
3636
logging.debug(f"Running make in --just-print mode for target {make_target}")
3737
for line in subprocess.check_output([MAKE, make_target, "--just-print"], encoding="utf-8",
3838
cwd=PROJECT_ROOT).splitlines():
3939
if m := pattern.match(line):
40-
directories.append(m["dir"])
40+
results.append(m["result"])
4141
except subprocess.CalledProcessError as e:
4242
print(e.stderr, e.stdout)
4343
raise
4444

45-
if len(directories) != 1:
46-
raise Exception(f"Expected a single build directory for target '{make_target}': {directories}")
45+
if len(results) != 1:
46+
raise Exception(f"Expected a single query result for target '{make_target}': {results}")
47+
48+
logging.debug(f"Target {make_target} builds from {results[0]}")
49+
return results[0]
50+
51+
52+
def get_build_directory(make_target) -> str:
53+
return _query_build(make_target, "Image build directory")
54+
55+
56+
def get_build_dockerfile(make_target) -> str:
57+
return _query_build(make_target, "Image build Dockerfile")
4758

48-
logging.debug(f"Target {make_target} buildes from {directories[0]}")
49-
return directories[0]
5059

5160
def find_dockerfiles(directory: str) -> list:
5261
"""Finds and returns a list of files matching the pattern 'Dockerfile*' in the specified directory."""
@@ -67,10 +76,9 @@ def should_build_target(changed_files: list[str], target_directory: str) -> str:
6776
return changed_file
6877
# detect change in any of the files outside
6978
dockerfiles = find_dockerfiles(target_directory)
70-
results = ""
7179
for dockerfile in dockerfiles:
7280
stdout = subprocess.check_output([PROJECT_ROOT / "bin/buildinputs", target_directory + "/" + dockerfile],
73-
text=True, cwd=PROJECT_ROOT)
81+
text=True, cwd=PROJECT_ROOT)
7482
logging.debug(f"{target_directory=} {dockerfile=} {stdout=}")
7583
if stdout == "\n":
7684
# no dependencies
@@ -106,5 +114,9 @@ def test_get_build_directory(self):
106114
directory = get_build_directory("rocm-jupyter-pytorch-ubi9-python-3.11")
107115
assert directory == "jupyter/rocm/pytorch/ubi9-python-3.11"
108116

117+
def test_get_build_dockerfile(self):
118+
dockerfile = get_build_dockerfile("rocm-jupyter-pytorch-ubi9-python-3.11")
119+
assert dockerfile == "jupyter/rocm/pytorch/ubi9-python-3.11/Dockerfile.rocm"
120+
109121
def test_should_build_target(self):
110122
assert "" == should_build_target(["README.md"], "jupyter/datascience/ubi9-python-3.11")

0 commit comments

Comments
 (0)