Skip to content

Commit 7b1bcab

Browse files
sutaakarsutaakar
committed
OLM install and upgrade PR check for CodeFlare stack
1 parent 70f0719 commit 7b1bcab

File tree

8 files changed

+308
-43
lines changed

8 files changed

+308
-43
lines changed

Diff for: ‎.github/actions/kind/action.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "Set up KinD"
2+
description: "Step to start and configure KinD cluster"
3+
4+
runs:
5+
using: "composite"
6+
steps:
7+
- name: Init directories
8+
shell: bash
9+
run: |
10+
TEMP_DIR="$(pwd)/tmp"
11+
mkdir -p "${TEMP_DIR}"
12+
echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
13+
14+
mkdir -p "$(pwd)/bin"
15+
echo "$(pwd)/bin" >> $GITHUB_PATH
16+
17+
- name: Container image registry
18+
shell: bash
19+
run: |
20+
podman run -d -p 5000:5000 --name registry registry:2.8.1
21+
22+
export REGISTRY_ADDRESS=$(hostname -i):5000
23+
echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
24+
echo "Container image registry started at ${REGISTRY_ADDRESS}"
25+
26+
KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
27+
echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
28+
envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE}
29+
30+
sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
31+
[[registry]]
32+
prefix = "$REGISTRY_ADDRESS"
33+
insecure = true
34+
location = "$REGISTRY_ADDRESS"
35+
EOF'
36+
37+
- name: Setup KinD cluster
38+
uses: helm/[email protected]
39+
with:
40+
cluster_name: cluster
41+
version: v0.17.0
42+
config: ${{ env.KIND_CONFIG_FILE }}
43+
44+
- name: Print cluster info
45+
shell: bash
46+
run: |
47+
echo "KinD cluster:"
48+
kubectl cluster-info
49+
kubectl describe nodes

Diff for: ‎test/e2e/kind.yaml renamed to ‎.github/resources-kind/kind.yaml

File renamed without changes.

Diff for: ‎.github/resources-olm-upgrade/catalogsource.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: operators.coreos.com/v1alpha1
2+
kind: CatalogSource
3+
metadata:
4+
name: codeflare-olm-test
5+
namespace: olm
6+
spec:
7+
displayName: ''
8+
grpcPodConfig:
9+
securityContextConfig: restricted
10+
image: "${CATALOG_BASE_IMG}"
11+
publisher: ''
12+
sourceType: grpc

Diff for: ‎.github/resources-olm-upgrade/operatorgroup.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: operators.coreos.com/v1
2+
kind: OperatorGroup
3+
metadata:
4+
name: openshift-operators
5+
namespace: openshift-operators

Diff for: ‎.github/resources-olm-upgrade/subscription.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: operators.coreos.com/v1alpha1
2+
kind: Subscription
3+
metadata:
4+
name: codeflare-operator
5+
namespace: openshift-operators
6+
spec:
7+
channel: alpha
8+
installPlanApproval: Automatic
9+
name: codeflare-operator
10+
source: codeflare-olm-test
11+
sourceNamespace: olm

Diff for: ‎.github/workflows/e2e_tests.yaml

+2-40
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,6 @@ jobs:
5454
with:
5555
submodules: recursive
5656

57-
- name: Init directories
58-
run: |
59-
TEMP_DIR="$(pwd)/tmp"
60-
mkdir -p "${TEMP_DIR}"
61-
echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
62-
63-
mkdir -p "$(pwd)/bin"
64-
echo "$(pwd)/bin" >> $GITHUB_PATH
65-
6657
- name: Set Go
6758
uses: actions/setup-go@v3
6859
with:
@@ -73,37 +64,8 @@ jobs:
7364
with:
7465
token: ${{ secrets.GITHUB_TOKEN }}
7566

76-
- name: Container image registry
77-
run: |
78-
podman run -d -p 5000:5000 --name registry registry:2.8.1
79-
80-
export REGISTRY_ADDRESS=$(hostname -i):5000
81-
echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
82-
echo "Container image registry started at ${REGISTRY_ADDRESS}"
83-
84-
KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
85-
echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
86-
envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE}
87-
88-
sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
89-
[[registry]]
90-
prefix = "$REGISTRY_ADDRESS"
91-
insecure = true
92-
location = "$REGISTRY_ADDRESS"
93-
EOF'
94-
95-
- name: Setup KinD cluster
96-
uses: helm/[email protected]
97-
with:
98-
cluster_name: cluster
99-
version: v0.17.0
100-
config: ${{ env.KIND_CONFIG_FILE }}
101-
102-
- name: Print cluster info
103-
run: |
104-
echo "KinD cluster:"
105-
kubectl cluster-info
106-
kubectl describe nodes
67+
- name: Setup and start KinD cluster
68+
uses: ./.github/actions/kind
10769

10870
- name: Deploy CodeFlare stack
10971
id: deploy

Diff for: ‎.github/workflows/olm_tests.yaml

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog
2+
3+
name: OLM Install and Upgrade
4+
5+
on:
6+
pull_request:
7+
branches:
8+
- main
9+
- 'release-*'
10+
paths-ignore:
11+
- 'docs/**'
12+
- '**.adoc'
13+
- '**.md'
14+
- 'LICENSE'
15+
16+
concurrency:
17+
group: ${{ github.head_ref }}-${{ github.workflow }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
kubernetes:
22+
runs-on: ubuntu-20.04
23+
timeout-minutes: 60
24+
env:
25+
OLM_VERSION: v0.24.0
26+
VERSION: "v0.0.0-ghaction" # Need to supply some semver version for bundle to be properly generated
27+
CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13"
28+
CODEFLARE_TEST_TIMEOUT_SHORT: "1m"
29+
CODEFLARE_TEST_TIMEOUT_MEDIUM: "5m"
30+
CODEFLARE_TEST_TIMEOUT_LONG: "10m"
31+
32+
steps:
33+
- name: Cleanup
34+
run: |
35+
ls -lart
36+
echo "Initial status:"
37+
df -h
38+
39+
echo "Cleaning up resources:"
40+
sudo swapoff -a
41+
sudo rm -f /swapfile
42+
sudo apt clean
43+
sudo rm -rf /usr/share/dotnet
44+
sudo rm -rf /opt/ghc
45+
sudo rm -rf "/usr/local/share/boost"
46+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
47+
docker rmi $(docker image ls -aq)
48+
49+
echo "Final status:"
50+
df -h
51+
52+
- uses: actions/checkout@v3
53+
with:
54+
fetch-depth: 0 # fetching also previous commits to get tags
55+
56+
- name: Set Go
57+
uses: actions/setup-go@v3
58+
with:
59+
go-version: v1.18
60+
61+
- name: Set up gotestfmt
62+
uses: gotesttools/gotestfmt-action@v2
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Setup and start KinD cluster
67+
uses: ./.github/actions/kind
68+
69+
- name: Install OLM
70+
run: |
71+
kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml
72+
# wait for a while to be sure CRDs are installed
73+
sleep 1
74+
kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml
75+
76+
- name: Create openshift-operator namespace and OperatorGroup
77+
run: |
78+
# Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161
79+
kubectl create namespace openshift-operators
80+
kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml
81+
82+
- name: Deploy latest released CodeFlare operator from OLM
83+
id: deploy
84+
run: |
85+
echo Deploying CodeFlare operator using Subscription
86+
envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml
87+
envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml
88+
89+
kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml
90+
make wait-for-catalog-source
91+
92+
kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml
93+
94+
echo Waiting for Subscription to be ready
95+
make wait-for-subscription
96+
97+
echo Waiting for Deployment to be ready
98+
make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
99+
env:
100+
CATALOG_SOURCE_NAME: "codeflare-olm-test"
101+
CATALOG_SOURCE_NAMESPACE: "olm"
102+
SUBSCRIPTION_NAME: "codeflare-operator"
103+
SUBSCRIPTION_NAMESPACE: "openshift-operators"
104+
105+
- name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build)
106+
run: |
107+
CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version)
108+
echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV
109+
110+
- name: Deploy CodeFlare stack (MCAD, KubeRay)
111+
run: |
112+
make setup-e2e
113+
114+
- name: Run e2e tests
115+
run: |
116+
export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
117+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
118+
119+
set -euo pipefail
120+
go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-original.log | gotestfmt
121+
122+
- name: Build operator and catalog image
123+
run: |
124+
make image-push
125+
make bundle-build
126+
make bundle-push
127+
make catalog-build-from-index
128+
make catalog-push
129+
env:
130+
IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1"
131+
BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1"
132+
CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
133+
OPM_BUNDLE_OPT: "--use-http"
134+
BUNDLE_PUSH_OPT: "--tls-verify=false"
135+
CATALOG_PUSH_OPT: "--tls-verify=false"
136+
137+
- name: Update Operator to the built version
138+
run: |
139+
ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name)
140+
echo "Running old operator pod name is ${ORIGINAL_POD_NAME}"
141+
142+
echo Updating custom CatalogSource image to the built CatalogSource with latest operator
143+
kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}"
144+
145+
echo Waiting for previous operator pod to get deleted
146+
kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators
147+
148+
echo Waiting for Subscription to be ready
149+
make wait-for-subscription
150+
151+
echo Waiting for Deployment to be ready
152+
make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
153+
154+
echo Checking that correct CSV is available
155+
CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version)
156+
if [ "v${CSV_VERSION}" != "${VERSION}" ]; then
157+
echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}"
158+
exit 1
159+
fi
160+
env:
161+
CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
162+
SUBSCRIPTION_NAME: "codeflare-operator"
163+
SUBSCRIPTION_NAMESPACE: "openshift-operators"
164+
165+
- name: Run e2e tests against built operator
166+
run: |
167+
set -euo pipefail
168+
go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-upgraded.log | gotestfmt
169+
170+
- name: Print CodeFlare operator logs
171+
if: always() && steps.deploy.outcome == 'success'
172+
run: |
173+
echo "Printing CodeFlare operator logs"
174+
kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
175+
176+
- name: Print MCAD controller logs
177+
if: always() && steps.deploy.outcome == 'success'
178+
run: |
179+
echo "Printing MCAD controller logs"
180+
kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log
181+
182+
- name: Print KubeRay operator logs
183+
if: always() && steps.deploy.outcome == 'success'
184+
run: |
185+
echo "Printing KubeRay operator logs"
186+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
187+
188+
- name: Upload logs
189+
uses: actions/upload-artifact@v3
190+
if: always() && steps.deploy.outcome == 'success'
191+
with:
192+
name: logs
193+
retention-days: 10
194+
path: |
195+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

Diff for: ‎Makefile

+34-3
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releas
342342
.PHONY: install-operator-sdk
343343
install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome
344344
$(OPERATOR_SDK): $(LOCALBIN)
345-
curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output-dir $(LOCALBIN) --output operator-sdk
345+
curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk
346346
chmod +x $(OPERATOR_SDK)
347347

348348
.PHONY: validate-bundle
@@ -366,7 +366,7 @@ bundle-build: bundle ## Build the bundle image.
366366

367367
.PHONY: bundle-push
368368
bundle-push: ## Push the bundle image.
369-
$(MAKE) image-push IMG=$(BUNDLE_IMG)
369+
podman push $(BUNDLE_IMG) $(BUNDLE_PUSH_OPT)
370370

371371
.PHONY: openshift-community-operator-release
372372
openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository
@@ -413,10 +413,21 @@ endif
413413
catalog-build: opm ## Build a catalog image.
414414
$(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
415415

416+
# Build a catalog image by adding bundle images to existing catalog using the operator package manager tool, 'opm'.
417+
.PHONY: catalog-build-from-index
418+
catalog-build-from-index: opm ## Build a catalog image.
419+
mkdir catalog
420+
$(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml
421+
$(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml
422+
sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml
423+
$(OPM) validate catalog
424+
$(OPM) generate dockerfile catalog
425+
podman build . -f catalog.Dockerfile -t $(CATALOG_IMG)
426+
416427
# Push the catalog image.
417428
.PHONY: catalog-push
418429
catalog-push: ## Push a catalog image.
419-
$(MAKE) image-push IMG=$(CATALOG_IMG)
430+
podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT)
420431

421432
.PHONY: test-unit
422433
test-unit: defaults manifests generate fmt vet envtest ## Run unit tests.
@@ -429,3 +440,23 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests.
429440
.PHONY: setup-e2e
430441
setup-e2e: ## Set up e2e tests.
431442
KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh
443+
444+
# Wait until CatalogSource is in "READY" state
445+
.PHONY: wait-for-catalog-source
446+
wait-for-catalog-source:
447+
timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done'
448+
449+
# Wait until Subscription is in "AtLatestKnown" state
450+
.PHONY: wait-for-subscription
451+
wait-for-subscription:
452+
timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done'
453+
454+
# Default timeout for waiting actions
455+
TIMEOUT ?= 180
456+
457+
# Wait until Deployment is in "Available" state
458+
.PHONY: wait-for-deployment
459+
wait-for-deployment:
460+
# Wait until Deployment exists first, then use kubectl wait
461+
timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done'
462+
kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE)

0 commit comments

Comments
 (0)