Skip to content

Commit 7129092

Browse files
sutaakarsutaakar
committed
OLM install and upgrade PR check for CodeFlare stack
1 parent d376f12 commit 7129092

File tree

9 files changed

+305
-61
lines changed

9 files changed

+305
-61
lines changed

Diff for: ‎.github/actions/kind/action.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "Set up KinD"
2+
description: "Step to start and configure KinD cluster"
3+
4+
runs:
5+
using: "composite"
6+
steps:
7+
- name: Init directories
8+
shell: bash
9+
run: |
10+
TEMP_DIR="$(pwd)/tmp"
11+
mkdir -p "${TEMP_DIR}"
12+
echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
13+
14+
mkdir -p "$(pwd)/bin"
15+
echo "$(pwd)/bin" >> $GITHUB_PATH
16+
17+
- name: Container image registry
18+
shell: bash
19+
run: |
20+
podman run -d -p 5000:5000 --name registry registry:2.8.1
21+
22+
export REGISTRY_ADDRESS=$(hostname -i):5000
23+
echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
24+
echo "Container image registry started at ${REGISTRY_ADDRESS}"
25+
26+
KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
27+
echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
28+
envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE}
29+
30+
sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
31+
[[registry]]
32+
prefix = "$REGISTRY_ADDRESS"
33+
insecure = true
34+
location = "$REGISTRY_ADDRESS"
35+
EOF'
36+
37+
- name: Setup KinD cluster
38+
uses: helm/[email protected]
39+
with:
40+
cluster_name: cluster
41+
version: v0.17.0
42+
config: ${{ env.KIND_CONFIG_FILE }}
43+
44+
- name: Print cluster info
45+
shell: bash
46+
run: |
47+
echo "KinD cluster:"
48+
kubectl cluster-info
49+
kubectl describe nodes

Diff for: ‎test/e2e/kind.yaml renamed to ‎.github/resources-kind/kind.yaml

File renamed without changes.

Diff for: ‎.github/resources-olm-upgrade/catalogsource.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: operators.coreos.com/v1alpha1
2+
kind: CatalogSource
3+
metadata:
4+
name: codeflare-olm-test
5+
namespace: olm
6+
spec:
7+
displayName: ''
8+
grpcPodConfig:
9+
securityContextConfig: restricted
10+
image: "${CATALOG_BASE_IMG}"
11+
publisher: ''
12+
sourceType: grpc

Diff for: ‎.github/resources-olm-upgrade/operatorgroup.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: operators.coreos.com/v1
2+
kind: OperatorGroup
3+
metadata:
4+
name: openshift-operators
5+
namespace: openshift-operators

Diff for: ‎.github/resources-olm-upgrade/subscription.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: operators.coreos.com/v1alpha1
2+
kind: Subscription
3+
metadata:
4+
name: codeflare-operator
5+
namespace: openshift-operators
6+
spec:
7+
channel: alpha
8+
installPlanApproval: Automatic
9+
name: codeflare-operator
10+
source: codeflare-olm-test
11+
sourceNamespace: olm
12+
startingCSV: codeflare-operator.${PREVIOUS_VERSION}

Diff for: ‎.github/workflows/e2e_tests.yaml

+2-40
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,6 @@ jobs:
5454
with:
5555
submodules: recursive
5656

57-
- name: Init directories
58-
run: |
59-
TEMP_DIR="$(pwd)/tmp"
60-
mkdir -p "${TEMP_DIR}"
61-
echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
62-
63-
mkdir -p "$(pwd)/bin"
64-
echo "$(pwd)/bin" >> $GITHUB_PATH
65-
6657
- name: Set Go
6758
uses: actions/setup-go@v3
6859
with:
@@ -73,37 +64,8 @@ jobs:
7364
with:
7465
token: ${{ secrets.GITHUB_TOKEN }}
7566

76-
- name: Container image registry
77-
run: |
78-
podman run -d -p 5000:5000 --name registry registry:2.8.1
79-
80-
export REGISTRY_ADDRESS=$(hostname -i):5000
81-
echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
82-
echo "Container image registry started at ${REGISTRY_ADDRESS}"
83-
84-
KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
85-
echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
86-
envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE}
87-
88-
sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
89-
[[registry]]
90-
prefix = "$REGISTRY_ADDRESS"
91-
insecure = true
92-
location = "$REGISTRY_ADDRESS"
93-
EOF'
94-
95-
- name: Setup KinD cluster
96-
uses: helm/[email protected]
97-
with:
98-
cluster_name: cluster
99-
version: v0.17.0
100-
config: ${{ env.KIND_CONFIG_FILE }}
101-
102-
- name: Print cluster info
103-
run: |
104-
echo "KinD cluster:"
105-
kubectl cluster-info
106-
kubectl describe nodes
67+
- name: Setup and start KinD cluster
68+
uses: ./.github/actions/kind
10769

10870
- name: Deploy CodeFlare stack
10971
id: deploy

Diff for: ‎.github/workflows/olm_tests.yaml

+196
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog
2+
3+
name: OLM Install and Upgrade
4+
5+
on:
6+
pull_request:
7+
branches:
8+
- main
9+
- 'release-*'
10+
paths-ignore:
11+
- 'docs/**'
12+
- '**.adoc'
13+
- '**.md'
14+
- 'LICENSE'
15+
16+
concurrency:
17+
group: ${{ github.head_ref }}-${{ github.workflow }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
kubernetes:
22+
runs-on: ubuntu-20.04
23+
timeout-minutes: 60
24+
env:
25+
OLM_VERSION: v0.24.0
26+
VERSION: "v999.0.0" # Need to supply some semver version for bundle to be properly generated
27+
CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13"
28+
CODEFLARE_TEST_TIMEOUT_SHORT: "1m"
29+
CODEFLARE_TEST_TIMEOUT_MEDIUM: "3m"
30+
CODEFLARE_TEST_TIMEOUT_LONG: "10m"
31+
32+
steps:
33+
- name: Cleanup
34+
run: |
35+
ls -lart
36+
echo "Initial status:"
37+
df -h
38+
39+
echo "Cleaning up resources:"
40+
sudo swapoff -a
41+
sudo rm -f /swapfile
42+
sudo apt clean
43+
sudo rm -rf /usr/share/dotnet
44+
sudo rm -rf /opt/ghc
45+
sudo rm -rf "/usr/local/share/boost"
46+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
47+
docker rmi $(docker image ls -aq)
48+
49+
echo "Final status:"
50+
df -h
51+
52+
- uses: actions/checkout@v3
53+
with:
54+
fetch-depth: 0 # fetching also previous commits to get tags
55+
56+
- name: Set Go
57+
uses: actions/setup-go@v3
58+
with:
59+
go-version: v1.18
60+
61+
- name: Set up gotestfmt
62+
uses: gotesttools/gotestfmt-action@v2
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Store latest tag on branch
67+
run: |
68+
echo "PREVIOUS_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
69+
70+
- name: Setup and start KinD cluster
71+
uses: ./.github/actions/kind
72+
73+
- name: Install OLM
74+
run: |
75+
kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml
76+
# wait for a while to be sure CRDs are installed
77+
sleep 1
78+
kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml
79+
80+
- name: Create openshift-operator namespace and OperatorGroup
81+
run: |
82+
# Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161
83+
kubectl create namespace openshift-operators
84+
kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml
85+
86+
- name: Deploy latest released CodeFlare operator from OLM
87+
id: deploy
88+
run: |
89+
echo Deploying CodeFlare operator using Subscription
90+
envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml
91+
envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml
92+
kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml
93+
kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml
94+
95+
echo Waiting for Subscription to be ready
96+
make subscription-ready
97+
98+
echo Waiting for Deployment to be ready
99+
make deployment-available -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
100+
101+
echo Checking that correct CSV is available
102+
CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${PREVIOUS_VERSION} -n openshift-operators -o json | jq -r .spec.version)
103+
if [ "v${CSV_VERSION}" != "${PREVIOUS_VERSION}" ]; then
104+
echo "CSV version v${CSV_VERSION} doesn't match expected version ${PREVIOUS_VERSION}"
105+
exit 1
106+
fi
107+
env:
108+
SUBSCRIPTION_NAME: "codeflare-operator"
109+
SUBSCRIPTION_NAMESPACE: "openshift-operators"
110+
111+
- name: Deploy CodeFlare stack (MCAD, KubeRay)
112+
run: |
113+
make setup-e2e
114+
115+
- name: Run e2e tests
116+
run: |
117+
export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
118+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
119+
120+
set -euo pipefail
121+
go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-original.log | gotestfmt
122+
123+
- name: Build operator and catalog image
124+
run: |
125+
make image-push
126+
make bundle-build
127+
make bundle-push
128+
make catalog-build-from-index
129+
make catalog-push
130+
env:
131+
IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1"
132+
BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1"
133+
CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
134+
OPM_BUNDLE_OPT: "--use-http"
135+
BUNDLE_PUSH_OPT: "--tls-verify=false"
136+
CATALOG_PUSH_OPT: "--tls-verify=false"
137+
138+
- name: Update Operator to the built version
139+
run: |
140+
ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name)
141+
echo "Running old operator pod name is ${ORIGINAL_POD_NAME}"
142+
143+
echo Updating custom CatalogSource image to the built CatalogSource with latest operator
144+
kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}"
145+
146+
echo Waiting for previous operator pod to get deleted
147+
kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators
148+
149+
echo Waiting for Subscription to be ready
150+
make subscription-ready
151+
152+
echo Waiting for Deployment to be ready
153+
make deployment-available -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
154+
155+
echo Checking that correct CSV is available
156+
CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version)
157+
if [ "v${CSV_VERSION}" != "${VERSION}" ]; then
158+
echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}"
159+
exit 1
160+
fi
161+
env:
162+
CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
163+
SUBSCRIPTION_NAME: "codeflare-operator"
164+
SUBSCRIPTION_NAMESPACE: "openshift-operators"
165+
166+
- name: Run e2e tests against built operator
167+
run: |
168+
set -euo pipefail
169+
go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-upgraded.log | gotestfmt
170+
171+
- name: Print CodeFlare operator logs
172+
if: always() && steps.deploy.outcome == 'success'
173+
run: |
174+
echo "Printing CodeFlare operator logs"
175+
kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
176+
177+
- name: Print MCAD controller logs
178+
if: always() && steps.deploy.outcome == 'success'
179+
run: |
180+
echo "Printing MCAD controller logs"
181+
kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log
182+
183+
- name: Print KubeRay operator logs
184+
if: always() && steps.deploy.outcome == 'success'
185+
run: |
186+
echo "Printing KubeRay operator logs"
187+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
188+
189+
- name: Upload logs
190+
uses: actions/upload-artifact@v3
191+
if: always() && steps.deploy.outcome == 'success'
192+
with:
193+
name: logs
194+
retention-days: 10
195+
path: |
196+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

Diff for: ‎Makefile

+29-3
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releas
342342
.PHONY: install-operator-sdk
343343
install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome
344344
$(OPERATOR_SDK): $(LOCALBIN)
345-
curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output-dir $(LOCALBIN) --output operator-sdk
345+
curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk
346346
chmod +x $(OPERATOR_SDK)
347347

348348
.PHONY: validate-bundle
@@ -366,7 +366,7 @@ bundle-build: bundle ## Build the bundle image.
366366

367367
.PHONY: bundle-push
368368
bundle-push: ## Push the bundle image.
369-
$(MAKE) image-push IMG=$(BUNDLE_IMG)
369+
podman push $(BUNDLE_IMG) $(BUNDLE_PUSH_OPT)
370370

371371
.PHONY: openshift-community-operator-release
372372
openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository
@@ -413,10 +413,21 @@ endif
413413
catalog-build: opm ## Build a catalog image.
414414
$(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
415415

416+
# Build a catalog image by adding bundle images to existing catalog using the operator package manager tool, 'opm'.
417+
.PHONY: catalog-build-from-index
418+
catalog-build-from-index: opm ## Build a catalog image.
419+
mkdir catalog
420+
$(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml
421+
$(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml
422+
sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml
423+
$(OPM) validate catalog
424+
$(OPM) generate dockerfile catalog
425+
podman build . -f catalog.Dockerfile -t $(CATALOG_IMG)
426+
416427
# Push the catalog image.
417428
.PHONY: catalog-push
418429
catalog-push: ## Push a catalog image.
419-
$(MAKE) image-push IMG=$(CATALOG_IMG)
430+
podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT)
420431

421432
.PHONY: test-unit
422433
test-unit: defaults manifests generate fmt vet envtest ## Run unit tests.
@@ -429,3 +440,18 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests.
429440
.PHONY: setup-e2e
430441
setup-e2e: ## Set up e2e tests.
431442
KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh
443+
444+
# Wait until Subscription is in "AtLatestKnown" state
445+
.PHONY: subscription-ready
446+
subscription-ready:
447+
timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done'
448+
449+
# Default timeout for waiting actions
450+
TIMEOUT ?= 180
451+
452+
# Wait until Deployment is in "Available" state
453+
.PHONY: deployment-available
454+
deployment-available:
455+
# Wait until Deployment exists first, then use kubectl wait
456+
timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done'
457+
kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE)

0 commit comments

Comments
 (0)