Skip to content

Commit 53d0f2f

Browse files
Yuggupta27Yuggupta27
authored
Merge pull request #40 from subhamkrai/intree-ci
ci: add ci to test intree migration to ceph-csi
2 parents f2f8788 + 6856db8 commit 53d0f2f

File tree

4 files changed

+208
-40
lines changed

4 files changed

+208
-40
lines changed

‎.github/workflows/flex-migration-test.yaml

+37-31
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,47 @@ jobs:
1111
Flex-to-CSI:
1212
runs-on: ubuntu-latest
1313
steps:
14-
- name: checkout
15-
uses: actions/checkout@v2
16-
with:
17-
fetch-depth: 0
14+
- name: checkout
15+
uses: actions/checkout@v2
16+
with:
17+
fetch-depth: 0
1818

19-
- name: setup cluster resources
20-
uses: ./.github/workflows/setup-cluster-resources
21-
with:
22-
github-token: ${{ secrets.GITHUB_TOKEN }}
19+
- name: setup cluster resources
20+
uses: ./.github/workflows/setup-cluster-resources
21+
with:
22+
github-token: ${{ secrets.GITHUB_TOKEN }}
2323

24-
- name: Test migration of all PVC
25-
run: tests/script/github_action_helper.sh test_flex_migration_for_all_pvc
24+
- name: setup flex
25+
run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex
2626

27-
- name: setup tmate session for debugging when event is PR
28-
if: failure()
29-
uses: mxschmitt/action-tmate@v3
30-
timeout-minutes: 300
27+
- name: Test migration of all PVC
28+
run: tests/script/github_action_helper.sh test_flex_migration_for_all_pvc
29+
30+
- name: setup tmate session for debugging when event is PR
31+
if: failure()
32+
uses: mxschmitt/action-tmate@v3
33+
timeout-minutes: 300
3134

3235
Flex-Mirgate-single-PVC:
3336
runs-on: ubuntu-latest
3437
steps:
35-
- name: checkout
36-
uses: actions/checkout@v2
37-
with:
38-
fetch-depth: 0
39-
40-
- name: setup cluster resources
41-
uses: ./.github/workflows/setup-cluster-resources
42-
with:
43-
github-token: ${{ secrets.GITHUB_TOKEN }}
44-
45-
- name: Test migration of all PVC
46-
run: tests/script/github_action_helper.sh test_flex_migration_for_single_pvc
47-
48-
- name: setup tmate session for debugging when event is PR
49-
if: failure()
50-
uses: mxschmitt/action-tmate@v3
51-
timeout-minutes: 300
38+
- name: checkout
39+
uses: actions/checkout@v2
40+
with:
41+
fetch-depth: 0
42+
43+
- name: setup cluster resources
44+
uses: ./.github/workflows/setup-cluster-resources
45+
with:
46+
github-token: ${{ secrets.GITHUB_TOKEN }}
47+
48+
- name: setup flex
49+
run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex
50+
51+
- name: Test migration of all PVC
52+
run: tests/script/github_action_helper.sh test_flex_migration_for_single_pvc
53+
54+
- name: setup tmate session for debugging when event is PR
55+
if: failure()
56+
uses: mxschmitt/action-tmate@v3
57+
timeout-minutes: 300

‎.github/workflows/in-tree-migtation.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Migrate-to-In-tree
2+
on:
3+
pull_request:
4+
5+
defaults:
6+
run:
7+
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
8+
shell: bash --noprofile --norc -eo pipefail -x {0}
9+
10+
jobs:
11+
In-tree-to-CSI:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: checkout
15+
uses: actions/checkout@v2
16+
with:
17+
fetch-depth: 0
18+
19+
- name: setup cluster resources
20+
uses: ./.github/workflows/setup-cluster-resources
21+
with:
22+
github-token: ${{ secrets.GITHUB_TOKEN }}
23+
24+
- name: setup In-tree
25+
run: tests/script/github_action_helper.sh deploy_rook_ceph_with_intree
26+
27+
- name: Test migration of all PVC
28+
run: tests/script/github_action_helper.sh test_intree_migration_for_all_pvc
29+
30+
- name: setup tmate session for debugging when event is PR
31+
if: failure()
32+
uses: mxschmitt/action-tmate@v3
33+
timeout-minutes: 300
34+
35+
In-tree-Mirgate-single-PVC:
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: checkout
39+
uses: actions/checkout@v2
40+
with:
41+
fetch-depth: 0
42+
43+
- name: setup cluster resources
44+
uses: ./.github/workflows/setup-cluster-resources
45+
with:
46+
github-token: ${{ secrets.GITHUB_TOKEN }}
47+
48+
- name: setup In-tree
49+
run: tests/script/github_action_helper.sh deploy_rook_ceph_with_intree
50+
51+
- name: Test migration of all PVC
52+
run: tests/script/github_action_helper.sh test_intree_migration_for_single_pvc
53+
54+
- name: setup tmate session for debugging when event is PR
55+
if: failure()
56+
uses: mxschmitt/action-tmate@v3
57+
timeout-minutes: 300

‎.github/workflows/setup-cluster-resources/action.yaml

+1-5
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ runs:
1818

1919
- name: print k8s cluster status
2020
shell: bash --noprofile --norc -eo pipefail -x {0}
21-
run: |
21+
run: |
2222
minikube status
2323
kubectl get nodes
2424
@@ -31,7 +31,3 @@ runs:
3131
run: |
3232
sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64
3333
sudo chmod +x /usr/local/bin/yq
34-
35-
- name: setup flex
36-
shell: bash --noprofile --norc -eo pipefail -x {0}
37-
run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex

‎tests/script/github_action_helper.sh

+113-4
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ deploy_rook_ceph_with_flex() {
2626
kubectl create -f manifests/migrator.yaml
2727
# wait_for_pod_to_be_ready_state check for osd pod to in ready state
2828
wait_for_osd_pod_to_be_ready_state
29-
# wait_for_pod_to_be_ready_state check for toolbox pod to in ready state
30-
wait_for_toolboxpod_to_be_ready_state
29+
# wait_for_migrator_pod_to_be_ready_state check for migrator pod to in ready state
30+
wait_for_migrator_pod_to_be_ready_state
3131
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml
3232
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml
3333
# creating sample application pod, writing some data into pod and deletes the pod
@@ -72,6 +72,11 @@ test_flex_migration_for_single_pvc(){
7272
MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
7373
kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
7474
kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --pvc=rbd-pvc --pvc-ns=default --destination-sc=csi-rook-ceph-block"
75+
exit_code_of_last_command=$?
76+
if [ $exit_code_of_last_command -ne 0 ]; then
77+
echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
78+
exit 1
79+
fi
7580
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml
7681
wait_for_sample_pod_to_be_ready_state
7782
verify_file_data_and_file_data
@@ -103,8 +108,8 @@ wait_for_osd_pod_to_be_ready_state() {
103108
EOF
104109
}
105110

106-
# wait_for_pod_to_be_ready_state check for osd pod to in ready state
107-
wait_for_toolboxpod_to_be_ready_state() {
111+
# wait_for_migrator_pod_to_be_ready_state check for migrator pod to in ready state
112+
wait_for_migrator_pod_to_be_ready_state() {
108113
timeout 200 bash <<-'EOF'
109114
until [ $(kubectl get pod -l app=rook-ceph-migrator -n rook-ceph -o jsonpath='{.items[*].metadata.name}' -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 1 ]; do
110115
echo "waiting for the toolbox pods to be in ready state"
@@ -122,6 +127,110 @@ wait_for_sample_pod_to_be_ready_state() {
122127
EOF
123128
}
124129

130+
deploy_rook_ceph_with_intree() {
131+
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/common.yaml
132+
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/crds.yaml
133+
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/operator.yaml
134+
wget https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/cluster-test.yaml
135+
sed -i "s|#deviceFilter:|deviceFilter: $(lsblk|awk '/14G/ {print $1}'| head -1)|g" cluster-test.yaml
136+
kubectl create -f cluster-test.yaml
137+
wait_for_osd_pod_to_be_ready_state
138+
kubectl create -f manifests/migrator.yaml
139+
wait_for_toolboxpod_to_be_ready_state
140+
kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/pool-test.yaml
141+
142+
kubectl patch storageclass standard -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'
143+
sudo sed -i 's/image: k8s.gcr.io\/kube-controller-manager:v1.22.2/image: gcr.io\/google_containers\/hyperkube:v1.16.3/g' /etc/kubernetes/manifests/kube-controller-manager.yaml
144+
kubectl create -f manifests/migrator.yaml
145+
# wait_for_pod_to_be_ready_state check for migrator pod to in ready state
146+
wait_for_migrator_pod_to_be_ready_state
147+
MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
148+
ADMIN_KEY=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-key client.admin")
149+
AKEY=$(echo "$ADMIN_KEY"|base64)
150+
cat <<EOF | kubectl create -f -
151+
apiVersion: v1
152+
kind: Secret
153+
metadata:
154+
name: ceph-secret
155+
namespace: kube-system
156+
data:
157+
key: ${AKEY}
158+
type: kubernetes.io/rbd
159+
EOF
160+
kubectl -n kube-system get secret ceph-secret
161+
kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-or-create client.replicapool mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=kube' -o ceph.client.replicapool.keyring"
162+
USER_KEY=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-key client.replicapool")
163+
UKEY=$(echo "$USER_KEY"|base64)
164+
cat <<EOF | kubectl create -f -
165+
apiVersion: v1
166+
kind: Secret
167+
metadata:
168+
name: ceph-user-secret
169+
namespace: default
170+
data:
171+
key: ${UKEY}
172+
type: kubernetes.io/rbd
173+
EOF
174+
kubectl get secret ceph-user-secret
175+
MON_STAT=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph mon stat")
176+
MON_IP=$(echo "$MON_STAT" | awk -F "v1:" '{print $2}' | cut -d/ -f1)
177+
cat <<EOF | kubectl create -f -
178+
apiVersion: storage.k8s.io/v1
179+
kind: StorageClass
180+
metadata:
181+
name: dynamic
182+
annotations:
183+
storageclass.beta.kubernetes.io/is-default-class: "true"
184+
provisioner: kubernetes.io/rbd
185+
parameters:
186+
monitors: ${MON_IP}
187+
adminId: admin
188+
adminSecretName: ceph-secret
189+
adminSecretNamespace: kube-system
190+
pool: replicapool
191+
userId: replicapool
192+
userSecretName: ceph-user-secret
193+
EOF
194+
cat <<EOF | kubectl create -f -
195+
kind: PersistentVolumeClaim
196+
apiVersion: v1
197+
metadata:
198+
name: ceph-claim
199+
spec:
200+
accessModes:
201+
- ReadWriteOnce
202+
resources:
203+
requests:
204+
storage: 1Gi
205+
EOF
206+
207+
create_csi_resources
208+
}
209+
210+
test_intree_migration_for_all_pvc(){
211+
go build -o pv-migrator
212+
MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
213+
kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
214+
kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --source-sc=dynamic --destination-sc=csi-rook-ceph-block"
215+
exit_code_of_last_command=$?
216+
if [ $exit_code_of_last_command -ne 0 ]; then
217+
echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
218+
exit 1
219+
fi
220+
}
221+
222+
test_intree_migration_for_single_pvc(){
223+
go build -o pv-migrator
224+
MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
225+
kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
226+
kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --pvc=ceph-claim --pvc-ns=default --destination-sc=csi-rook-ceph-block"
227+
exit_code_of_last_command=$?
228+
if [ $exit_code_of_last_command -ne 0 ]; then
229+
echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
230+
exit 1
231+
fi
232+
}
233+
125234
########
126235
# MAIN #
127236
########

0 commit comments

Comments
 (0)