-
Notifications
You must be signed in to change notification settings - Fork 8
204 lines (181 loc) · 7.44 KB
/
test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
name: Scrapyd-k8s CI
on:
push:
branches:
- main
pull_request:
jobs:
test-unit:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Run tests
run: pytest -vv --color=yes scrapyd_k8s/tests/unit/
test-docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Pull example spider
run: docker pull ghcr.io/q-m/scrapyd-k8s-spider-example
- name: Run tests
run: |
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo; echo "# $test"
# run scrapyd-k8s with test-specific configuration file
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
python -m scrapyd_k8s -c scrapyd_k8s.sample-docker.conf -c "$cfg" &
# wait for scrapyd-k8s to become ready
curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
# run test
pytest -vv --color=yes "$test"
# stop scrapyd-k8s again
kill %1; wait %1 || true
done
test-manifest:
container:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build container
uses: docker/build-push-action@v5
with:
context: .
push: false
load: true
tags: test:latest
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Start minikube
uses: medyagh/setup-minikube@master
- name: Deploy to minikube
run: |
minikube image load test:latest
# already pull image so we don't have to wait for it later
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
# load manifest
sed -i 's/\(imagePullPolicy:\s*\)\w\+/\1Never/' kubernetes.yaml
sed -i 's/\(image:\s*\)ghcr\.io\/q-m\/scrapyd-k8s:/\1test:/' kubernetes.yaml
sed -i 's/\(type:\s*\)ClusterIP/\1NodePort/' kubernetes.yaml
kubectl create -f kubernetes.yaml
# don't start deployment just yet, as we want to run it with test-specific configuration
kubectl scale --replicas=0 deploy/scrapyd-k8s
# add second configuration file for test-specific configuration
kubectl patch deploy scrapyd-k8s --type=json -p='[
{
"op": "add",
"path": "/spec/template/spec/volumes/-",
"value": { "configMap": { "name": "scrapyd-k8s-testcfg" }, "name": "scrapyd-k8s-testcfg" }
},
{
"op": "add",
"path": "/spec/template/spec/containers/0/volumeMounts/-",
"value": { "name": "scrapyd-k8s-testcfg", "mountPath": "/opt/app/scrapyd_k8s.test.conf", "readOnly": true, "subPath": "scrapyd_k8s.test.conf" }
},
{
"op": "replace",
"path": "/spec/template/spec/containers/0/command",
"value": ["python3", "-m", "scrapyd_k8s", "-c", "scrapyd_k8s.conf", "-c", "scrapyd_k8s.test.conf"]
}
]'
- name: Run tests
run: |
# setup for in-cluster k8s
# for each integration test file
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo; echo "# $test"
# run scrapyd-k8s with test-specific configuration file, run k8s patch if available
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
kubectl create cm scrapyd-k8s-testcfg --from-file=scrapyd_k8s.test.conf="$cfg"
k8sconfig=`echo "$test" | sed 's/\.py$/\.k8s.sh/'`
[ -x "$k8sconfig" ] && "$k8sconfig" up
kubectl scale --replicas=1 deploy/scrapyd-k8s
# wait for scrapyd-k8s to become ready
kubectl wait --for=condition=Available deploy/scrapyd-k8s --timeout=60s
curl -s --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
# run test
TEST_WITH_K8S=1 \
TEST_BASE_URL=`minikube service scrapyd-k8s --url` \
TEST_MAX_WAIT=60 \
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
pytest -vv --color=yes "$test"
# delete al jobs to start with a clean slate next time
kubectl delete job --all
# stop scrapyd-k8s and delete test-specific configmap
kubectl scale --replicas=0 deploy/scrapyd-k8s
kubectl wait --for=delete pod -l app.kubernetes.io/name=scrapyd-k8s --timeout=90s
kubectl delete cm scrapyd-k8s-testcfg --wait
[ -x "$k8sconfig" ] && "$k8sconfig" down
done
test-k8s:
container:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip'
- name: Install dependencies
run: |
pip install -r requirements.txt
pip install -r requirements-test.txt
- name: Start minikube
uses: medyagh/setup-minikube@master
- name: Prepare Kubernetes environment
run: |
kubectl create secret generic example-env-secret --from-literal=FOO_1=bar
kubectl create configmap example-env-configmap --from-literal=FOO_2=baz
# already pull image so we don't have to wait for it later
minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
- name: Run tests
run: |
for test in scrapyd_k8s/tests/integration/test_*.py; do
echo "# $test"
# run scrapyd-k8s with test-specific configuration file
cfg=`echo "$test" | sed 's/\.py$/.conf/'`
[ -e "$cfg" ] || cfg=/dev/null
python -m scrapyd_k8s -c scrapyd_k8s.sample-k8s.conf -c "$cfg" &
# wait for scrapyd-k8s to become ready
curl -s --retry 30 --retry-delay 1 --retry-all-errors http://localhost:6800/daemonstatus.json
# run test
TEST_WITH_K8S=1 \
TEST_MAX_WAIT=60 \
TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
pytest -vv --color=yes "$test"
# stop scrapyd-k8s again
kill %1; wait %1 || true
done