Skip to content

Commit b9713ea

Browse files
vlerkinvlerkin
committed
add integration tests for scheduler with two different configs
1 parent f4049bc commit b9713ea

File tree

3 files changed

+784
-0
lines changed

3 files changed

+784
-0
lines changed

‎kubernetes-0.yaml

+191
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: scrapyd-k8s
6+
name: scrapyd-k8s
7+
spec:
8+
selector:
9+
matchLabels:
10+
app.kubernetes.io/name: scrapyd-k8s
11+
template:
12+
metadata:
13+
labels:
14+
app.kubernetes.io/name: scrapyd-k8s
15+
spec:
16+
securityContext:
17+
fsGroup: 1000
18+
serviceAccountName: scrapyd-k8s
19+
containers:
20+
- image: scrapyd-k8s-test:latest
21+
imagePullPolicy: Never
22+
name: scrapyd-k8s
23+
ports:
24+
- containerPort: 6800
25+
name: http
26+
protocol: TCP
27+
env:
28+
- name: MY_POD_NAMESPACE
29+
valueFrom:
30+
fieldRef:
31+
fieldPath: metadata.namespace
32+
- name: MY_DEPLOYMENT_NAME
33+
valueFrom:
34+
fieldRef:
35+
fieldPath: metadata.labels['app.kubernetes.io/name']
36+
readinessProbe:
37+
failureThreshold: 3
38+
httpGet:
39+
path: /healthz
40+
port: http
41+
livenessProbe:
42+
failureThreshold: 30
43+
httpGet:
44+
path: /healthz
45+
port: http
46+
resources:
47+
limits:
48+
memory: 128Mi # TODO check
49+
requests:
50+
memory: 64Mi # TODO check
51+
volumeMounts:
52+
- name: scrapyd-k8s-config
53+
mountPath: /opt/app/scrapyd_k8s.conf
54+
readOnly: true
55+
subPath: scrapyd_k8s.conf
56+
#- name: joblogs
57+
# mountPath: /data
58+
# Enable if your spider repository needs a pull secret
59+
# - name: scrapyd-k8s-pull-secret
60+
# mountPath: /opt/app/.docker
61+
# readOnly: true
62+
volumes:
63+
- configMap:
64+
name: scrapyd-k8s-config
65+
name: scrapyd-k8s-config
66+
#- name: joblogs
67+
# persistentVolumeClaim:
68+
# claimName: pv-claim
69+
# Enable if your spider repository needs a pull secret
70+
# - secret:
71+
# secretName: pull-secret
72+
# items:
73+
# - key: .dockerconfigjson
74+
# path: config.json
75+
---
76+
apiVersion: v1
77+
kind: ConfigMap
78+
metadata:
79+
name: scrapyd-k8s-config
80+
labels:
81+
app.kubernetes.io/name: scrapyd-k8s
82+
data:
83+
scrapyd_k8s.conf: |-
84+
[scrapyd]
85+
bind_address = 0.0.0.0
86+
http_port = 6800
87+
88+
repository = scrapyd_k8s.repository.Remote
89+
launcher = scrapyd_k8s.launcher.K8s
90+
91+
namespace = default
92+
93+
max_proc = 0
94+
95+
# This is an example spider that should work out of the box.
96+
# Adapt the spider config to your use-case.
97+
[project.example]
98+
env_secret = spider-example-env
99+
env_config = spider-example-env
100+
repository = ghcr.io/q-m/scrapyd-k8s-spider-example
101+
102+
# It is strongly recomended to set resource requests and limits on production.
103+
# They can be overridden on the project and spider level.
104+
[default.resources]
105+
requests_cpu = 0.2
106+
requests_memory = 0.2G
107+
limits_cpu = 0.8
108+
limits_memory = 0.5G
109+
110+
[joblogs]
111+
logs_dir = /data/joblogs
112+
---
113+
apiVersion: v1
114+
kind: Secret
115+
metadata:
116+
name: spider-example-env
117+
labels:
118+
app.kubernetes.io/name: spider-example
119+
stringData:
120+
FOO_API_KEY: "1234567890abcdef"
121+
#---
122+
#apiVersion: v1
123+
#kind: PersistentVolumeClaim
124+
#metadata:
125+
# name: pv-claim
126+
#spec:
127+
# accessModes:
128+
# - ReadWriteOnce
129+
# resources:
130+
# requests:
131+
# storage: 5Gi
132+
---
133+
apiVersion: v1
134+
kind: ConfigMap
135+
metadata:
136+
name: spider-example-env
137+
labels:
138+
app.kubernetes.io/name: spider-example
139+
data:
140+
BAR_VALUE: "baz"
141+
---
142+
apiVersion: v1
143+
kind: Service
144+
metadata:
145+
name: scrapyd-k8s
146+
labels:
147+
app.kubernetes.io/name: scrapyd-k8s
148+
spec:
149+
type: ClusterIP
150+
ports:
151+
- name: http
152+
port: 6800
153+
protocol: TCP
154+
targetPort: http
155+
selector:
156+
app.kubernetes.io/name: scrapyd-k8s
157+
---
158+
apiVersion: v1
159+
kind: ServiceAccount
160+
metadata:
161+
name: scrapyd-k8s
162+
---
163+
apiVersion: rbac.authorization.k8s.io/v1
164+
kind: Role
165+
metadata:
166+
name: scrapyd-k8s
167+
rules:
168+
- apiGroups: [""]
169+
resources: ["pods"]
170+
verbs: ["get", "list", "watch"]
171+
- apiGroups: [""]
172+
resources: ["pods/exec"]
173+
verbs: ["get"]
174+
- apiGroups: [""]
175+
resources: ["pods/log"]
176+
verbs: ["get"]
177+
- apiGroups: ["batch"]
178+
resources: ["jobs"]
179+
verbs: ["get", "list", "create", "patch", "delete"]
180+
---
181+
apiVersion: rbac.authorization.k8s.io/v1
182+
kind: RoleBinding
183+
metadata:
184+
name: scrapyd-k8s
185+
subjects:
186+
- kind: ServiceAccount
187+
name: scrapyd-k8s
188+
roleRef:
189+
kind: Role
190+
name: scrapyd-k8s
191+
apiGroup: rbac.authorization.k8s.io

‎kubernetes-1.yaml

+191
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: scrapyd-k8s
6+
name: scrapyd-k8s
7+
spec:
8+
selector:
9+
matchLabels:
10+
app.kubernetes.io/name: scrapyd-k8s
11+
template:
12+
metadata:
13+
labels:
14+
app.kubernetes.io/name: scrapyd-k8s
15+
spec:
16+
securityContext:
17+
fsGroup: 1000
18+
serviceAccountName: scrapyd-k8s
19+
containers:
20+
- image: scrapyd-k8s-test:latest
21+
imagePullPolicy: Never
22+
name: scrapyd-k8s
23+
ports:
24+
- containerPort: 6800
25+
name: http
26+
protocol: TCP
27+
env:
28+
- name: MY_POD_NAMESPACE
29+
valueFrom:
30+
fieldRef:
31+
fieldPath: metadata.namespace
32+
- name: MY_DEPLOYMENT_NAME
33+
valueFrom:
34+
fieldRef:
35+
fieldPath: metadata.labels['app.kubernetes.io/name']
36+
readinessProbe:
37+
failureThreshold: 3
38+
httpGet:
39+
path: /healthz
40+
port: http
41+
livenessProbe:
42+
failureThreshold: 30
43+
httpGet:
44+
path: /healthz
45+
port: http
46+
resources:
47+
limits:
48+
memory: 128Mi # TODO check
49+
requests:
50+
memory: 64Mi # TODO check
51+
volumeMounts:
52+
- name: scrapyd-k8s-config
53+
mountPath: /opt/app/scrapyd_k8s.conf
54+
readOnly: true
55+
subPath: scrapyd_k8s.conf
56+
#- name: joblogs
57+
# mountPath: /data
58+
# Enable if your spider repository needs a pull secret
59+
# - name: scrapyd-k8s-pull-secret
60+
# mountPath: /opt/app/.docker
61+
# readOnly: true
62+
volumes:
63+
- configMap:
64+
name: scrapyd-k8s-config
65+
name: scrapyd-k8s-config
66+
#- name: joblogs
67+
# persistentVolumeClaim:
68+
# claimName: pv-claim
69+
# Enable if your spider repository needs a pull secret
70+
# - secret:
71+
# secretName: pull-secret
72+
# items:
73+
# - key: .dockerconfigjson
74+
# path: config.json
75+
---
76+
apiVersion: v1
77+
kind: ConfigMap
78+
metadata:
79+
name: scrapyd-k8s-config
80+
labels:
81+
app.kubernetes.io/name: scrapyd-k8s
82+
data:
83+
scrapyd_k8s.conf: |-
84+
[scrapyd]
85+
bind_address = 0.0.0.0
86+
http_port = 6800
87+
88+
repository = scrapyd_k8s.repository.Remote
89+
launcher = scrapyd_k8s.launcher.K8s
90+
91+
namespace = default
92+
93+
max_proc = 1
94+
95+
# This is an example spider that should work out of the box.
96+
# Adapt the spider config to your use-case.
97+
[project.example]
98+
env_secret = spider-example-env
99+
env_config = spider-example-env
100+
repository = ghcr.io/q-m/scrapyd-k8s-spider-example
101+
102+
# It is strongly recomended to set resource requests and limits on production.
103+
# They can be overridden on the project and spider level.
104+
[default.resources]
105+
requests_cpu = 0.2
106+
requests_memory = 0.2G
107+
limits_cpu = 0.8
108+
limits_memory = 0.5G
109+
110+
[joblogs]
111+
logs_dir = /data/joblogs
112+
---
113+
apiVersion: v1
114+
kind: Secret
115+
metadata:
116+
name: spider-example-env
117+
labels:
118+
app.kubernetes.io/name: spider-example
119+
stringData:
120+
FOO_API_KEY: "1234567890abcdef"
121+
#---
122+
#apiVersion: v1
123+
#kind: PersistentVolumeClaim
124+
#metadata:
125+
# name: pv-claim
126+
#spec:
127+
# accessModes:
128+
# - ReadWriteOnce
129+
# resources:
130+
# requests:
131+
# storage: 5Gi
132+
---
133+
apiVersion: v1
134+
kind: ConfigMap
135+
metadata:
136+
name: spider-example-env
137+
labels:
138+
app.kubernetes.io/name: spider-example
139+
data:
140+
BAR_VALUE: "baz"
141+
---
142+
apiVersion: v1
143+
kind: Service
144+
metadata:
145+
name: scrapyd-k8s
146+
labels:
147+
app.kubernetes.io/name: scrapyd-k8s
148+
spec:
149+
type: ClusterIP
150+
ports:
151+
- name: http
152+
port: 6800
153+
protocol: TCP
154+
targetPort: http
155+
selector:
156+
app.kubernetes.io/name: scrapyd-k8s
157+
---
158+
apiVersion: v1
159+
kind: ServiceAccount
160+
metadata:
161+
name: scrapyd-k8s
162+
---
163+
apiVersion: rbac.authorization.k8s.io/v1
164+
kind: Role
165+
metadata:
166+
name: scrapyd-k8s
167+
rules:
168+
- apiGroups: [""]
169+
resources: ["pods"]
170+
verbs: ["get", "list", "watch"]
171+
- apiGroups: [""]
172+
resources: ["pods/exec"]
173+
verbs: ["get"]
174+
- apiGroups: [""]
175+
resources: ["pods/log"]
176+
verbs: ["get"]
177+
- apiGroups: ["batch"]
178+
resources: ["jobs"]
179+
verbs: ["get", "list", "create", "patch", "delete"]
180+
---
181+
apiVersion: rbac.authorization.k8s.io/v1
182+
kind: RoleBinding
183+
metadata:
184+
name: scrapyd-k8s
185+
subjects:
186+
- kind: ServiceAccount
187+
name: scrapyd-k8s
188+
roleRef:
189+
kind: Role
190+
name: scrapyd-k8s
191+
apiGroup: rbac.authorization.k8s.io

0 commit comments

Comments
 (0)