Skip to content

Commit 39d2ca5

Browse files
hodgesrmhodgesrm
committed
Demo changes
1 parent 109e758 commit 39d2ca5

6 files changed

+335
-5
lines changed

‎developer-tricks-ch-on-k8s/basic-00-generic.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# ClickHouse using default storage class.
2+
apiVersion: "clickhouse.altinity.com/v1"
3+
kind: "ClickHouseInstallation"
4+
metadata:
5+
name: "basic"
6+
spec:
7+
configuration:
8+
clusters:
9+
- name: "cl"
10+
layout:
11+
replicasCount: 1
12+
shardsCount: 1
13+
templates:
14+
volumeClaimTemplate: storage
15+
podTemplate: replica
16+
zookeeper:
17+
nodes:
18+
- host: zookeeper
19+
port: 2181
20+
templates:
21+
podTemplates:
22+
- name: replica
23+
spec:
24+
containers:
25+
- name: clickhouse
26+
image: altinity/clickhouse-server:23.8.8.21.altinitystable
27+
volumeClaimTemplates:
28+
- name: storage
29+
reclaimPolicy: Retain
30+
spec:
31+
accessModes:
32+
- ReadWriteOnce
33+
resources:
34+
requests:
35+
storage: 50Gi

‎developer-tricks-ch-on-k8s/basic-01-generic.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# ClickHouse using default storage class.
2+
apiVersion: "clickhouse.altinity.com/v1"
3+
kind: "ClickHouseInstallation"
4+
metadata:
5+
name: "basic"
6+
spec:
7+
configuration:
8+
clusters:
9+
- name: "cl"
10+
layout:
11+
replicasCount: 2
12+
shardsCount: 2
13+
templates:
14+
volumeClaimTemplate: storage
15+
podTemplate: replica
16+
zookeeper:
17+
nodes:
18+
- host: zookeeper
19+
port: 2181
20+
templates:
21+
podTemplates:
22+
- name: replica
23+
spec:
24+
containers:
25+
- name: clickhouse
26+
image: altinity/clickhouse-server:23.8.8.21.altinitystable
27+
volumeClaimTemplates:
28+
- name: storage
29+
reclaimPolicy: Retain
30+
spec:
31+
accessModes:
32+
- ReadWriteOnce
33+
resources:
34+
requests:
35+
storage: 50Gi

‎developer-tricks-ch-on-k8s/demo-00-generic.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,20 @@
22
apiVersion: "clickhouse.altinity.com/v1"
33
kind: "ClickHouseInstallation"
44
metadata:
5-
name: "demo"
5+
name: "basic"
66
spec:
77
configuration:
88
clusters:
99
- name: "cl"
1010
layout:
1111
replicasCount: 1
12-
shardsCount: 2
12+
shardsCount: 1
1313
templates:
1414
volumeClaimTemplate: storage
1515
podTemplate: replica
1616
zookeeper:
1717
nodes:
18-
- host: zookeeper.zoo1ns
18+
- host: keeper-keeper
1919
port: 2181
2020
templates:
2121
podTemplates:

‎developer-tricks-ch-on-k8s/demo-01-aws-m6i-large.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ spec:
1515
podTemplate: replica
1616
zookeeper:
1717
nodes:
18-
- host: zookeeper.zoo1ns
18+
- host: zookeeper
1919
port: 2181
2020
templates:
2121
podTemplates:

‎developer-tricks-ch-on-k8s/demo-02-aws-m6i-xlarge.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ spec:
1515
podTemplate: replica
1616
zookeeper:
1717
nodes:
18-
- host: zookeeper.zoo1ns
18+
- host: zookeeper
1919
port: 2181
2020
templates:
2121
podTemplates:

‎developer-tricks-ch-on-k8s/zookeeper-1-node.yaml

+260
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
# Setup Service to provide access to Zookeeper for clients
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
# DNS would be like zookeeper.zoons
6+
name: zookeeper
7+
labels:
8+
app: zookeeper
9+
spec:
10+
ports:
11+
- port: 2181
12+
name: client
13+
- port: 7000
14+
name: prometheus
15+
selector:
16+
app: zookeeper
17+
what: node
18+
---
19+
# Setup Headless Service for StatefulSet
20+
apiVersion: v1
21+
kind: Service
22+
metadata:
23+
# DNS would be like zookeeper-0.zookeepers.etc
24+
name: zookeepers
25+
labels:
26+
app: zookeeper
27+
spec:
28+
ports:
29+
- port: 2888
30+
name: server
31+
- port: 3888
32+
name: leader-election
33+
clusterIP: None
34+
selector:
35+
app: zookeeper
36+
what: node
37+
---
38+
# Setup max number of unavailable pods in StatefulSet
39+
apiVersion: policy/v1
40+
kind: PodDisruptionBudget
41+
metadata:
42+
name: zookeeper-pod-disruption-budget
43+
spec:
44+
selector:
45+
matchLabels:
46+
app: zookeeper
47+
maxUnavailable: 1
48+
---
49+
# Setup Zookeeper StatefulSet
50+
# Possible params:
51+
# 1. replicas
52+
# 2. memory
53+
# 3. cpu
54+
# 4. storage
55+
# 5. storageClassName
56+
# 6. user to run app
57+
apiVersion: apps/v1
58+
kind: StatefulSet
59+
metadata:
60+
# nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2
61+
name: zookeeper
62+
labels:
63+
app: zookeeper
64+
spec:
65+
selector:
66+
matchLabels:
67+
app: zookeeper
68+
serviceName: zookeepers
69+
replicas: 1
70+
updateStrategy:
71+
type: RollingUpdate
72+
podManagementPolicy: OrderedReady
73+
template:
74+
metadata:
75+
labels:
76+
app: zookeeper
77+
what: node
78+
annotations:
79+
prometheus.io/port: '7000'
80+
prometheus.io/scrape: 'true'
81+
spec:
82+
affinity:
83+
podAntiAffinity:
84+
requiredDuringSchedulingIgnoredDuringExecution:
85+
- labelSelector:
86+
matchExpressions:
87+
- key: "app"
88+
operator: In
89+
values:
90+
- zookeeper
91+
# TODO think about multi-AZ EKS
92+
# topologyKey: topology.kubernetes.io/zone
93+
topologyKey: "kubernetes.io/hostname"
94+
containers:
95+
- name: kubernetes-zookeeper
96+
imagePullPolicy: IfNotPresent
97+
image: "docker.io/zookeeper:3.8.3"
98+
resources:
99+
requests:
100+
memory: "512M"
101+
cpu: "1"
102+
limits:
103+
memory: "4Gi"
104+
cpu: "2"
105+
ports:
106+
- containerPort: 2181
107+
name: client
108+
- containerPort: 2888
109+
name: server
110+
- containerPort: 3888
111+
name: leader-election
112+
- containerPort: 7000
113+
name: prometheus
114+
env:
115+
- name: SERVERS
116+
value: "1"
117+
118+
# See those links for proper startup settings:
119+
# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
120+
# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
121+
# https://github.com/ClickHouse/ClickHouse/issues/11781
122+
command:
123+
- bash
124+
- -x
125+
- -c
126+
- |
127+
HOST=`hostname -s` &&
128+
DOMAIN=`hostname -d` &&
129+
CLIENT_PORT=2181 &&
130+
SERVER_PORT=2888 &&
131+
ELECTION_PORT=3888 &&
132+
PROMETHEUS_PORT=7000 &&
133+
ZOO_DATA_DIR=/var/lib/zookeeper/data &&
134+
ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog &&
135+
{
136+
echo "clientPort=${CLIENT_PORT}"
137+
echo 'tickTime=2000'
138+
echo 'initLimit=300'
139+
echo 'syncLimit=10'
140+
echo 'maxClientCnxns=2000'
141+
echo 'maxTimeToWaitForEpoch=2000'
142+
echo 'maxSessionTimeout=60000000'
143+
echo "dataDir=${ZOO_DATA_DIR}"
144+
echo "dataLogDir=${ZOO_DATA_LOG_DIR}"
145+
echo 'autopurge.snapRetainCount=10'
146+
echo 'autopurge.purgeInterval=1'
147+
echo 'preAllocSize=131072'
148+
echo 'snapCount=3000000'
149+
echo 'leaderServes=yes'
150+
echo 'standaloneEnabled=false'
151+
echo '4lw.commands.whitelist=*'
152+
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
153+
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
154+
echo "skipACL=true"
155+
echo "fastleader.maxNotificationInterval=10000"
156+
} > /conf/zoo.cfg &&
157+
{
158+
echo "zookeeper.root.logger=CONSOLE"
159+
echo "zookeeper.console.threshold=INFO"
160+
echo "log4j.rootLogger=\${zookeeper.root.logger}"
161+
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender"
162+
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}"
163+
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout"
164+
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"
165+
} > /conf/log4j.properties &&
166+
echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:ActiveProcessorCount=8 -XX:+AlwaysPreTouch -Djute.maxbuffer=8388608 -XX:MaxGCPauseMillis=50"' > /conf/java.env &&
167+
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
168+
NAME=${BASH_REMATCH[1]} &&
169+
ORD=${BASH_REMATCH[2]};
170+
else
171+
echo "Failed to parse name and ordinal of Pod" &&
172+
exit 1;
173+
fi &&
174+
mkdir -pv ${ZOO_DATA_DIR} &&
175+
mkdir -pv ${ZOO_DATA_LOG_DIR} &&
176+
whoami &&
177+
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" &&
178+
export MY_ID=$((ORD+1)) &&
179+
echo $MY_ID > $ZOO_DATA_DIR/myid &&
180+
for (( i=1; i<=$SERVERS; i++ )); do
181+
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
182+
done &&
183+
if [[ $SERVERS -eq 1 ]]; then
184+
echo "group.1=1" >> /conf/zoo.cfg;
185+
else
186+
echo "group.1=1:2:3" >> /conf/zoo.cfg;
187+
fi &&
188+
for (( i=1; i<=$SERVERS; i++ )); do
189+
WEIGHT=1
190+
if [[ $i == 1 ]]; then
191+
WEIGHT=10
192+
fi
193+
echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg;
194+
done &&
195+
zkServer.sh start-foreground
196+
readinessProbe:
197+
exec:
198+
command:
199+
- bash
200+
- -c
201+
- '
202+
IFS=;
203+
MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
204+
while [[ "$MNTR" == "This ZooKeeper instance is not currently serving requests" ]];
205+
do
206+
echo "wait mntr works";
207+
sleep 1;
208+
MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
209+
done;
210+
STATE=$(echo -e $MNTR | grep zk_server_state | cut -d " " -f 2);
211+
if [[ "$STATE" =~ "leader" ]]; then
212+
echo "check leader state";
213+
SYNCED_FOLLOWERS=$(echo -e $MNTR | grep zk_synced_followers | awk -F"[[:space:]]+" "{print \$2}" | cut -d "." -f 1);
214+
if [[ "$SYNCED_FOLLOWERS" != "0" ]]; then
215+
./bin/zkCli.sh ls /;
216+
exit $?;
217+
else
218+
exit 0;
219+
fi;
220+
elif [[ "$STATE" =~ "follower" ]]; then
221+
echo "check follower state";
222+
PEER_STATE=$(echo -e $MNTR | grep zk_peer_state);
223+
if [[ "$PEER_STATE" =~ "following - broadcast" ]]; then
224+
./bin/zkCli.sh ls /;
225+
exit $?;
226+
else
227+
exit 1;
228+
fi;
229+
else
230+
exit 1;
231+
fi
232+
'
233+
initialDelaySeconds: 10
234+
periodSeconds: 60
235+
timeoutSeconds: 60
236+
livenessProbe:
237+
exec:
238+
command:
239+
- bash
240+
- -xc
241+
- 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi'
242+
initialDelaySeconds: 10
243+
periodSeconds: 30
244+
timeoutSeconds: 5
245+
volumeMounts:
246+
- name: datadir-volume
247+
mountPath: /var/lib/zookeeper
248+
# Run as a non-privileged user
249+
securityContext:
250+
runAsUser: 1000
251+
fsGroup: 1000
252+
volumeClaimTemplates:
253+
- metadata:
254+
name: datadir-volume
255+
spec:
256+
accessModes:
257+
- ReadWriteOnce
258+
resources:
259+
requests:
260+
storage: 25Gi

0 commit comments

Comments
 (0)