clusterversion_v1_nodeport.yaml

apiVersion: tenancy.x-k8s.io/v1alpha1
kind: ClusterVersion
metadata:
  labels:
    controller-tools.k8s.io: "1.0"
  name: cv-sample-np
spec:
  # a statefulset and service bundle for etcd
  etcd:
    metadata:
      name: etcd 
    statefulset:
      apiVersion: apps/v1
      kind: StatefulSet
      metadata:
        name: etcd
      spec:
        replicas: 1
        revisionHistoryLimit: 10
        serviceName: etcd
        selector:
          matchLabels:
            component-name: etcd
        # etcd will not be updated, unless it is deleted
        updateStrategy:
          type: OnDelete
        template:
          metadata:
            labels:
              component-name: etcd
          spec:
            subdomain: etcd
            containers:
            - name: etcd
              image: virtualcluster/etcd-v3.4.0
              imagePullPolicy: Always
              command: 
              - etcd
              # pass the pod name(hostname) to container for composing the advertise-urls args
              env:
              - name: HOSTNAME
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.name 
              args:
              - --name=$(HOSTNAME)
              - --trusted-ca-file=/etc/kubernetes/pki/root/tls.crt
              - --client-cert-auth 
              - --cert-file=/etc/kubernetes/pki/etcd/tls.crt
              - --key-file=/etc/kubernetes/pki/etcd/tls.key
              - --peer-client-cert-auth 
              - --peer-trusted-ca-file=/etc/kubernetes/pki/root/tls.crt
              - --peer-cert-file=/etc/kubernetes/pki/etcd/tls.crt
              - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key
              - --listen-peer-urls=https://0.0.0.0:2380 
              - --listen-client-urls=https://0.0.0.0:2379
              - --initial-advertise-peer-urls=https://$(HOSTNAME).etcd:2380
              # we use a headless service to encapsulate each pod
              - --advertise-client-urls=https://$(HOSTNAME).etcd:2379
              - --initial-cluster-state=new
              - --initial-cluster-token=vc-etcd
              - --data-dir=/var/lib/etcd/data
              # --initial-cluster option will be set during runtime based on the number of replicas
              livenessProbe:
                exec:
                  command: 
                  - sh
                  - -c
                  - ETCDCTL_API=3 etcdctl --endpoints=https://etcd:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health
                failureThreshold: 8
                initialDelaySeconds: 60
                timeoutSeconds: 15
              readinessProbe:
                exec:
                  command: 
                  - sh
                  - -c
                  - ETCDCTL_API=3 etcdctl --endpoints=https://etcd:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health
                failureThreshold: 8
                initialDelaySeconds: 15
                periodSeconds: 2
                timeoutSeconds: 15
              volumeMounts:
              - mountPath: /etc/kubernetes/pki/etcd
                name: etcd-ca
                readOnly: true
              - mountPath: /etc/kubernetes/pki/root
                name: root-ca
                readOnly: true
            volumes: 
            - name: etcd-ca
              secret:
                defaultMode: 420
                secretName: etcd-ca
            - name: root-ca
              secret:
                defaultMode: 420
                secretName: root-ca
    # etcd will be accessed only by apiserver from inside the cluster, so we use a headless service to 
    # encapsulate it
    service:
      apiVersion: v1
      kind: Service
      metadata:
        name: etcd
        annotations:
          service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
      spec:
        type: ClusterIP
        clusterIP: None
        selector:
          component-name: etcd
  # a statefulset and service bundle for apiserver
  apiServer:
    metadata:
      name: apiserver
    statefulset:
      apiVersion: apps/v1
      kind: StatefulSet
      metadata:
        name: apiserver
      spec:
        replicas: 1
        revisionHistoryLimit: 10
        serviceName: apiserver-svc
        selector:
          matchLabels:
            component-name: apiserver
        # apiserver will not be updated, unless it is deleted
        updateStrategy:
          type: OnDelete
        template:
          metadata:
            labels:
              component-name: apiserver
          spec:
            hostname: apiserver
            subdomain: apiserver-svc
            containers:
            - name: apiserver
              image: virtualcluster/apiserver-v1.16.2
              imagePullPolicy: Always
              command:
              - kube-apiserver
              args:
              - --bind-address=0.0.0.0
              - --allow-privileged=true
              - --anonymous-auth=true
              - --client-ca-file=/etc/kubernetes/pki/root/tls.crt
              - --tls-cert-file=/etc/kubernetes/pki/apiserver/tls.crt
              - --tls-private-key-file=/etc/kubernetes/pki/apiserver/tls.key
              - --kubelet-https=true
              - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver/tls.crt
              - --kubelet-client-key=/etc/kubernetes/pki/apiserver/tls.key
              - --enable-bootstrap-token-auth=true
              - --etcd-servers=https://etcd-0.etcd:2379
              - --etcd-cafile=/etc/kubernetes/pki/root/tls.crt
              - --etcd-certfile=/etc/kubernetes/pki/apiserver/tls.crt
              - --etcd-keyfile=/etc/kubernetes/pki/apiserver/tls.key
              - --service-account-key-file=/etc/kubernetes/pki/service-account/tls.key
              - --service-cluster-ip-range=10.32.0.0/16
              - --service-node-port-range=30000-32767
              - --authorization-mode=Node,RBAC
              - --runtime-config=api/all
              - --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
              - --apiserver-count=1
              - --endpoint-reconciler-type=master-count
              - --v=2
              ports:
              - containerPort: 6443
                protocol: TCP
                name: api
              livenessProbe:
                # since we set anonymous-auth to false, we use tcp instead of https
                tcpSocket:
                  port: 6443
                failureThreshold: 8
                initialDelaySeconds: 15
                periodSeconds: 10
                timeoutSeconds: 15
              readinessProbe:
                httpGet:
                  port: 6443
                  path: /healthz
                  scheme: HTTPS
                failureThreshold: 8
                initialDelaySeconds: 5
                periodSeconds: 2
                timeoutSeconds: 30
              volumeMounts:
              - mountPath: /etc/kubernetes/pki/apiserver
                name: apiserver-ca
                readOnly: true
              - mountPath: /etc/kubernetes/pki/root
                name: root-ca
                readOnly: true
              - mountPath: /etc/kubernetes/pki/service-account
                name: serviceaccount-rsa
                readOnly: true
            securityContext: {}
            terminationGracePeriodSeconds: 30
            dnsConfig:
              searches:
              - cluster.local
            volumes:
            - name: apiserver-ca
              secret:
                defaultMode: 420
                secretName: apiserver-ca
            - name: root-ca
              secret:
                defaultMode: 420
                secretName: root-ca
            - name: serviceaccount-rsa
              secret:
                defaultMode: 420
                secretName: serviceaccount-rsa
    service:
      apiVersion: apps/v1
      kind: Service
      metadata:
        name: apiserver-svc
      spec:
        selector:
          component-name: apiserver
        type: NodePort
        ports:
        - port: 6443
          protocol: TCP
          targetPort: api
  # a statefulset and service bundle for controller-manager
  controllerManager:
    apiVersion: apps/v1
    kind: StatefulSet
    metadata:
      name: controller-manager
    # statefuleset template for controller-manager
    statefulset:  
      apiVersion: apps/v1
      kind: StatefulSet
      metadata:
        name: controller-manager
      spec:
        serviceName: controller-manager-svc
        replicas: 1
        selector:
          matchLabels:
            component-name: controller-manager
        updateStrategy:
          type: OnDelete
        template:
          metadata:
            labels:
              component-name: controller-manager
          spec:
            containers:
            - name: controller-manager
              image: virtualcluster/controller-manager-v1.16.2
              imagePullPolicy: Always
              command:
              - kube-controller-manager
              args:
              - --bind-address=0.0.0.0
              - --cluster-cidr=10.200.0.0/16
              - --cluster-signing-cert-file=/etc/kubernetes/pki/root/tls.crt
              - --cluster-signing-key-file=/etc/kubernetes/pki/root/tls.key
              - --kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
              - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
              - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig
              # control plane contains only one instance for now
              - --leader-elect=false
              - --root-ca-file=/etc/kubernetes/pki/root/tls.crt
              - --service-account-private-key-file=/etc/kubernetes/pki/service-account/tls.key
              - --service-cluster-ip-range=10.32.0.0/24
              - --use-service-account-credentials=true
              - --experimental-cluster-signing-duration=87600h
              - --node-monitor-grace-period=200s
              - --v=2
              livenessProbe:
                httpGet:
                  path: /healthz
                  port: 10252
                  scheme: HTTP
                failureThreshold: 8
                initialDelaySeconds: 15
                periodSeconds: 10
                timeoutSeconds: 15
              readinessProbe:
                httpGet:
                  port: 10252
                  path: /healthz
                  scheme: HTTP
                failureThreshold: 8
                initialDelaySeconds: 15
                periodSeconds: 2
                timeoutSeconds: 15
              volumeMounts:
              - mountPath: /etc/kubernetes/pki/root
                name: root-ca
                readOnly: true
              - mountPath: /etc/kubernetes/pki/service-account
                name: serviceaccount-rsa
                readOnly: true
              - mountPath: /etc/kubernetes/kubeconfig
                name: kubeconfig
                readOnly: true
            volumes:
            - name: root-ca
              secret:
                defaultMode: 420
                secretName: root-ca
            - name: serviceaccount-rsa 
              secret:
                defaultMode: 420
                secretName: serviceaccount-rsa
            - name: kubeconfig
              secret:
                defaultMode: 420
                secretName: controller-manager-kubeconfig
    # controller-manager will never be accessed proactively, no need to be exposed