Feature: Google Cloud Engine

README.md

@@ -45,8 +45,9 @@ kind load docker-image multinet-prysm --name kind
 *Customize `multinet-cluster/values.yaml`*
 
 ```
-helm install ./multinet-cluster  
+helm install $DEPLOYMENT_NAME ./multinet-cluster  
 ```
+where `$DEPLOYMENT_NAME` is your name choice
 
 ### Using a development slot
 
@@ -83,6 +84,36 @@ docker run -it --rm -v nimbus-source:/tmp/nimbus ubuntu bash
 ```
 Using this trick you could even attach via visual studio code and SSH integration.
 
+## Google Cloud Engine (GKE) Kubernetes
+In order to make `ReadWriteMany` on `common-data.yaml` and `deposits-storage.yml` you need to create NFS within region.
+To do this you must create 2 compute disks. One for common-data and one for deposits-storage
+Lets assume that you are using europe-west4-a for your GKE.
+Example using gcloud sdk:
+`gcloud compute disks create --size=200GB --zone=europe-west4-a nfs-data`
+`gcloud compute disks create --size=200GB --zone=europe-west4-a nfs-deposit`
+Notice that `nfs-disk` must match helm charts name, so in order to experiment with rename you must also rename it there.
+
+After creating disk you must deploy nfs using kubectl.
+Example from root of this repo:
+`./nfs/reapply.sh`
+
+Files should be mounted to `/data/$DIR` where `$DIR` is with consecutive: 
+`data` for common-data.yml
+`depostits` for deposits-storage.yml
+
+In order to check whats going on on disk itself do:
+`kubectl exec -it $NFS_SERVER_POD_NAME bash`
+
+To check `$NFS_SERVER_POD_NAME` do `kubectl get pods`
+
+When you want to expose eth2stats you need to export external ips:
+https://kubernetes.io/docs/tutorials/stateless-application/expose-external-ip-address/
+
+Example of exposure:
+`kubectl expose deployment eth2stats-server --type=LoadBalancer --name=svrbalancer`
+`kubectl expose deployment eth2stats-dashboard --type=LoadBalancer --name=dashbalancer`
+
+What is problematic is the config of ethstats.
 
 # License
 

multinet-cluster/templates/common-data-claim.yaml

@@ -5,9 +5,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteMany
+  storageClassName: slow
   resources:
     requests:
-      storage: 5Gi
-  selector:
-    matchLabels:
-      name: common-data
+      storage: 50Gi

multinet-cluster/templates/common-data.yaml

@@ -6,14 +6,13 @@ metadata:
     name: common-data
 spec:
   capacity:
-    storage: 5Gi
-  volumeMode: Filesystem
+    storage: 50Gi
   accessModes:
     - ReadWriteMany
   persistentVolumeReclaimPolicy: Retain
   storageClassName: slow
   mountOptions:
     - hard
   nfs:
-    path: /data/deposits
-    server: 172.17.0.3
+    path: /
+    server: nfs-server-data.default.svc.cluster.local

multinet-cluster/templates/deposits-storage-claim.yaml

@@ -5,9 +5,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteMany
+  storageClassName: slow
   resources:
     requests:
-      storage: 5Gi
-  selector:
-    matchLabels:
-      name: deposits-storage
+      storage: 50Gi

multinet-cluster/templates/deposits-storage.yaml

@@ -6,14 +6,13 @@ metadata:
     name: deposits-storage
 spec:
   capacity:
-    storage: 5Gi
-  volumeMode: Filesystem
+    storage: 50Gi
   accessModes:
     - ReadWriteMany
   persistentVolumeReclaimPolicy: Retain
   storageClassName: slow
   mountOptions:
     - hard
   nfs:
-    path: /data/data
-    server: 172.17.0.2
+    path: /
+    server: nfs-server-deposit.default.svc.cluster.local

multinet-cluster/templates/eth2stats-dashboard-cfg.yaml

@@ -9,10 +9,10 @@ data:
             {
                 "name": "multinet cluster",
                 "path": "/cluster",
-                "joinURL": "http://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_PORT }}/participate",
-                "HTTP_API": "http://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_PORT }}/api",
-                "WS_API": "ws://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_PORT }}/ws",
-                "SERVER_ADDR": "{{ .Values.ETH2STATS_DASHBOARD_HOST }}:{{ .Values.ETH2STATS_DASHBOARD_PORT }}"
+                "joinURL": "http://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_EXPOSURE_PORT }}/participate",
+                "HTTP_API": "http://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_EXPOSURE_PORT }}/api",
+                "WS_API": "ws://{{ .Values.ETH2STATS_SERVER_HOST }}:{{ .Values.ETH2STATS_SERVER_EXPOSURE_PORT }}/ws",
+                "SERVER_ADDR": "{{ .Values.ETH2STATS_DASHBOARD_HOST }}:{{ .Values.ETH2STATS_DASHBOARD_EXPOSURE_PORT }}"
             }
         ]
     }

multinet-cluster/templates/eth2stats-server-cfg.yaml

@@ -6,6 +6,7 @@ data:
   payload: >
     http:
       # Enable/disable development CORS
+      port: 8080
       dev-cors: true
       # Allowed hosts for HTTP API development CORS
       dev-cors-host: "*"

multinet-cluster/templates/lighthouse-dev.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: lighthouse-dev
 spec:
   replicas: {{ .Values.LIGHTHOUSE_DEV_NODES }}
+  serviceName: lighthouse-dev
   selector:
     matchLabels:
       tier: lighthouse-dev

multinet-cluster/templates/lighthouse.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: lighthouse
 spec:
   replicas: {{ .Values.LIGHTHOUSE_NODES }}
+  serviceName: lighthouse
   selector:
     matchLabels:
       tier: lighthouse
@@ -20,9 +21,11 @@ spec:
         - name: deposits
           persistentVolumeClaim:
             claimName: deposits-storage
+            readOnly: false
         - name: common-data
           persistentVolumeClaim:
             claimName: common-data
+            readOnly: false
       containers:
         - name: lighthouse
           image: eth2clients/multinet-lighthouse

multinet-cluster/templates/nimbus-dev.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: nimbus-dev
 spec:
   replicas: {{ .Values.NIMBUS_DEV_NODES }}
+  serviceName: nimbus-dev
   selector:
     matchLabels:
       tier: nimbus-dev

multinet-cluster/templates/nimbus.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: nimbus
 spec:
   replicas: {{ .Values.NIMBUS_NODES }}
+  serviceName: nimbus
   selector:
     matchLabels:
       tier: nimbus
@@ -76,13 +77,6 @@ spec:
           args:
           - if [[ $MULTINET_POD_NAME == "nimbus-0" ]]; then ./make_genesis.sh; else ./wait_for.sh -t 0 bootstrap:50000 -- echo "Done"; fi
           # inject environment variables
-          env:
-            # We use this to dedect with are 0, and run only on 0 node
-            - name: MULTINET_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            # inject environment variables
           env:
             - name: MULTINET_POD_NAME
               valueFrom:

multinet-cluster/templates/prysm-dev.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: prysm-dev
 spec:
   replicas: {{ .Values.PRYSM_DEV_NODES }}
+  serviceName: prysm-dev
   selector:
     matchLabels:
       tier: prysm-dev

multinet-cluster/templates/prysm.yaml

@@ -7,6 +7,7 @@ metadata:
     tier: prysm
 spec:
   replicas: {{ .Values.PRYSM_NODES }}
+  serviceName: prysm
   selector:
     matchLabels:
       tier: prysm

multinet-cluster/values.yaml

@@ -8,30 +8,31 @@ NIMBUS_SOURCE_PATH: /run/desktop/mnt/host/e/tmp/multinet/nbc
 NIMBUS_VALIDATORS: 64
 NIMBUS_LOG_LEVEL: INFO;TRACE:switch,multistream
 
-LIGHTHOUSE_NODES: 8
+LIGHTHOUSE_NODES: 2
 # external nodes we manually manage
 LIGHTHOUSE_DEV_NODES: 0
 # the path of the source code for the manually managed nodes
 # point it to an already existing source tree or
 # you will find the oritinal source in the container you can copy also
 LIGHTHOUSE_SOURCE_PATH: /run/desktop/mnt/host/e/tmp/multinet/lh
-LIGHTHOUSE_DEV_NODES: 0
 LIGHTHOUSE_VALIDATORS: 64
 
-PRYSM_NODES: 8
+PRYSM_NODES: 2
 # external nodes we manually manage
 PRYSM_DEV_NODES: 0
 # the path of the source code for the manually managed nodes
 # point it to an already existing source tree or
 # you will find the oritinal source in the container you can copy also
 PRYSM_SOURCE_PATH: /run/desktop/mnt/host/e/tmp/multinet/pry
-PRYSM_DEV_NODES: 0
 PRYSM_VALIDATORS: 64
 
 # DEV_NODES should use the <client>-dev-N folders for deposits details
 # see respective run_<client> scripts for usage
 
-ETH2STATS_DASHBOARD_HOST: localhost
+ETH2STATS_DASHBOARD_HOST: 34.90.31.103
 ETH2STATS_DASHBOARD_PORT: 30000
-ETH2STATS_SERVER_HOST: localhost
+ETH2STATS_SERVER_HOST: 34.91.196.32
 ETH2STATS_SERVER_PORT: 30001
+# Use it if you expose in GKE
+ETH2STATS_DASHBOARD_EXPOSURE_PORT: 80
+ETH2STATS_SERVER_EXPOSURE_PORT: 8080

nfs/deployment-data.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nfs-server-data
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      role: nfs-server-data
+  template:
+    metadata:
+      labels:
+        role: nfs-server-data
+    spec:
+      containers:
+        - name: nfs-server
+          image: gcr.io/google_containers/volume-nfs:0.8
+          ports:
+            - name: nfs
+              containerPort: 2049
+            - name: mountd
+              containerPort: 20048
+            - name: rpcbind
+              containerPort: 111
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /data
+              name: nfs-data
+
+      volumes:
+        - name: nfs-data
+          gcePersistentDisk:
+            pdName: nfs-data
+            fsType: ext4

nfs/deployment-deposit.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nfs-server-deposit
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      role: nfs-server-deposit
+  template:
+    metadata:
+      labels:
+        role: nfs-server-deposit
+    spec:
+      containers:
+        - name: nfs-server
+          image: gcr.io/google_containers/volume-nfs:0.8
+          ports:
+            - name: nfs
+              containerPort: 2049
+            - name: mountd
+              containerPort: 20048
+            - name: rpcbind
+              containerPort: 111
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /deposit
+              name: nfs-deposit
+
+      volumes:
+        - name: nfs-deposit
+          gcePersistentDisk:
+            pdName: nfs-deposit
+            fsType: ext4

nfs/reapply.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#WARNING: run this from root of the project
+
+# delete old nfs deployment
+kubectl delete -f $PWD/nfs/deployment-data.yaml
+kubectl delete -f $PWD/nfs/deployment-deposit.yaml
+kubectl delete -f $PWD/nfs/service-data.yaml
+kubectl delete -f $PWD/nfs/service-deposit.yaml
+
+#apply new nfs deployment
+kubectl apply -f $PWD/nfs/deployment-data.yaml
+kubectl apply -f $PWD/nfs/deployment-deposit.yaml
+kubectl apply -f $PWD/nfs/service-data.yaml
+kubectl apply -f $PWD/nfs/service-deposit.yaml
+

nfs/service-data.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nfs-server-data
+spec:
+  ports:
+    - name: nfs
+      port: 2049
+    - name: mountd
+      port: 20048
+    - name: rpcbind
+      port: 111
+  selector:
+    role: nfs-server-data

nfs/service-deposit.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nfs-server-deposit
+spec:
+  ports:
+    - name: nfs
+      port: 2049
+    - name: mountd
+      port: 20048
+    - name: rpcbind
+      port: 111
+  selector:
+    role: nfs-server-deposit