Automate management of required roles and user passwords (#671)

.env

@@ -1 +1,22 @@
 ELK_VERSION=8.0.0
+
+## Passwords for stack users
+#
+
+# User 'elastic' (built-in)
+#
+# Superuser role, full access to cluster management and data indices.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+ELASTIC_PASSWORD='changeme'
+
+# User 'logstash_internal' (custom)
+#
+# The user Logstash uses to connect and send data to Elasticsearch.
+# https://www.elastic.co/guide/en/logstash/current/ls-security.html
+LOGSTASH_INTERNAL_PASSWORD='changeme'
+
+# User 'kibana_system' (built-in)
+#
+# The user Kibana uses to connect and communicate with Elasticsearch.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+KIBANA_SYSTEM_PASSWORD='changeme'

.github/workflows/ci.yml

@@ -8,8 +8,8 @@ on:
 
 jobs:
 
-  test-compose:
-    name: 'Test suite: Compose'
+  test:
+    name: Test suite
     # List of supported runners:
     # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources
     runs-on: ubuntu-latest
@@ -26,13 +26,6 @@ jobs:
       - name: Prepare environment
         run: |
 
-          # Install Linux packages
-          #
-          # List of packages pre-installed in the runner:
-          # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-software
-
-          sudo apt install -y expect
-
           # Enable support for Compose V2
           #
           # Instructions:
@@ -63,24 +56,9 @@ jobs:
       - name: Set password of every built-in user to 'testpasswd'
         run: |
 
-          # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files
-
-          sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf
-          sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml
-          sed -i -e 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' -e 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
-          sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/apm-server/config/apm-server.yml
-          sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/metricbeat/config/metricbeat.yml
-          sed -i 's/\(password:\) '\''changeme'\''/\1 testpasswd/g' extensions/filebeat/config/filebeat.yml
-
-          # Run Elasticsearch and wait for its availability
-
-          docker compose up -d elasticsearch
-          source .github/workflows/scripts/lib/testing.sh
-          poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
-
-          # Set passwords
-
-          .github/workflows/scripts/elasticsearch-setup-passwords.exp
+          sed -i -e 's/\(ELASTIC_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
+                 -e 's/\(LOGSTASH_INTERNAL_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' \
+                 -e 's/\(KIBANA_SYSTEM_PASSWORD=\)'\''changeme'\''/\1testpasswd/g' .env
 
       ##########################################################
       #                                                        #
@@ -100,6 +78,7 @@ jobs:
         if: always()
         run: |
           docker compose ps
+          docker compose logs setup
           docker compose logs elasticsearch
           docker compose logs logstash
           docker compose logs kibana
@@ -162,6 +141,8 @@ jobs:
 
           # Run Enterprise Search and execute tests
 
+          sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
+
           docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up -d enterprise-search
           .github/workflows/scripts/run-tests-enterprise-search.sh
 
@@ -246,85 +227,3 @@ jobs:
           -f extensions/metricbeat/metricbeat-compose.yml
           -f extensions/filebeat/filebeat-compose.yml
           down -v
-
-  test-swarm:
-    name: 'Test suite: Swarm'
-    runs-on: ubuntu-latest
-
-    env:
-      MODE: swarm
-
-    steps:
-      - uses: actions/checkout@v2
-
-      #####################################################
-      #                                                   #
-      # Install all dependencies required by test suites. #
-      #                                                   #
-      #####################################################
-
-      - name: Prepare environment
-        run: |
-
-          # Install Linux packages
-
-          sudo apt install -y expect
-
-          # Enable Swarm mode
-
-          docker swarm init
-
-      ########################################################
-      #                                                      #
-      # Ensure §"Initial setup" of the README remains valid. #
-      #                                                      #
-      ########################################################
-
-      - name: Set password of every built-in user to 'testpasswd'
-        run: |
-
-          # Change password of 'elastic' user from 'changeme' to 'testpasswd' in config files
-
-          sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf
-          sed -i 's/\(elasticsearch.password:\) '\''changeme'\''/\1 testpasswd/g' kibana/config/kibana.yml
-
-          # Run Elasticsearch and wait for its availability
-
-          docker stack deploy -c ./docker-stack.yml elk
-          docker service scale elk_logstash=0 elk_kibana=0
-          source .github/workflows/scripts/lib/testing.sh
-          poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
-
-          # Set passwords
-
-          .github/workflows/scripts/elasticsearch-setup-passwords.exp swarm
-
-      ##########################################################
-      #                                                        #
-      # Test core components: Elasticsearch, Logstash, Kibana. #
-      #                                                        #
-      ##########################################################
-
-      - name:  Run the stack
-        run: docker service scale elk_logstash=1 elk_kibana=1
-
-      - name: Execute core test suite
-        run: .github/workflows/scripts/run-tests-core.sh swarm
-
-      - name: 'debug: Display state and logs (core)'
-        if: always()
-        run: |
-          docker stack services elk
-          docker service logs elk_elasticsearch
-          docker service logs elk_kibana
-          docker service logs elk_logstash
-
-        ##############
-        #            #
-        # Tear down. #
-        #            #
-        ##############
-
-      - name: Terminate all components
-        if: always()
-        run: docker stack rm elk

.github/workflows/scripts/lib/testing.sh

@@ -14,12 +14,7 @@ function err {
 function container_id {
 	local svc=$1
 
-	local label
-	if [[ "${MODE:-}" == "swarm" ]]; then
-		label="com.docker.swarm.service.name=elk_${svc}"
-	else
-		label="com.docker.compose.service=${svc}"
-	fi
+	local label="com.docker.compose.service=${svc}"
 
 	local cid
 
@@ -51,26 +46,11 @@ function container_id {
 
 # Return the IP address at which a service can be reached.
 # In Compose mode, returns the container's IP.
-# In Swarm mode, returns the IP of the node to ensure traffic enters the routing mesh (ingress).
 function service_ip {
 	local svc=$1
 
 	local ip
 
-	if [[ "${MODE:-}" == "swarm" ]]; then
-		#ingress_net="$(docker network inspect ingress --format '{{ .Id }}')"
-		#ip="$(docker service inspect elk_"$svc" --format "{{ range .Endpoint.VirtualIPs }}{{ if eq .NetworkID \"${ingress_net}\" }}{{ .Addr }}{{ end }}{{ end }}" | cut -d/ -f1)"
-		node="$(docker node ls --format '{{ .ID }}')"
-		ip="$(docker node inspect "$node" --format '{{ .Status.Addr }}')"
-		if [ -z "${ip:-}" ]; then
-			err "Node ${node} has no IP address"
-			return 1
-		fi
-
-		echo "$ip"
-		return
-	fi
-
 	local cid
 	cid="$(container_id "$svc")"
 

.github/workflows/update.yml

@@ -71,7 +71,7 @@ jobs:
           # Escape dot characters so sed interprets them as literal dots
           cur_ver="$(echo $cur_ver | sed 's/\./\\./g')"
 
-          for f in .env docker-stack.yml README.md; do
+          for f in .env README.md; do
               sed -i "s/${cur_ver}/${new_ver}/g" "$f"
           done