enable syslog collection for cosmic (#1282)

* enable syslog collection for cosmic

---------

Co-authored-by: Amol Agrawal <[email protected]>

Author: pfrcks

build/linux/installer/scripts/livenessprobe.sh

@@ -1,26 +1,33 @@
 #!/bin/bash
 source /opt/env_vars
 
-if [[ "${CONTROLLER_TYPE}" == "DaemonSet" && "${CONTAINER_TYPE}" != "PrometheusSidecar" ]]; then
-  syslog_status=$(cat /var/opt/microsoft/docker-cimprov/state/syslog.status 2>/dev/null)
-  if grep -qr LINUX_SYSLOGS_BLOB /etc/mdsd.d/config-cache/configchunks > /dev/null 2>&1; then
-          if [[ "$syslog_status" == "disabled" ]]; then
-                  echo "enabled" > /var/opt/microsoft/docker-cimprov/state/syslog.status
-                  [ -d "/var/run/mdsd-ci" ] && cp /etc/opt/microsoft/docker-cimprov/70-rsyslog-forward-mdsd-ci.conf /var/run/mdsd-ci && echo "add" > /var/run/mdsd-ci/update.status
-          fi
-  else
-          if [[ "$syslog_status" == "enabled" ]]; then
-                  echo "disabled" > /var/opt/microsoft/docker-cimprov/state/syslog.status
-                  [ -f "/var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf" ] && rm /var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf && echo "remove" > /var/run/mdsd-ci/update.status
-          fi
-  fi
+syslogSetup() {
+    syslog_status=$(cat /var/opt/microsoft/docker-cimprov/state/syslog.status 2>/dev/null)
+    if grep -qr LINUX_SYSLOGS_BLOB /etc/mdsd.d/config-cache/configchunks > /dev/null 2>&1; then
+            if [[ "$syslog_status" == "disabled" ]]; then
+                    echo "enabled" > /var/opt/microsoft/docker-cimprov/state/syslog.status
+                    [ -d "/var/run/mdsd-ci" ] && cp /etc/opt/microsoft/docker-cimprov/70-rsyslog-forward-mdsd-ci.conf /var/run/mdsd-ci && echo "add" > /var/run/mdsd-ci/update.status
+            fi
+    else
+            if [[ "$syslog_status" == "enabled" ]]; then
+                    echo "disabled" > /var/opt/microsoft/docker-cimprov/state/syslog.status
+                    [ -f "/var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf" ] && rm /var/run/mdsd-ci/70-rsyslog-forward-mdsd-ci.conf && echo "remove" > /var/run/mdsd-ci/update.status
+            fi
+    fi
+}
 
-  CURRENT_LOGS_AND_EVENTS_ONLY=${LOGS_AND_EVENTS_ONLY}
-  ruby /opt/dcr-config-parser.rb > /dev/write-to-traces 2>&1
-  source /opt/dcr_env_var
-  if [ "${LOGS_AND_EVENTS_ONLY}" != "${CURRENT_LOGS_AND_EVENTS_ONLY}" ]; then
-    echo "dcr_env_var has been updated - dcr config changed" > /dev/termination-log
-    exit 1
+if [[ "${CONTROLLER_TYPE}" == "DaemonSet" ]]; then
+  if [[ "${CONTAINER_TYPE}" == "PrometheusSidecar" && "${GENEVA_LOGS_INTEGRATION}" == "true" && -d "/var/run/mdsd-ci" ]]; then
+    syslogSetup
+  else
+    syslogSetup
+    CURRENT_LOGS_AND_EVENTS_ONLY=${LOGS_AND_EVENTS_ONLY}
+    ruby /opt/dcr-config-parser.rb > /dev/write-to-traces 2>&1
+    source /opt/dcr_env_var
+    if [ "${LOGS_AND_EVENTS_ONLY}" != "${CURRENT_LOGS_AND_EVENTS_ONLY}" ]; then
+      echo "dcr_env_var has been updated - dcr config changed" > /dev/termination-log
+      exit 1
+    fi
   fi
 fi
 

kubernetes/ama-logs.yaml

@@ -547,6 +547,8 @@ spec:
               value: "PrometheusSidecar"
             - name: CONTROLLER_TYPE
               value: "DaemonSet"
+            - name: SYSLOG_HOST_PORT
+              value: "28330"
             - name: NODE_IP
               valueFrom:
                 fieldRef:
@@ -567,6 +569,8 @@ spec:
             - name: kube-api-access
               mountPath: /var/run/secrets/kubernetes.io/serviceaccount
               readOnly: true
+            - mountPath: /var/run/mdsd-ci
+              name: mdsd-sock
             - mountPath: /var/run/mdsd-PrometheusSidecar
               name: mdsd-prometheus-sock
             - mountPath: /etc/kubernetes/host

kubernetes/linux/main.sh

@@ -660,6 +660,8 @@ else
     true > /etc/opt/microsoft/docker-cimprov/fluent-bit-internal-metrics.conf
 fi
 
+setGlobalEnvVar GENEVA_LOGS_INTEGRATION "${GENEVA_LOGS_INTEGRATION}"
+
 if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ] && [ "${GENEVA_LOGS_INTEGRATION_SERVICE_MODE}" != "true" ]; then
       #Parse the configmap to set the right environment variables.
       ruby tomlparser.rb
@@ -998,6 +1000,18 @@ else
       echo "MDSD backpressure threshold not set since tail_mem_buf_limit_megabytes is used in configmap. Use backpressure_memory_threshold_in_mb in configmap to set it."
 fi
 
+if [ -n "$SYSLOG_HOST_PORT" ] && [ "$SYSLOG_HOST_PORT" != "28330" ]; then
+      echo "Updating rsyslog config file with non default SYSLOG_HOST_PORT value ${SYSLOG_HOST_PORT}"
+      if sed -i "s/Port=\"[0-9]*\"/Port=\"$SYSLOG_HOST_PORT\"/g" /etc/opt/microsoft/docker-cimprov/70-rsyslog-forward-mdsd-ci.conf; then
+            echo "Successfully updated the rsylog config file."
+      else
+            echo "Failed to update the rsyslog config file."
+      fi
+else
+      echo "SYSLOG_HOST_PORT is ${SYSLOG_HOST_PORT}. No changes made."
+fi
+SYSLOG_PORT_CONFIG="-y 0" # disables syslog listener for mdsd
+
 if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
     if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
       echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in sidecar container..."
@@ -1009,8 +1023,15 @@ if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
       echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc
       source ~/.bashrc
       mkdir -p /var/run/mdsd-${CONTAINER_TYPE}
+      if [[ "${GENEVA_LOGS_INTEGRATION}" == "true" && -d "/var/run/mdsd-ci" && -n "${SYSLOG_HOST_PORT}" ]]; then
+            echo "enabling syslog listener for mdsd in prometheus sidecar container"
+            export MDSD_DEFAULT_TCP_SYSLOG_PORT=$SYSLOG_HOST_PORT
+            echo "export MDSD_DEFAULT_TCP_SYSLOG_PORT=$MDSD_DEFAULT_TCP_SYSLOG_PORT" >> ~/.bashrc
+            source ~/.bashrc
+            SYSLOG_PORT_CONFIG="" # enable syslog listener for mdsd for prometheus sidecar in geneva mode
+      fi
       # add -T 0xFFFF for full traces
-      mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -y 0 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
+      mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 "${SYSLOG_PORT_CONFIG}" -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
     else
       echo "not starting mdsd (no metrics to scrape since MUTE_PROM_SIDECAR is true)"
     fi
@@ -1019,24 +1040,18 @@ else
       if isHighLogScaleMode; then
             startAMACoreAgent
       fi
-      # add -T 0xFFFF for full traces
       export MDSD_ROLE_PREFIX=/var/run/mdsd-ci/default
       echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc
-      if [ -n "$SYSLOG_HOST_PORT" ] && [ "$SYSLOG_HOST_PORT" != "28330" ]; then
-            echo "Updating rsyslog config file with non default SYSLOG_HOST_PORT value ${SYSLOG_HOST_PORT}"
-            if sed -i "s/Port=\"[0-9]*\"/Port=\"$SYSLOG_HOST_PORT\"/g" /etc/opt/microsoft/docker-cimprov/70-rsyslog-forward-mdsd-ci.conf; then
-                  echo "Successfully updated the rsylog config file."
-            else
-                  echo "Failed to update the rsyslog config file."
-            fi
-      else
-            echo "SYSLOG_HOST_PORT is ${SYSLOG_HOST_PORT}. No changes made."
+      if [[ "${GENEVA_LOGS_INTEGRATION}" != "true" ]]; then
+            echo "enabling syslog listener for mdsd in main container"
+            export MDSD_DEFAULT_TCP_SYSLOG_PORT=28330
+            echo "export MDSD_DEFAULT_TCP_SYSLOG_PORT=$MDSD_DEFAULT_TCP_SYSLOG_PORT" >> ~/.bashrc
+            source ~/.bashrc
+            SYSLOG_PORT_CONFIG="" # enable syslog listener for mdsd for main container when not in geneva mode
       fi
-      export MDSD_DEFAULT_TCP_SYSLOG_PORT=28330
-      echo "export MDSD_DEFAULT_TCP_SYSLOG_PORT=$MDSD_DEFAULT_TCP_SYSLOG_PORT" >> ~/.bashrc
-      source ~/.bashrc
       mkdir -p /var/run/mdsd-ci
-      mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos 2>>/dev/null &
+      # add -T 0xFFFF for full traces
+      mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} "${SYSLOG_PORT_CONFIG}" -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos 2>>/dev/null &
 fi
 
 # # Set up a cron job for logrotation