@@ -7,26 +7,77 @@ global:
77hostNetwork : true
88dnsPolicy : ClusterFirstWithHostNet
99
10- # NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters
10+ # ----------------------------------------------------------#
11+ # PARSERS for k8s-custom-tag abd crio
12+ # NOTE: Read this link for more details about WHY CRIO parser used -> https://docs.fluentbit.io/manual/installation/kubernetes#container-runtime-interface-cri-parser
13+ # e.g., k8s log line for crio ->
14+ # 2023-02-19T21:28:48.495311051Z stdout F Unsetting extraneous env vars (UTC): 21:28:48
15+ # ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
16+ # ----------------------------------------------------------#
17+ service :
18+ parsersFiles :
19+ - /fluent-bit/parsers/parsers.conf
20+ extraParsers : |
21+ [PARSER]
22+ Name k8s-custom-tag
23+ Format regex
24+ Regex ^(?<namespace_name>[^_]+)\.(?<container_name>.+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<docker_id>[a-z0-9]{64})-$
25+
26+ [PARSER]
27+ Name crio
28+ Format Regex
29+ Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
30+ Time_Key time
31+ Time_Format %Y-%m-%dT%H:%M:%S.%L%z
32+
33+ # ----------------------------------------------------------#
34+ # FILTER logs with k8s-custom-tag parser
35+ # Tag_regex -> Use this to verify the regex https://rubular.com/
36+ # ----------------------------------------------------------#
37+ input :
38+ enabled : true
39+ tag : kube.<namespace_name>.<container_name>.<pod_name>.<docker_id>-
40+ path : " /var/log/containers/*.log"
41+ db : " /var/log/flb_kube.db"
42+ parser : crio
43+ memBufLimit : 5MB
44+ skipLongLines : " On"
45+ refreshInterval : 10
46+ extraInputs : |
47+ Tag_Regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
48+
49+ # ----------------------------------------------------------#
50+ # FILTER logs with k8s-custom-tag parser
51+ # ----------------------------------------------------------#
52+ # NOTE: The Kubernetes filter will enrich the logs with Kubernetes metadata, specifically labels and annotations.
53+ # The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
54+ # ----------------------------------------------------------#
1155filter :
56+ enabled : true
1257 name : " kubernetes"
1358 match : " kube.*"
1459 kubeURL : " https://kubernetes.default.svc.cluster.local:443"
1560 mergeLog : " On"
1661 mergeLogKey : " log_processed"
1762 keepLog : " On"
1863 k8sLoggingParser : " On"
19- k8sLoggingExclude : " Off "
64+ k8sLoggingExclude : " On "
2065 bufferSize : " 0"
2166 extraFilters : |
22- Kube_Tag_Prefix application.var.log.containers.
23- Labels Off
24- Annotations Off
67+ Kube_Tag_Prefix kube.
68+ Regex_Parser k8s-custom-tag
2569 Use_Kubelet true
2670 Kubelet_Port 10250
71+ Annotations Off
2772 Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
2873 Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
2974
75+ # extraFilters: |
76+ # Labels Off
77+
78+ # ----------------------------------------------------------#
79+ # OUTPUT logs to CloudWatch
80+ # ----------------------------------------------------------#
3081cloudWatch :
3182 enabled : true
3283 match : " *"
@@ -40,6 +91,8 @@ cloudWatch:
4091 autoCreateGroup : false
4192 endpoint :
4293 credentialsEndpoint : {}
94+ # extraOutputs: |
95+ # ...
4396
4497firehose :
4598 enabled : false
@@ -50,23 +103,36 @@ kinesis:
50103elasticsearch :
51104 enabled : false
52105
106+
107+ # ----------------------------------------------------------#
108+ # OUTPUT logs to S3
109+ # ----------------------------------------------------------#
53110# Use this config to write logs to an S3 bucket.
54111# Pre-req
55112# 1/ S3 bucket for logging
56113# 2/ Additional IAM policy for FluentBit add-on IRSA config
57114# 3/ Add this to Terraform to pass additional IAM policy "aws_for_fluentbit_irsa_policies = ["<ENTER_NEW_IAM_POLICY_FOR_S3>"]"
58-
115+ # ----------------------------------------------------------#
59116additionalOutputs : |
60117 [OUTPUT]
61118 Name s3
62119 Match *
63120 region ${region}
64121 bucket ${s3_bucket_name}
65122 total_file_size 100M
66- s3_key_format /fluentbit -logs/$TAG[4]/ year=%Y/month=%m/day=%d/hour=%H/
67- s3_key_format_tag_delimiters ._
123+ s3_key_format /${cluster_name}/application -logs/year=%Y/month=%m/day=%d/$TAG[1]/$TAG[2]/$TAG[3]/$TAG[3]_%H%M%S_$UUID.log
124+ s3_key_format_tag_delimiters ..
68125 store_dir /home/ec2-user/buffer
69126 upload_timeout 10m
127+ workers 2
128+
129+ # ----------------------------------------------------------#
130+ # Use below when compression is enabled for S3 logs with gzip. Multipart upload cannot be used with gzip compression
131+ # use_put_object On
132+ # content_type application/json
133+ # compression gzip
134+ # preserve_data_ordering On
135+ # ----------------------------------------------------------#
70136
71137serviceAccount :
72138 create : true
@@ -88,3 +154,18 @@ updateStrategy:
88154
89155nodeSelector :
90156 kubernetes.io/os : linux
157+
158+ volumes :
159+ - name : varlog
160+ hostPath :
161+ path : /var/log
162+ - name : varlibdockercontainers
163+ hostPath :
164+ path : /var/lib/docker/containers
165+
166+ volumeMounts :
167+ - name : varlog
168+ mountPath : /var/log
169+ - name : varlibdockercontainers
170+ mountPath : /var/lib/docker/containers
171+ readOnly : true
0 commit comments