Parse container logs using Docker or CRI-O format

Author: phoswald

Dockerfile

@@ -19,3 +19,5 @@ RUN set -e \
 #&& rm -rf /var/lib/apt/lists/* \
 #&& rm -rf /tmp/* /var/tmp/* $GEM_HOME/cache/*.gem \
  && echo OK
+ 
+COPY templates/kubernetes.conf templates/

templates/kubernetes.conf

@@ -0,0 +1,184 @@
+# Sync with:
+# https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.2/debian-elasticsearch/conf/kubernetes.conf
+
+# Parse /var/log/containers/*.log using Docker (JSON) or CRI-O (text) format.
+# Docker always logs full lines. CRI-O may log full (F) or partial (P) lines.
+<source>
+  @type tail
+  @id in_tail_container_logs
+  path /var/log/containers/*.log
+  pos_file /var/log/{{.ID}}-fluentd-containers.log.pos
+  tag kubernetes.*
+  read_from_head true
+  <parse>
+    @type multi_format
+    <pattern>
+      format json
+      time_format %Y-%m-%dT%H:%M:%S.%NZ
+    </pattern>
+    <pattern>
+      format regexp
+      expression /^(?<time>.+) (?<stream>stdout|stderr) (?<partialflag>F|P) (?<log>.*)$/
+      time_format %Y-%m-%dT%H:%M:%S.%N%:z
+    </pattern>
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_startupscript
+  path /var/log/startupscript.log
+  pos_file /var/log/{{.ID}}-fluentd-startupscript.log.pos
+  tag startupscript
+  <parse>
+    @type syslog
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_docker
+  path /var/log/docker.log
+  pos_file /var/log/{{.ID}}-fluentd-docker.log.pos
+  tag docker
+  <parse>
+    @type regexp
+    expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_etcd
+  path /var/log/etcd.log
+  pos_file /var/log/{{.ID}}-fluentd-etcd.log.pos
+  tag k8s.etcd
+  <parse>
+    @type none
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_kubelet
+  multiline_flush_interval 5s
+  path /var/log/kubelet.log
+  pos_file /var/log/{{.ID}}-fluentd-kubelet.log.pos
+  tag k8s.kubelet
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_kube_proxy
+  multiline_flush_interval 5s
+  path /var/log/kube-proxy.log
+  pos_file /var/log/{{.ID}}-fluentd-kube-proxy.log.pos
+  tag k8s.kube-proxy
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_kube_apiserver
+  multiline_flush_interval 5s
+  path /var/log/kube-apiserver.log
+  pos_file /var/log/{{.ID}}-fluentd-kube-apiserver.log.pos
+  tag k8s.kube-apiserver
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_kube_controller_manager
+  multiline_flush_interval 5s
+  path /var/log/kube-controller-manager.log
+  pos_file /var/log/{{.ID}}-fluentd-kube-controller-manager.log.pos
+  tag k8s.kube-controller-manager
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_kube_scheduler
+  multiline_flush_interval 5s
+  path /var/log/kube-scheduler.log
+  pos_file /var/log/{{.ID}}-fluentd-kube-scheduler.log.pos
+  tag k8s.kube-scheduler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_rescheduler
+  multiline_flush_interval 5s
+  path /var/log/rescheduler.log
+  pos_file /var/log/{{.ID}}-fluentd-rescheduler.log.pos
+  tag k8s.rescheduler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_glbc
+  multiline_flush_interval 5s
+  path /var/log/glbc.log
+  pos_file /var/log/{{.ID}}-fluentd-glbc.log.pos
+  tag k8s.glbc
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+<source>
+  @type tail
+  @id in_tail_cluster_autoscaler
+  multiline_flush_interval 5s
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/{{.ID}}-fluentd-cluster-autoscaler.log.pos
+  tag k8s.cluster-autoscaler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>
+
+# Example:
+# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
+# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
+<source>
+  @type tail
+  @id in_tail_kube_apiserver_audit
+  multiline_flush_interval 5s
+  path /var/log/kubernetes/kube-apiserver-audit.log
+  pos_file /var/log/{{.ID}}-kube-apiserver-audit.log.pos
+  tag k8s.kube-apiserver-audit
+  <parse>
+    @type multiline
+    format_firstline /^\S+\s+AUDIT:/
+    # Fields must be explicitly captured by name to be parsed into the record.
+    # Fields may not always be present, and order may change, so this just looks
+    # for a list of key="\"quoted\" value" pairs separated by spaces.
+    # Unknown fields are ignored.
+    # Note: We can't separate query/response lines as format1/format2 because
+    #       they don't always come one after the other for a given query.
+    format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
+    time_format %Y-%m-%dT%T.%L%Z
+  </parse>
+</source>
+
+<filter kubernetes.**>
+  @type kubernetes_metadata
+  @id filter_kube_metadata
+</filter>