Skip to content
This repository was archived by the owner on Apr 1, 2020. It is now read-only.

Commit 28e517d

Browse files
pipo02mixpipo02mix
authored
Merge pull request #49 from giantswarm/fluentd
Add EFK chart
2 parents 300f258 + cbd8df9 commit 28e517d

24 files changed

+854
-0
lines changed

‎helm/g8s-efk-chart/.helmignore

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*~
18+
# Various IDEs
19+
.project
20+
.idea/
21+
*.tmproj

‎helm/g8s-efk-chart/Chart.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: efk
2+
version: 1.0.0
3+
appVersion: 6.1.1
4+
description: Elasticsearch, Fluentbit and Kibana stack ready to be your logging system.
5+
icon: https://static-www.elastic.co/assets/blteb1c97719574938d/logo-elastic-elasticsearch-lt.svg
6+
sources:
7+
- https://www.elastic.co/products/elasticsearch
8+
- https://github.com/jetstack/elasticsearch-pet
9+
- https://github.com/GoogleCloudPlatform/elasticsearch-docker
10+
- https://github.com/clockworksoul/helm-elasticsearch
11+
- https://github.com/pires/kubernetes-elasticsearch-cluster
12+
maintainers:
13+
- name: giantswarm
14+
email: [email protected]
15+
engine: gotpl
16+
tillerVersion: ">=2.8.0"

‎helm/g8s-efk-chart/templates/curator/cronjob.yaml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
apiVersion: batch/v2alpha1
2+
kind: CronJob
3+
metadata:
4+
namespace: "{{ .Values.namespace }}"
5+
name: curator
6+
spec:
7+
schedule: "{{ .Values.curator.cron }}"
8+
successfulJobsHistoryLimit: 2
9+
failedJobsHistoryLimit: 2
10+
jobTemplate:
11+
spec:
12+
template:
13+
metadata:
14+
name: curator
15+
labels:
16+
app: curator
17+
spec:
18+
containers:
19+
- name: curator
20+
image: quay.io/giantswarm/curator:latest
21+
imagePullPolicy: Always
22+
env:
23+
- name: ELASTICSEARCH_HOST
24+
value: elasticsearch:9200
25+
- name: RETENTION_DAYS
26+
value: "{{ .Values.curator.retention }}"
27+
- name: INDEX_NAME_PREFIX
28+
value: "{{ .Values.logsPrefix }}-"
29+
- name: INDEX_NAME_TIMEFORMAT
30+
value: "%Y.%m.%d"
31+
resources:
32+
limits:
33+
cpu: 50m
34+
memory: 50Mi
35+
requests:
36+
cpu: 50m
37+
memory: 50Mi
38+
restartPolicy: OnFailure
39+
# retry for a maximum of 10 minutes
40+
activeDeadlineSeconds: 600

‎helm/g8s-efk-chart/templates/elasticsearch/configmap.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: elasticsearch
5+
namespace: "{{ .Values.namespace }}"
6+
labels:
7+
app: elasticsearch
8+
data:
9+
elasticsearch.yml: |
10+
cluster.name: {{ .Values.clusterName }}
11+
node.name: "es_node"
12+
path.data: /usr/share/elasticsearch/data
13+
http:
14+
host: 0.0.0.0
15+
port: 9200
16+
bootstrap.memory_lock: true
17+
transport.host: 127.0.0.1
18+
discovery:
19+
zen:
20+
minimum_master_nodes: 1
21+
logger.org.elasticsearch.transport: debug

‎helm/g8s-efk-chart/templates/elasticsearch/deployment.yaml

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: elasticsearch
5+
namespace: "{{ .Values.namespace }}"
6+
labels:
7+
app: elasticsearch
8+
spec:
9+
replicas: 1
10+
revisionHistoryLimit: 3
11+
strategy:
12+
type: Recreate
13+
template:
14+
metadata:
15+
annotations:
16+
releasetime: {{ $.Release.Time }}
17+
labels:
18+
app: elasticsearch
19+
spec:
20+
affinity:
21+
nodeAffinity:
22+
requiredDuringSchedulingIgnoredDuringExecution:
23+
nodeSelectorTerms:
24+
- matchExpressions:
25+
- key: role
26+
operator: NotIn
27+
values:
28+
- master
29+
{{- if .Values.elasticsearch.nodeSelector }}
30+
nodeSelector:
31+
{{ toYaml .Values.data.nodeSelector | indent 8 }}
32+
{{- end }}
33+
{{- if .Values.elasticsearch.tolerations }}
34+
tolerations:
35+
{{ toYaml .Values.elasticsearch.tolerations | indent 8 }}
36+
{{- end }}
37+
initContainers:
38+
- name: set-vm-max-map-count
39+
image: quay.io/giantswarm/busybox:1.28.3
40+
imagePullPolicy: IfNotPresent
41+
command: ['sysctl', '-w', 'vm.max_map_count=262144']
42+
securityContext:
43+
privileged: true
44+
{{- if .Values.elasticsearch.persistence.enabled }}
45+
- name: volume-mount-hack
46+
image: quay.io/giantswarm/busybox:1.28.3
47+
imagePullPolicy: IfNotPresent
48+
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
49+
volumeMounts:
50+
- name: elasticsearch-data
51+
mountPath: /usr/share/elasticsearch/data
52+
{{- end }}
53+
serviceAccountName: elasticsearch
54+
containers:
55+
- name: elasticsearch
56+
image: "{{ .Values.elasticsearch.image.repository }}:{{ .Values.elasticsearch.image.tag }}"
57+
imagePullPolicy: {{ .Values.elasticsearch.image.pullPolicy | quote }}
58+
env:
59+
- name: ES_JAVA_OPTS
60+
value: "-Djava.net.preferIPv4Stack=true -Xms4g -Xmx4g"
61+
ports:
62+
- containerPort: 9200
63+
livenessProbe:
64+
httpGet:
65+
path: /_cluster/health?local=true
66+
port: 9200
67+
initialDelaySeconds: 60
68+
readinessProbe:
69+
httpGet:
70+
path: /_cluster/health?local=true
71+
port: 9200
72+
initialDelaySeconds: 30
73+
resources:
74+
{{ toYaml .Values.elasticsearch.resources | indent 12 }}
75+
volumeMounts:
76+
- name: config
77+
mountPath: /usr/share/elasticsearch/elasticsearch.yml
78+
subPath: elasticsearch.yml
79+
- name: elasticsearch-data
80+
mountPath: /usr/share/elasticsearch/data
81+
restartPolicy: Always
82+
volumes:
83+
- name: config
84+
configMap:
85+
name: elasticsearch
86+
- name: elasticsearch-data
87+
{{- if .Values.elasticsearch.persistence.enabled }}
88+
persistentVolumeClaim:
89+
claimName: {{ .Values.elasticsearch.persistence.pvcName | quote }}
90+
{{- else }}
91+
emptyDir: {}
92+
{{- end }}

‎helm/g8s-efk-chart/templates/elasticsearch/persistent-volume-claim.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{{- if .Values.elasticsearch.persistence.enabled }}
2+
kind: PersistentVolumeClaim
3+
apiVersion: v1
4+
metadata:
5+
labels:
6+
app: elasticsearch
7+
name: {{ .Values.elasticsearch.persistence.pvcName }}
8+
namespace: "{{ .Values.namespace }}"
9+
annotations:
10+
"helm.sh/resource-policy": keep
11+
spec:
12+
accessModes:
13+
- ReadWriteOnce
14+
resources:
15+
requests:
16+
storage: {{ .Values.elasticsearch.persistence.size }}
17+
{{- end }}

‎helm/g8s-efk-chart/templates/elasticsearch/psp.yaml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: extensions/v1beta1
2+
kind: PodSecurityPolicy
3+
metadata:
4+
name: elasticsearch-psp
5+
spec:
6+
privileged: true
7+
fsGroup:
8+
rule: RunAsAny
9+
runAsUser:
10+
rule: RunAsAny
11+
seLinux:
12+
rule: RunAsAny
13+
supplementalGroups:
14+
rule: RunAsAny
15+
volumes:
16+
- 'secret'
17+
- 'configMap'
18+
- 'hostPath'
19+
- 'persistentVolumeClaim'
20+
- 'emptyDir'
21+
hostNetwork: false
22+
hostIPC: false
23+
hostPID: false

‎helm/g8s-efk-chart/templates/elasticsearch/rbac.yaml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
kind: ClusterRole
2+
apiVersion: rbac.authorization.k8s.io/v1beta1
3+
metadata:
4+
name: elasticsearch
5+
rules:
6+
- apiGroups:
7+
- ""
8+
resources:
9+
- "services"
10+
- "namespaces"
11+
- "endpoints"
12+
verbs:
13+
- "get"
14+
---
15+
apiVersion: rbac.authorization.k8s.io/v1beta1
16+
kind: ClusterRoleBinding
17+
metadata:
18+
name: elasticsearch
19+
subjects:
20+
- kind: ServiceAccount
21+
name: elasticsearch
22+
namespace: "{{ .Values.namespace }}"
23+
roleRef:
24+
kind: ClusterRole
25+
name: elasticsearch
26+
apiGroup: rbac.authorization.k8s.io
27+
---
28+
apiVersion: rbac.authorization.k8s.io/v1beta1
29+
kind: ClusterRole
30+
metadata:
31+
name: elasticsearch-psp
32+
rules:
33+
- apiGroups:
34+
- extensions
35+
resources:
36+
- podsecuritypolicies
37+
verbs:
38+
- use
39+
resourceNames:
40+
- elasticsearch-psp
41+
---
42+
apiVersion: rbac.authorization.k8s.io/v1beta1
43+
kind: ClusterRoleBinding
44+
metadata:
45+
name: elasticsearch-psp
46+
subjects:
47+
- kind: ServiceAccount
48+
name: elasticsearch
49+
namespace: "{{ .Values.namespace }}"
50+
roleRef:
51+
kind: ClusterRole
52+
name: elasticsearch-psp
53+
apiGroup: rbac.authorization.k8s.io

‎helm/g8s-efk-chart/templates/elasticsearch/service.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: elasticsearch
5+
namespace: "{{ .Values.namespace }}"
6+
labels:
7+
app: elasticsearch
8+
spec:
9+
ports:
10+
- name: nginx
11+
port: 8000
12+
targetPort: 8000
13+
- name: elasticsearch
14+
port: 9200
15+
targetPort: 9200
16+
selector:
17+
app: elasticsearch

‎helm/g8s-efk-chart/templates/elasticsearch/serviceaccount.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
piVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: elasticsearch
5+
namespace: "{{ .Values.namespace }}"

‎helm/g8s-efk-chart/templates/fluentbit/configmap.yaml

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: fluentbit-config
5+
namespace: "{{ .Values.namespace }}"
6+
labels:
7+
app: fluentbit
8+
data:
9+
# Configuration files: server, input, filters and output
10+
# ======================================================
11+
fluentbit.conf: |
12+
[SERVICE]
13+
Flush 5
14+
Log_Level info
15+
Daemon off
16+
Parsers_File parsers.conf
17+
HTTP_Server On
18+
HTTP_Listen 0.0.0.0
19+
HTTP_Port 2020
20+
21+
[INPUT]
22+
Name tail
23+
Tag kube.*
24+
Path /var/log/containers/*.log
25+
Parser docker
26+
DB /var/log/flb_kube.db
27+
Buffer_Max_Size 128k
28+
Mem_Buf_Limit 10MB
29+
Skip_Long_Lines On
30+
Refresh_Interval 10
31+
32+
[FILTER]
33+
# Remove garbage log entries from fluent-bit (https://github.com/fluent/fluent-bit/issues/429)
34+
Name grep
35+
Match *
36+
Exclude log \"took\"\"errors\"\"took\"\"errors\"
37+
38+
[FILTER]
39+
Name kubernetes
40+
Match kube.*
41+
Kube_URL https://kubernetes.default.svc:443
42+
Merge_Log Off
43+
K8S-Logging.Parser On
44+
45+
[OUTPUT]
46+
Name es
47+
Match *
48+
Host ${FLUENT_ELASTICSEARCH_HOST}
49+
Port ${FLUENT_ELASTICSEARCH_PORT}
50+
Logstash_Format On
51+
Logstash_Prefix ${FLUENT_ELASTICSEARCH_PREFIX}
52+
Retry_Limit False
53+
54+
parsers.conf: |
55+
[PARSER]
56+
Name json-test
57+
Format json
58+
Time_Key time
59+
Time_Format %d/%b/%Y:%H:%M:%S %z
60+
61+
[PARSER]
62+
Name docker
63+
Format json
64+
Time_Key time
65+
Time_Format %Y-%m-%dT%H:%M:%S.%L
66+
Time_Keep On
67+
# Command | Decoder | Field | Optional Action
68+
# =============|==================|=================
69+
Decode_Field_As escaped log
70+
71+
[PARSER]
72+
Name syslog
73+
Format regex
74+
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
75+
Time_Key time
76+
Time_Format %b %d %H:%M:%S

0 commit comments

Comments
 (0)