Feature/multi fluentbit support (#665)

NodeAgent Support

Author: ahma

charts/logging-demo/requirements.lock

@@ -6,4 +6,4 @@ dependencies:
   repository: https://kubernetes-charts.banzaicloud.com/
   version: 0.1.5
 digest: sha256:8c6a3777c9e971fc03bad54b76fa62ef58b07f73436121522c7a6e97fce6ece5
-generated: "2021-01-20T11:35:41.461008+01:00"
+generated: "2021-01-08T16:16:09.458546+01:00"

charts/logging-operator-logging/README.md

@@ -42,6 +42,7 @@ The following tables lists the configurable parameters of the logging-operator-l
 | `fluentd.tolerations`                               | Tolerations for fluentd statefulset                                      | none                                                       |
 | `fluentd.nodeSelector`                              | Node selector for fluentd pods                                           | none                                                       |
 | `fluentd.podPriorityClassName`                      | Priority class name for fluentd pods                                     | none                                                       |
+| `nodeAgents`             | Node agents definitions                                     | {}}                                                       |
 | `defaultFlow`                                       | Default Flow                                                             | {}                                                         |
 | `clusterFlows`                                      | Array of ClusterFlows to be created                                      | []                                                         |
 | `clusterOutputs`                                    | Array of ClusterOutputs to be created                                    | []                                                         |

charts/logging-operator-logging/templates/logging.yaml

@@ -54,4 +54,9 @@ spec:
   {{- else }}
   fluentbit: {}
   {{- end }}
-
+  {{- if .Values.nodeAgents }}
+  nodeAgents:
+{{ toYaml .Values.nodeAgents | indent 4 }}
+  {{- else }}
+  nodeAgents: []
+  {{- end }}

charts/logging-operator-logging/values.yaml

@@ -18,6 +18,31 @@ enableRecreateWorkloadOnImmutableFieldChange: false
 
 # Fluent-bit configurations https://banzaicloud.com/docs/one-eye/logging-operator/configuration/crds/#fluent-bit-spec
 fluentbit: {}
+
+#Node agent Definition
+nodeAgents: {}
+#  - name: win-agent
+#    profile: windows
+#    nodeAgentFluentbit:
+#      daemonSet:
+#        spec:
+#          template:
+#            spec:
+#              containers:
+#              - image: banzaicloud/fluentbit:1.6.10
+#                name: fluent-bit
+#      tls:
+#        enabled: false
+#  - name: linux-agent
+#    profile: linux
+#    nodeAgentFluentbit:
+#      metrics:
+#        prometheusAnnotations: true
+#        serviceMonitor: false
+#      tls:
+#        enabled: false
+
+
 # Fluentd configurations https://banzaicloud.com/docs/one-eye/logging-operator/configuration/crds/#fluentd-spec
 fluentd: {}
 # 20Gi persistent storage is configured for fluentd by default.

charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml

@@ -66,7 +66,7 @@ func plugins() {
 		For more information please click on the plugin name
 		<center>
 
-		| Name | Type | Description | Status |Version |
+		| Name | Profile | Description | Status |Version |
 		|:---|---|:---|:---:|---:|`,
 	)
 

cmd/docs.go

@@ -24,6 +24,7 @@ import (
 	"github.com/banzaicloud/logging-operator/pkg/resources/fluentbit"
 	"github.com/banzaicloud/logging-operator/pkg/resources/fluentd"
 	"github.com/banzaicloud/logging-operator/pkg/resources/model"
+	"github.com/banzaicloud/logging-operator/pkg/resources/nodeagent"
 	"github.com/banzaicloud/logging-operator/pkg/sdk/model/render"
 	"github.com/banzaicloud/operator-tools/pkg/reconciler"
 	"github.com/banzaicloud/operator-tools/pkg/secret"
@@ -120,6 +121,10 @@ func (r *LoggingReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		reconcilers = append(reconcilers, fluentbit.New(r.Client, r.Log, &logging, reconcilerOpts).Reconcile)
 	}
 
+	if len(logging.Spec.NodeAgents) > 0 {
+		reconcilers = append(reconcilers, nodeagent.New(r.Client, r.Log, &logging, reconcilerOpts).Reconcile)
+	}
+
 	for _, rec := range reconcilers {
 		result, err := rec()
 		if err != nil {
@@ -224,6 +229,7 @@ func SetupLoggingWithManager(mgr ctrl.Manager, logger logr.Logger) *ctrl.Builder
 
 	fluentd.RegisterWatches(builder)
 	fluentbit.RegisterWatches(builder)
+	nodeagent.RegisterWatches(builder)
 
 	return builder
 }

config/crd/bases/logging.banzaicloud.io_loggings.yaml

@@ -0,0 +1,69 @@
+# escape=`
+
+ARG WINDOWS_VERSION=ltsc2019
+
+#
+# Builder Image - Windows Server Core
+#
+FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION as builder
+
+# The FLUENTBIT_VERSION ARG must be after the FROM instruction
+ARG FLUENTBIT_VERSION=1.6.10
+ARG IMAGE_CREATE_DATE
+ARG IMAGE_SOURCE_REVISION
+
+# Metadata as defined in OCI image spec annotations
+# https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.title="Fluent Bit" `
+      org.opencontainers.image.description="Fluent Bit is an open source and multi-platform Log Processor and Forwarder which allows you to collect data/logs from different sources, unify and send them to multiple destinations. It's fully compatible with Docker and Kubernetes environments." `
+      org.opencontainers.image.created=$IMAGE_CREATE_DATE `
+      org.opencontainers.image.version=$FLUENTBIT_VERSION `
+      org.opencontainers.image.authors="Eduardo Silva <[email protected]>" `
+      org.opencontainers.image.url="https://hub.docker.com/r/fluent/fluent-bit" `
+      org.opencontainers.image.documentation="https://docs.fluentbit.io/manual/" `
+      org.opencontainers.image.vendor="Fluent Organization" `
+      org.opencontainers.image.licenses="Apache-2.0" `
+      org.opencontainers.image.source="https://github.com/fluent/fluent-bit" `
+      org.opencontainers.image.revision=$IMAGE_SOURCE_REVISION
+
+#
+# Basic setup
+#
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN Write-Host ('Creating folders'); `
+    New-Item -Type Directory -Path /local; `
+    New-Item -Type Directory -Path /fluent-bit;
+
+WORKDIR /local
+
+#
+# Install Fluent Bit
+#
+RUN Write-Host ('Installing Fluent Bit'); `
+    $majorminor = ([Version]::Parse("$env:FLUENTBIT_VERSION")).toString(2); `
+    Invoke-WebRequest -Uri "https://fluentbit.io/releases/$($majorminor)/td-agent-bit-$($env:FLUENTBIT_VERSION)-win64.zip" -OutFile td-agent-bit.zip; `
+    Expand-Archive -Path td-agent-bit.zip -Destination /local/fluent-bit; `
+    Move-Item -Path /local/fluent-bit/*/* -Destination /fluent-bit/;
+
+#
+# Technique from https://github.com/StefanScherer/dockerfiles-windows/blob/master/mongo/3.6/Dockerfile
+#
+ADD https://aka.ms/vs/15/release/vc_redist.x64.exe /local/vc_redist.x64.exe
+
+RUN Write-Host ('Installing Visual C++ Redistributable Package'); `
+    Start-Process /local/vc_redist.x64.exe -ArgumentList '/install', '/quiet', '/norestart' -NoNewWindow -Wait; `
+    Copy-Item -Path /Windows/System32/msvcp140.dll -Destination /fluent-bit/bin/; `
+    Copy-Item -Path /Windows/System32/vccorlib140.dll -Destination /fluent-bit/bin/; `
+    Copy-Item -Path /Windows/System32/vcruntime140.dll -Destination /fluent-bit/bin/;
+
+#
+# Runtime Image - Windows Server Nano
+#
+FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION as runtime
+
+COPY --from=builder /fluent-bit /fluent-bit
+
+RUN setx /M PATH "%PATH%;C:\fluent-bit\bin"
+
+ENTRYPOINT ["fluent-bit.exe", "-i", "dummy", "-o", "stdout"]

controllers/logging_controller.go

@@ -7,7 +7,7 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
 	github.com/banzaicloud/logging-operator/pkg/sdk v0.0.0
-	github.com/banzaicloud/operator-tools v0.15.0
+	github.com/banzaicloud/operator-tools v0.20.1
 	github.com/go-logr/logr v0.2.1
 	github.com/onsi/gomega v1.10.1
 	github.com/pborman/uuid v1.2.0
@@ -24,5 +24,3 @@ require (
 replace k8s.io/client-go => k8s.io/client-go v0.19.2
 
 replace github.com/banzaicloud/logging-operator/pkg/sdk => ./pkg/sdk
-
-//replace github.com/banzaicloud/operator-tools => ../operator-tools