Merge pull request #1737 from kube-logging/output-fields

kristofgyuracz · web-flow · commit ee668e7122cb · 2024-05-13T10:56:57.000+02:00
fix(syslog-ng-outputs): fix output fields where necessary
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_syslogngclusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_syslogngclusteroutputs.yaml
@@ -1698,6 +1698,30 @@ spec:
                     type: boolean
                   compresslevel:
                     type: integer
+                  disk_buffer:
+                    properties:
+                      compaction:
+                        type: boolean
+                      dir:
+                        type: string
+                      disk_buf_size:
+                        format: int64
+                        type: integer
+                      mem_buf_length:
+                        format: int64
+                        type: integer
+                      mem_buf_size:
+                        format: int64
+                        type: integer
+                      q_out_size:
+                        format: int64
+                        type: integer
+                      reliable:
+                        type: boolean
+                    required:
+                    - disk_buf_size
+                    - reliable
+                    type: object
                   flush_grace_period:
                     type: integer
                   log-fifo-size:
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_syslogngoutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_syslogngoutputs.yaml
@@ -1694,6 +1694,30 @@ spec:
                     type: boolean
                   compresslevel:
                     type: integer
+                  disk_buffer:
+                    properties:
+                      compaction:
+                        type: boolean
+                      dir:
+                        type: string
+                      disk_buf_size:
+                        format: int64
+                        type: integer
+                      mem_buf_length:
+                        format: int64
+                        type: integer
+                      mem_buf_size:
+                        format: int64
+                        type: integer
+                      q_out_size:
+                        format: int64
+                        type: integer
+                      reliable:
+                        type: boolean
+                    required:
+                    - disk_buf_size
+                    - reliable
+                    type: object
                   flush_grace_period:
                     type: integer
                   log-fifo-size:
diff --git a/config/crd/bases/logging.banzaicloud.io_syslogngclusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_syslogngclusteroutputs.yaml
@@ -1698,6 +1698,30 @@ spec:
                     type: boolean
                   compresslevel:
                     type: integer
+                  disk_buffer:
+                    properties:
+                      compaction:
+                        type: boolean
+                      dir:
+                        type: string
+                      disk_buf_size:
+                        format: int64
+                        type: integer
+                      mem_buf_length:
+                        format: int64
+                        type: integer
+                      mem_buf_size:
+                        format: int64
+                        type: integer
+                      q_out_size:
+                        format: int64
+                        type: integer
+                      reliable:
+                        type: boolean
+                    required:
+                    - disk_buf_size
+                    - reliable
+                    type: object
                   flush_grace_period:
                     type: integer
                   log-fifo-size:
diff --git a/config/crd/bases/logging.banzaicloud.io_syslogngoutputs.yaml b/config/crd/bases/logging.banzaicloud.io_syslogngoutputs.yaml
@@ -1694,6 +1694,30 @@ spec:
                     type: boolean
                   compresslevel:
                     type: integer
+                  disk_buffer:
+                    properties:
+                      compaction:
+                        type: boolean
+                      dir:
+                        type: string
+                      disk_buf_size:
+                        format: int64
+                        type: integer
+                      mem_buf_length:
+                        format: int64
+                        type: integer
+                      mem_buf_size:
+                        format: int64
+                        type: integer
+                      q_out_size:
+                        format: int64
+                        type: integer
+                      reliable:
+                        type: boolean
+                    required:
+                    - disk_buf_size
+                    - reliable
+                    type: object
                   flush_grace_period:
                     type: integer
                   log-fifo-size:
diff --git a/docs/configuration/plugins/syslogng-outputs/elasticsearch.md b/docs/configuration/plugins/syslogng-outputs/elasticsearch.md
@@ -41,6 +41,12 @@ spec:
 The document ID. If no ID is specified, a document ID is automatically generated. 
 
 
+### disk_buffer (*DiskBuffer, optional) {#elasticsearchoutput-disk_buffer}
+
+This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/).
+
+Default: false
+
 ### index (string, optional) {#elasticsearchoutput-index}
 
 Name of the data stream, index, or index alias to perform the action on. 
diff --git a/docs/configuration/plugins/syslogng-outputs/openobserve.md b/docs/configuration/plugins/syslogng-outputs/openobserve.md
@@ -41,6 +41,12 @@ For details on the available options of the output, see the [documentation of th
 ###  (HTTPOutput, required) {#openobserveoutput-}
 
 
+### disk_buffer (*DiskBuffer, optional) {#openobserveoutput-disk_buffer}
+
+This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/).
+
+Default: false
+
 ### organization (string, optional) {#openobserveoutput-organization}
 
 Name of the organization in OpenObserve. 
diff --git a/docs/configuration/plugins/syslogng-outputs/s3.md b/docs/configuration/plugins/syslogng-outputs/s3.md
@@ -73,6 +73,12 @@ Enable or disable compression.
 
 Default: false
 
+### disk_buffer (*DiskBuffer, optional) {#s3output-disk_buffer}
+
+This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/).
+
+Default: false
+
 ### flush_grace_period (int, optional) {#s3output-flush_grace_period}
 
 Set the number of seconds for flush period.
@@ -101,7 +107,7 @@ Default: 32
 The object_key for the S3 server. 
 
 
-### object_key_timestamp (RawString, optional) {#s3output-object_key_timestamp}
+### object_key_timestamp (string, optional) {#s3output-object_key_timestamp}
 
 Set object_key_timestamp 
 
@@ -131,7 +137,7 @@ The secret_key for the S3 server.
 Set the storage_class option. 
 
 
-### template (RawString, optional) {#s3output-template}
+### template (string, optional) {#s3output-template}
 
 Template 
 
diff --git a/docs/configuration/plugins/syslogng-outputs/splunk_hec.md b/docs/configuration/plugins/syslogng-outputs/splunk_hec.md
@@ -55,6 +55,12 @@ Fallback option for source field.
 Fallback option for sourcetype field. 
 
 
+### disk_buffer (*DiskBuffer, optional) {#splunkhecoutput-disk_buffer}
+
+This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/).
+
+Default: false
+
 ### event (string, optional) {#splunkhecoutput-event}
 
 event() accepts a template, which declares the content of the log message sent to Splunk. Default value: `${MSG}` 
diff --git a/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go b/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go
@@ -194,8 +194,8 @@ destination "output_default_test-s3-out" {
 						},
 					},
 					ObjectKey:          "${HOST}/my-logs",
-					ObjectKeyTimestamp: `"timestamp"`,
-					Template:           `"${MESSAGE}\n"`,
+					ObjectKeyTimestamp: "timestamp",
+					Template:           "${MESSAGE}\n",
 					Compression:        config.NewFalse(),
 					CompressLevel:      9,
 					ChunkSize:          5,
diff --git a/pkg/sdk/logging/model/syslogng/output/elasticsearch.go b/pkg/sdk/logging/model/syslogng/output/elasticsearch.go
@@ -69,6 +69,8 @@ type ElasticsearchOutput struct {
 	LogstashPrefixSeparator string `json:"logstash_prefix_separator,omitempty" syslog-ng:"ignore"`
 	// Set the suffix for logs in logstash format. Default: `"${YEAR}.${MONTH}.${DAY}"`
 	LogStashSuffix string `json:"logstash_suffix,omitempty" syslog-ng:"ignore"`
+	// This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). (default: false)
+	DiskBuffer *DiskBuffer `json:"disk_buffer,omitempty"`
 }
 
 func (o *ElasticsearchOutput) BeforeRender() {
diff --git a/pkg/sdk/logging/model/syslogng/output/openobserve.go b/pkg/sdk/logging/model/syslogng/output/openobserve.go
@@ -68,6 +68,8 @@ type OpenobserveOutput struct {
 	// Arguments to the `$format-json()` template function.
 	// Default: `"--scope rfc5424 --exclude DATE --key ISODATE @timestamp=${ISODATE}"`
 	Record string `json:"record,omitempty"`
+	// This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). (default: false)
+	DiskBuffer *DiskBuffer `json:"disk_buffer,omitempty"`
 }
 
 func (o *OpenobserveOutput) BeforeRender() {
diff --git a/pkg/sdk/logging/model/syslogng/output/s3.go b/pkg/sdk/logging/model/syslogng/output/s3.go
@@ -74,9 +74,9 @@ type S3Output struct {
 	// The object_key for the S3 server.
 	ObjectKey string `json:"object_key,omitempty"`
 	// Set object_key_timestamp
-	ObjectKeyTimestamp RawString `json:"object_key_timestamp,omitempty"`
+	ObjectKeyTimestamp string `json:"object_key_timestamp,omitempty"`
 	// Template
-	Template RawString `json:"template,omitempty"`
+	Template string `json:"template,omitempty"`
 	// Enable or disable compression. (default: false)
 	Compression *bool `json:"compression,omitempty"`
 	// Set the compression level (1-9). (default: 9)
@@ -105,4 +105,6 @@ type S3Output struct {
 	Retries int `json:"retries,omitempty"`
 	//  Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. (default: 0)
 	Throttle int `json:"throttle,omitempty"`
+	// This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). (default: false)
+	DiskBuffer *DiskBuffer `json:"disk_buffer,omitempty"`
 }
diff --git a/pkg/sdk/logging/model/syslogng/output/splunk_hec.go b/pkg/sdk/logging/model/syslogng/output/splunk_hec.go
@@ -83,4 +83,6 @@ type SplunkHECOutput struct {
 	ExtraQueries []string `json:"extra_queries,omitempty"`
 	// Additional HTTP request content-type option.
 	ContentType string `json:"content_type,omitempty"`
+	// This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). (default: false)
+	DiskBuffer *DiskBuffer `json:"disk_buffer,omitempty"`
 }
diff --git a/pkg/sdk/logging/model/syslogng/output/zz_generated.deepcopy.go b/pkg/sdk/logging/model/syslogng/output/zz_generated.deepcopy.go

Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,8 @@ type ElasticsearchOutput struct {`
`69`	`69`	LogstashPrefixSeparator string `json:"logstash_prefix_separator,omitempty" syslog-ng:"ignore"`
`70`	`70`	// Set the suffix for logs in logstash format. Default: `"${YEAR}.${MONTH}.${DAY}"`
`71`	`71`	LogStashSuffix string `json:"logstash_suffix,omitempty" syslog-ng:"ignore"`
	`72`	`+ // This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). (default: false)`
	`73`	+ DiskBuffer *DiskBuffer `json:"disk_buffer,omitempty"`
`72`	`74`	`}`
`73`	`75`
`74`	`76`	`func (o *ElasticsearchOutput) BeforeRender() {`