From c30dc140ecd5a7a433babf08b1b9daeaaff4cd64 Mon Sep 17 00:00:00 2001 From: Justin McCann Date: Wed, 17 Jul 2024 23:25:11 +0000 Subject: [PATCH] fluentd exporter: Disconnect and retry DNS lookup on socket failure --- .../exporters/fluentd/common/fluentd_common.h | 1 + exporters/fluentd/src/log/fluentd_exporter.cc | 10 ++++++++++ exporters/fluentd/src/trace/fluentd_exporter.cc | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h b/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h index a017fac61..17b24f012 100644 --- a/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h +++ b/exporters/fluentd/include/opentelemetry/exporters/fluentd/common/fluentd_common.h @@ -142,6 +142,7 @@ struct FluentdExporterOptions { TransportFormat format = TransportFormat::kForward; std::string tag = "tag.service"; size_t retry_count = 2; // number of retries before drop + uint32_t retry_delay_ms = 1000; std::string endpoint; bool convert_event_to_trace = false; // convert events to trace. Not used for Logs. diff --git a/exporters/fluentd/src/log/fluentd_exporter.cc b/exporters/fluentd/src/log/fluentd_exporter.cc index 7efb106d5..23833b4a7 100644 --- a/exporters/fluentd/src/log/fluentd_exporter.cc +++ b/exporters/fluentd/src/log/fluentd_exporter.cc @@ -124,6 +124,7 @@ sdk::common::ExportResult FluentdExporter::Export( */ bool FluentdExporter::Send(std::vector &packet) { size_t retryCount = options_.retry_count; + auto retrySleepMs = std::chrono::milliseconds(options_.retry_delay_ms); while (retryCount--) { int error_code = 0; // Check if socket is Okay @@ -135,8 +136,12 @@ bool FluentdExporter::Send(std::vector &packet) { } // Reconnect if not Okay if (!connected_) { + Initialize(); // try a DNS lookup, etc // Establishing socket connection may take time if (!Connect()) { + if (retryCount) { + std::this_thread::sleep_for(retrySleepMs); + } continue; } LOG_DEBUG("socket connected"); @@ -152,10 +157,15 @@ bool FluentdExporter::Send(std::vector &packet) { } LOG_WARN("send failed, retrying %lu ...", retryCount); + Disconnect(); // Retry to connect and/or send + if (retryCount) { + std::this_thread::sleep_for(retrySleepMs); + } } LOG_ERROR("send failed!"); + Disconnect(); return false; } diff --git a/exporters/fluentd/src/trace/fluentd_exporter.cc b/exporters/fluentd/src/trace/fluentd_exporter.cc index 35950d33d..278f206b0 100644 --- a/exporters/fluentd/src/trace/fluentd_exporter.cc +++ b/exporters/fluentd/src/trace/fluentd_exporter.cc @@ -220,6 +220,7 @@ bool FluentdExporter::Connect() { */ bool FluentdExporter::Send(std::vector &packet) { size_t retryCount = options_.retry_count; + auto retrySleepMs = std::chrono::milliseconds(options_.retry_delay_ms); while (retryCount--) { int error_code = 0; // Check if socket is Okay @@ -231,8 +232,12 @@ bool FluentdExporter::Send(std::vector &packet) { } // Reconnect if not Okay if (!connected_) { + Initialize(); // try a DNS lookup, etc // Establishing socket connection may take time if (!Connect()) { + if (retryCount) { + std::this_thread::sleep_for(retrySleepMs); + } continue; } LOG_DEBUG("socket connected"); @@ -248,10 +253,15 @@ bool FluentdExporter::Send(std::vector &packet) { } LOG_WARN("send failed, retrying %u ...", (unsigned int)retryCount); + Disconnect(); // Retry to connect and/or send + if (retryCount) { + std::this_thread::sleep_for(retrySleepMs); + } } LOG_ERROR("send failed!"); + Disconnect(); return false; }