Merge pull request #19456 from rpastrana/HPCC-33298-JTrace-Sampling

HPCC-33298 JTrace support sampling configuration

Reviewed-by: Jake Smith <[email protected]>
Reviewed-by: Gavin Halliday <[email protected]>
Merged-by: Gavin Halliday <[email protected]>

Author: ghalliday

helm/examples/tracing/README.md

@@ -13,6 +13,10 @@ All configuration options detailed here are part of the HPCC Systems Helm chart,
 - optAlwaysCreateTraceIds - If true components generate trace/span ids if none are provided by the remote caller.
 - enableDefaultLogExporter - If true, creates a trace exporter outputting to the log using the default options
 - enableOTELDebugLogging - If true, OTel library logging level set to debug, otherwise warning
+- sampling - Defines head sampling strategy. Decision to sample or drop a span or trace is not made by inspecting the trace as a whole. https://opentelemetry.io/docs/concepts/sampling/
+  - type "AlwaysOff" | "AlwaysOn" | "Ratio"
+  - ratio - Required if Ratio sampling type enabled. Represents the ratio of trace/spans to sample Must be a numeric value betwen 0.0 and 1.0. 
+  - parentBased - Optional boolean. Determines if the sampling policy honors the remote root span sampled flag
 - resourceAttributes: - Defines OTel specific resource attribute configuration values
                         which are appended to the runtime OTEL_RESOURCE_ATTRIBUTES. See OTel doc: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration
   - deploymentEnvironment - Defines deployment.environment, which is used to specify

helm/examples/tracing/baremetal-otlp-http-localhost-sample.xml

@@ -5,6 +5,7 @@
    <exporters consoleDebug="true" endpoint="http://localhost:4318/v1/traces" type="OTLP-HTTP">
     <batch enabled="false" maxQueueSize="4095" scheduledDelayMillis="6001" maxExportBatchSize="511"/>
    </exporters>
+   <sampling type="Ratio" ratio="0.1" parentBased="true"/>
   </tracing>
  </Software>
 </Environment>

helm/examples/tracing/otlp-http-collector-default-sampled.yaml

@@ -0,0 +1,10 @@
+global:
+  tracing:
+    sampling:
+      type: Ratio       #Head sampling based on simple ratio
+      ratio: 0.1        #only sample 10% of traces/spans
+      parentBased: true #Sampling strategy based on parent's sampling
+    exporters:
+    - type: OTLP-HTTP
+      endpoint: "localhost:4318/v1/traces"
+      consoleDebug: true

helm/hpcc/values.schema.json

@@ -1171,6 +1171,25 @@
           "type": "boolean",
           "description": "If true, sets OTEL library logging to debug level (otherwise set to warning)"
         },
+        "sampling": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": ["AlwaysOff", "AlwaysOn", "Ratio"],
+              "description": "Name of the Head Sampling type AlwaysOff|AlwaysOn|Ratio"
+            },
+            "ratio" : {
+              "type": "number",
+              "description": "Required if sampling type set to Ratio - Represents the sampling ratio value."
+            },
+            "parentBased" : {
+              "type": "boolean",
+              "description": "Optional - Determines if the sampling policy honors the remote root span sampled flag."
+            },
+            "additionalProperties": false
+          }
+        },
         "exporters": {
           "type": "array",
           "description": "List of trace exporters",

system/jlib/jtrace.cpp

@@ -23,6 +23,12 @@
 #include "opentelemetry/sdk/trace/tracer_context_factory.h" //opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors));
 #include "opentelemetry/sdk/trace/simple_processor_factory.h"
 #include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
+#include <opentelemetry/sdk/trace/samplers/always_on_factory.h>
+#include <opentelemetry/sdk/trace/samplers/always_off_factory.h>
+#include <opentelemetry/sdk/trace/samplers/trace_id_ratio_factory.h>
+#include <opentelemetry/sdk/trace/samplers/trace_id_ratio.h>
+#include <opentelemetry/sdk/trace/samplers/always_off.h>
+#include <opentelemetry/sdk/trace/samplers/parent.h>
 #include "opentelemetry/exporters/ostream/span_exporter_factory.h"// auto exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create();
 #include "opentelemetry/exporters/ostream/common_utils.h"
 #include "opentelemetry/exporters/memory/in_memory_span_exporter_factory.h"
@@ -1382,6 +1388,50 @@ std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor> CTraceManager::createP
     return opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter));
 }
 
+static std::unique_ptr<opentelemetry::sdk::trace::Sampler> createSampler(IPropertyTree * samplerTree)
+{
+    std::unique_ptr<opentelemetry::sdk::trace::Sampler> sampler;
+
+    if (samplerTree)
+    {
+        const char * samplerType = samplerTree->queryProp("@type");
+        if (!isEmptyString(samplerType))
+        {
+            if (streq(samplerType, "AlwaysOff"))
+            {
+                sampler.reset(new opentelemetry::sdk::trace::AlwaysOffSampler());
+            }
+            else if (streq(samplerType, "AlwaysOn"))
+            {
+                sampler.reset(new opentelemetry::sdk::trace::AlwaysOnSampler());
+            }
+            else if (streq(samplerType,"Ratio"))
+            {
+                double ratio = samplerTree->getPropReal("@ratio", -1.0);
+                if (ratio >= 0.0 && ratio <= 1.0)
+                {
+                    sampler.reset(new opentelemetry::sdk::trace::TraceIdRatioBasedSampler(ratio));
+                }
+                else
+                {
+                    OERRLOG("JTrace invalid ratio sampling configuration. Ratio must be between 0.0 and 1.0");
+                }
+            }
+            else
+            {
+                WARNLOG("JTrace initialization: Invalid sampling type configured: '%s'", samplerType);
+            }
+
+            if (sampler && samplerTree->getPropBool("@parentBased", true))
+            {
+                return std::unique_ptr<opentelemetry::sdk::trace::ParentBasedSampler>(new opentelemetry::sdk::trace::ParentBasedSampler(std::move(sampler)));
+            }
+        }
+    }
+
+    return sampler;
+}
+
 void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * traceConfig)
 {
     /*
@@ -1399,6 +1449,7 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t
         };
 
     std::vector<std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor>> processors;
+    std::unique_ptr<opentelemetry::sdk::trace::Sampler> sampler;
 
     bool enableOTELDebugLogging = false;
 
@@ -1407,6 +1458,8 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t
     bool enableDefaultLogExporter = isDebugBuild();
     if (traceConfig)
     {
+        sampler = createSampler(traceConfig->queryPropTree("sampling"));
+
         IPropertyTree * resourceAttributesTree = traceConfig->queryPropTree("resourceAttributes");
         if (resourceAttributesTree)
         {
@@ -1433,6 +1486,11 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t
         enableDefaultLogExporter = traceConfig->getPropBool("@enableDefaultLogExporter", enableDefaultLogExporter);
     }
 
+    if (!sampler) 
+    {
+        sampler = std::unique_ptr<opentelemetry::sdk::trace::AlwaysOnSampler>(new opentelemetry::sdk::trace::AlwaysOnSampler);
+    }
+
     if (enableDefaultLogExporter)
     {
         //Simple option to create logging to the log file - primarily to aid developers.
@@ -1442,9 +1500,9 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t
 
     auto jtraceResource = opentelemetry::sdk::resource::Resource::Create(resourceAtts);
 
-    // Default is an always-on sampler.
     std::unique_ptr<opentelemetry::sdk::trace::TracerContext> context =
-        opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors), jtraceResource);
+        opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors), jtraceResource, std::move(sampler));
+
     std::shared_ptr<opentelemetry::trace::TracerProvider> provider =
         opentelemetry::sdk::trace::TracerProviderFactory::Create(std::move(context));
 
@@ -1468,9 +1526,14 @@ Expected Configuration format:
         disabled: true                  #optional - disable OTel tracing
         alwaysCreateGlobalIds : false   #optional - should global ids always be created?
         alwaysCreateTraceIds            #optional - should trace ids always be created?
-        exporters:                       #optional - Controls how trace data is exported/reported
-        -   type: OTLP                    #OS|OTLP|Prometheus|JLOG
-            endpoint: "localhost:4317"    #exporter specific key/value pairs
+        sampling:                       #optional - controls how traces are either suppressed or sampled
+          type:                         #"AlwaysOff" | "AlwaysOn" | "Ratio"
+          ratio:                        #optional - Required if Ratio sampling used, represents the sampling ratio [0.0 - 1.0]
+          parentBased:                  #optional - Sets OTel's parentbased sampling option as defined here:
+                                        #           https://opentelemetry.io/docs/languages/erlang/sampling/#parentbasedsampler
+        exporters:                      #optional - Controls how trace data is exported/reported
+        -   type: OTLP                  #OS|OTLP|Prometheus|JLOG
+            endpoint: "localhost:4317"  #exporter specific key/value pairs
             useSslCredentials: true
             sslCredentialsCACcert: "ssl-certificate"
             batch:                        #optional - Controls span processing style