Otel changes for adding host dimension (paypal#93)

* Occ config logging (paypal#394) * occ configurations logging * cal event success * adding cal data * added TODOs * Remove log_occ_confogs.go * Remove testing files * source of configs - files * whitelist format change * code clean up * code review changes-1 * CR fixes * CR fixes * Delete tests/unittest/config_logging/main_test.go * clean up * Merge branch 'occ-config-logging' of /Users/simmidisetty/Documents/GitHub/OpenSourceHera/src/github.com/paypal/hera with conflicts. * test for config logging * removing test changes * tests for all cases * test * making minor changes for logging feature specific data * changes for incorporate review comments --------- Co-authored-by: simmidisetty <[email protected]> Co-authored-by: Rajesh S <[email protected]> * adding host dimension in data-point level --------- Co-authored-by: satyakamala03 <[email protected]> Co-authored-by: simmidisetty <[email protected]> Co-authored-by: Rajesh S <[email protected]>
rasamala83 · Jul 25, 2024 · db14446 · db14446
1 parent 629c290
commit db14446
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 26 deletions.
diff --git a/tests/unittest/otel_basic/main_test.go b/tests/unittest/otel_basic/main_test.go
@@ -28,7 +28,7 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) {
 	appcfg["rac_sql_interval"] = "0"
 	appcfg["child.executable"] = "mysqlworker"
 	appcfg["enable_otel"] = "true"
-	appcfg["otel_resolution_time_in_sec"] = "10"
+	appcfg["otel_resolution_time_in_sec"] = "1"
 	opscfg := make(map[string]string)
 	opscfg["opscfg.default.server.max_connections"] = "3"
 	opscfg["opscfg.default.server.log_level"] = "5"
@@ -108,9 +108,9 @@ func TestOTELMetricsBasic(t *testing.T) {
 	if count < 1 {
 		t.Fatalf("OTEL event should contain application as hera-test")
 	}
-	initCount := testutil.RegexCountFile("\"name\":\"pp.occ.init_connection.count\"", logFilePath)
+	initCount := testutil.RegexCountFile("\"name\":\"pp.occ.init_connection.cnt\"", logFilePath)
 	if initCount < 1 {
-		t.Fatalf("OTEL event should contain metric name pp.occ.init_connection.count")
+		t.Fatalf("OTEL event should contain metric name pp.occ.init_connection.cnt")
 	}
 	tagsCount := testutil.RegexCountFile("{\"key\":\"InstanceId\",\"value\":{\"intValue\":\"0\"}},{\"key\":\"ShardId\",\"value\":{\"intValue\":\"0\"}},{\"key\":\"WorkerType\",\"value\":{\"intValue\":\"0\"}",
 		logFilePath)

diff --git a/tests/unittest/otel_remote_endpoint_tls/main_test.go b/tests/unittest/otel_remote_endpoint_tls/main_test.go
@@ -102,30 +102,22 @@ func TestOTELMetricsRemoteEndPointWithTLS(t *testing.T) {
 		t.Fatalf("Expected 1 row")
 	}
 
-	time.Sleep(10 * time.Second)
+	time.Sleep(15 * time.Second)
 	rows.Close()
 	stmt.Close()
 
-	time.Sleep(10 * time.Second)
+	time.Sleep(15 * time.Second)
 	publishingErrors := testutil.RegexCountFile("otel publishing error", "hera.log")
 	if publishingErrors > 1 {
 		t.Fatalf("should not fail while publishing metrics remote host")
 	}
 
-	time.Sleep(5 * time.Second)
 	calPublishingErrors := testutil.RegexCountFile("failed to send metrics", "cal.log")
 	if calPublishingErrors > 1 {
 		t.Fatalf("should not fail while publishing metrics remote host")
 	}
 
 	cancel()
 	conn.Close()
-<<<<<<< HEAD
-
-	for counter := 0; counter < 1000; counter++ {
-		time.Sleep(10 * time.Second)
-	}
-=======
->>>>>>> otel_logging_changes
 	logger.GetLogger().Log(logger.Debug, "TestOTELMetricsRemoteEndPointWithTLS done  -------------------------------------------------------------")
 }
diff --git a/utility/logger/otel/defs.go b/utility/logger/otel/defs.go
@@ -36,16 +36,19 @@ const (
 )
 
 const (
-	Target             = string("target")
-	Endpoint           = string("target_ip_port")
-	TLS_version        = string("tls_version")
-	Application        = string("Application")
-	ShardId            = string("ShardId")
-	WorkerType         = string("WorkerType")
-	InstanceId         = string("InstanceId")
-	Datapoints         = string("datapoints")
-	otelSource         = string("otel")
-	OccWorkerParamName = string("occ_worker")
+	Target               = string("target")
+	Endpoint             = string("target_ip_port")
+	TLS_version          = string("tls_version")
+	ApplicationDimName   = string("application")
+	ShardId              = string("ShardId")
+	WorkerType           = string("WorkerType")
+	InstanceId           = string("InstanceId")
+	Datapoints           = string("datapoints")
+	OtelSourceName       = string("source")
+	otelSource           = string("otel")
+	OccWorkerParamName   = string("occ_worker")
+	HostDimensionName    = string("host")
+	ContainerHostDimName = string("container_host")
 )
 
 const OtelInstrumentationVersion string = "v1.0"
@@ -99,6 +102,8 @@ type StateLogMetrics struct {
 	//Statelog metrics configuration data
 	metricsConfig stateLogMetricsConfig
 
+	hostname string
+
 	meter metric.Meter
 
 	//Channel to receive statelog data

diff --git a/utility/logger/otel/logger.go b/utility/logger/otel/logger.go
@@ -386,9 +386,9 @@ func getResourceInfo(appName string) *resource.Resource {
 
 	// Create a slice to hold the attributes
 	attributes := []attribute.KeyValue{
-		attribute.String("container_host", hostname),
-		attribute.String("application", appName),
-		attribute.String("source", otelSource),
+		attribute.String(ContainerHostDimName, hostname),
+		attribute.String(ApplicationDimName, appName),
+		attribute.String(OtelSourceName, otelSource),
 	}
 
 	environment, isEnvPresent := os.LookupEnv("ENVIRONMENT")

diff --git a/utility/logger/otel/state_logger.go b/utility/logger/otel/state_logger.go
@@ -8,6 +8,7 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
+	"os"
 	"sync"
 	"time"
 )
@@ -71,11 +72,16 @@ func StartMetricsCollection(totalWorkersCount int, opt ...StateLogOption) error
 	var err error
 	//Registers instrumentation for metrics
 	registerStateMetrics.Do(func() {
+		hostName, hostErr := os.Hostname()
+		if hostErr != nil {
+			logger.GetLogger().Log(logger.Alert, "Failed to fetch hostname for current container", err)
+		}
 		//Initialize state-log metrics
 		metricsStateLogger = &StateLogMetrics{
 			meter: stateLogMetricsConfig.MeterProvider.Meter(StateLogMeterName,
 				metric.WithInstrumentationVersion(OtelInstrumentationVersion)),
 			metricsConfig:  stateLogMetricsConfig,
+			hostname:       hostName,
 			mStateDataChan: make(chan *WorkersStateData, totalWorkersCount*otelconfig.OTelConfigData.ResolutionTimeInSec*2), //currently OTEL polling interval hardcoded as 10. Size of bufferred channel = totalWorkersCount * pollingInterval * 2,
 			doneCh:         make(chan struct{}),
 		}
@@ -344,10 +350,18 @@ mainloop:
 					minKey := key + "Min"
 					stateLogsData[keyName][key] = value
 					//check max update max value
+					_, keyPresent := stateLogsData[keyName][maxKey]
+					if !keyPresent {
+						stateLogsData[keyName][maxKey] = value
+					}
 					if stateLogsData[keyName][maxKey] < value {
 						stateLogsData[keyName][maxKey] = value
 					}
 					//Min value
+					_, keyPresent = stateLogsData[keyName][minKey]
+					if !keyPresent {
+						stateLogsData[keyName][minKey] = value
+					}
 					if stateLogsData[keyName][minKey] > value {
 						stateLogsData[keyName][minKey] = value
 					}
@@ -380,6 +394,7 @@ func (stateLogMetrics *StateLogMetrics) sendMetricsDataToCollector(observer metr
 			attribute.Int(WorkerType, int(aggStatesData[WorkerType])),
 			attribute.Int(InstanceId, int(aggStatesData[InstanceId])),
 			attribute.String(OccWorkerParamName, *stateLogTitle),
+			attribute.String(HostDimensionName, stateLogMetrics.hostname),
 		}
 		//Observe states data
 		//1. Worker States