[OneDS] Integrate Event Tracking Mechanism in OneDS (#1115)

shivpratap1992 · web-flow · commit aaa4b65f27dc · 2023-03-20T19:52:45.000-07:00
* Add

* Integrate Event Tracking Mechanism in 1DS

* Fix unnecessary file

* Remove file checked in by mistake

* Update description for macro

* Comment Fix
diff --git a/docs/building-custom-SKU.md b/docs/building-custom-SKU.md
@@ -26,6 +26,7 @@ Build recipe must contain the following preprocessor definitions:
 | HAVE_MAT_ZLIB | on | Use zlib for HTTP requests compression. This option must always be turned on for any high-volume telemetry project |
 | HAVE_MAT_LOGGING | on | Enable internal SDK tracing / debug logging |
 | HAVE_MAT_WIN_LOG | off | Will log statements to disk on windows if trace enabled and HAVE_MAT_LOGGING defined |
+| HAVE_MAT_EVT_TRACEID  | off | Enable event tracking by adding trace-id to http request header on Windows. This is for debugging purpose, and not recommended to be enabled in production. The collector doesn't parse/read this header. As of now, this is meant to be used through the capi, where the http-send handler should remove this header from the event data before sending it to collector. |
 | HAVE_MAT_STORAGE | on | Enable SQLite persistent offline storage |
 | HAVE_MAT_NETDETECT | on | _Win32 Desktop only_: Use NLM COM object for network cost detection on Windows 8+ |
 | HAVE_MAT_SHORT_NS | off | Use short "MAT::" namespace instead of "Microsoft::Applications::Events::" to reduce the .DLL size |
diff --git a/lib/http/HttpRequestEncoder.cpp b/lib/http/HttpRequestEncoder.cpp
@@ -45,6 +45,11 @@ namespace MAT_NS_BEGIN {
         ctx->httpRequest->SetUrl(m_config.GetCollectorUrl());
 
         ctx->httpRequest->GetHeaders().set("Expect", "100-continue");
+        
+#ifdef HAVE_MAT_EVT_TRACEID 
+        ctx->httpRequest->GetHeaders().set("Trace-Id", ctx->traceId);
+#endif //HAVE_MAT_EVT_TRACEID 
+
         ctx->httpRequest->GetHeaders().set("SDK-Version", PAL::getSdkVersion());
         ctx->httpRequest->GetHeaders().set("Client-Id", "NO_AUTH");
         ctx->httpRequest->GetHeaders().set("Content-Type", "application/bond-compact-binary");
diff --git a/lib/include/mat/config-MIP.h b/lib/include/mat/config-MIP.h
@@ -13,6 +13,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 #define HAVE_MAT_WIN_LOG
+#define HAVE_MAT_EVT_TRACEID
 #define HAVE_MAT_STORAGE
 #if defined(_WIN32)
 #define WIN_DESKTOP
diff --git a/lib/include/mat/config-MSIPC.h b/lib/include/mat/config-MSIPC.h
@@ -16,6 +16,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 #define WIN_DESKTOP
 #define HAVE_MAT_DEFAULT_HTTP_CLIENT
diff --git a/lib/include/mat/config-compact-dll.h b/lib/include/mat/config-compact-dll.h
@@ -14,6 +14,7 @@
 #define HAVE_MAT_ZLIB
 /* #define HAVE_MAT_LOGGING    */
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 /* #define HAVE_MAT_STORAGE    */
 /* #define HAVE_MAT_NETDETECT  */
 /* #define HAVE_MAT_SHORT_NS   */
diff --git a/lib/include/mat/config-compact-exp.h b/lib/include/mat/config-compact-exp.h
@@ -14,6 +14,7 @@
 #define HAVE_MAT_ZLIB
 /* #define HAVE_MAT_LOGGING    */
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 /* #define HAVE_MAT_NETDETECT  */
 /* #define HAVE_MAT_SHORT_NS   */
diff --git a/lib/include/mat/config-compact-min.h b/lib/include/mat/config-compact-min.h
@@ -14,6 +14,7 @@
 /* #define HAVE_MAT_ZLIB       */
 /* #define HAVE_MAT_LOGGING    */
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 /* #define HAVE_MAT_STORAGE    */
 /* #define HAVE_MAT_NETDETECT  */
 /* #define HAVE_MAT_SHORT_NS   */
diff --git a/lib/include/mat/config-compact-noutc.h b/lib/include/mat/config-compact-noutc.h
@@ -14,6 +14,7 @@
 /* #define HAVE_MAT_ZLIB        */
 /* #define HAVE_MAT_LOGGING     */
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 /* #define HAVE_MAT_STORAGE     */
 /* #define HAVE_MAT_NETDETECT   */
 #define HAVE_MAT_SHORT_NS
diff --git a/lib/include/mat/config-compact.h b/lib/include/mat/config-compact.h
@@ -14,6 +14,7 @@
 /* #define HAVE_MAT_ZLIB        */
 /* #define HAVE_MAT_LOGGING     */
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 /* #define HAVE_MAT_STORAGE     */
 /* #define HAVE_MAT_NETDETECT   */
 #define HAVE_MAT_SHORT_NS
diff --git a/lib/include/mat/config-default-cs4.h b/lib/include/mat/config-default-cs4.h
@@ -24,6 +24,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 #define HAVE_MAT_DEFAULT_HTTP_CLIENT
 #define HAVE_MAT_LIVEEVENTINSPECTOR
diff --git a/lib/include/mat/config-default-exp.h b/lib/include/mat/config-default-exp.h
@@ -22,6 +22,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 #define HAVE_MAT_DEFAULT_HTTP_CLIENT
 #define HAVE_MAT_LIVEEVENTINSPECTOR
diff --git a/lib/include/mat/config-default.h b/lib/include/mat/config-default.h
@@ -24,6 +24,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 #define HAVE_MAT_DEFAULT_HTTP_CLIENT
 #define HAVE_MAT_LIVEEVENTINSPECTOR
diff --git a/lib/include/mat/config-net40.h b/lib/include/mat/config-net40.h
@@ -16,6 +16,7 @@
 #define HAVE_MAT_ZLIB
 #define HAVE_MAT_LOGGING
 /* #define HAVE_MAT_WIN_LOG     */
+/* #define HAVE_MAT_EVT_TRACEID     */
 #define HAVE_MAT_STORAGE
 /*
 #if defined(_WIN32) && !defined(_WINRT_DLL)
diff --git a/lib/include/public/IOfflineStorage.hpp b/lib/include/public/IOfflineStorage.hpp
@@ -33,13 +33,22 @@ namespace MAT_NS_BEGIN {
         StorageBlob     blob;
         int             retryCount = 0;
         int64_t         reservedUntil = 0;
+#ifdef HAVE_MAT_EVT_TRACEID 
+        std::string     traceId;
+#endif // HAVE_MAT_EVT_TRACEID
 
         StorageRecord()
         {}
 
+#ifdef HAVE_MAT_EVT_TRACEID
+        StorageRecord(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence, std::string traceId)
+            : id(id), tenantToken(tenantToken), latency(latency), persistence(persistence), traceId(traceId)
+        {}
+#else
         StorageRecord(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence)
             : id(id), tenantToken(tenantToken), latency(latency), persistence(persistence)
         {}
+#endif // HAVE_MAT_EVT_TRACEID
 
         StorageRecord(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence,
             int64_t timestamp, std::vector<uint8_t>&& blob, int retryCount = 0, int64_t reservedUntil = 0)
diff --git a/lib/packager/Packager.cpp b/lib/packager/Packager.cpp
@@ -48,6 +48,10 @@ namespace MAT_NS_BEGIN {
             LOG_TRACE("Adding event %s:%s, size %u bytes",
                 tenantTokenToId(record.tenantToken).c_str(), record.id.c_str(), static_cast<unsigned>(record.blob.size()));
 
+            #ifdef HAVE_MAT_EVT_TRACEID
+                        ctx->traceId = record.traceId;
+            #endif // HAVE_MAT_EVT_TRACEID
+
             std::string const& tenantToken = m_forcedTenantToken.empty() ? record.tenantToken : m_forcedTenantToken;
             auto it = ctx->packageIds.lower_bound(tenantToken);
             if (it == ctx->packageIds.end() || it->first != tenantToken)
diff --git a/lib/system/Contexts.hpp b/lib/system/Contexts.hpp
@@ -32,12 +32,21 @@ namespace MAT_NS_BEGIN {
         {
         }
 
+#ifdef HAVE_MAT_EVT_TRACEID   
+        IncomingEventContext(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence, ::CsProtocol::Record* source)
+            : source(source),
+            record{ id, tenantToken, latency, persistence, (source != nullptr) ? source->cV : "" },
+	    policyBitFlags(0)
+        {
+        }
+#else
         IncomingEventContext(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence, ::CsProtocol::Record* source)
             : source(source),
             record{ id, tenantToken, latency, persistence },
 	    policyBitFlags(0)
         {
         }
+#endif
 
         virtual ~IncomingEventContext()
         {
@@ -88,6 +97,9 @@ namespace MAT_NS_BEGIN {
         unsigned                             maxUploadSize = 0;
         EventLatency                         latency = EventLatency_Unspecified;
         std::map<std::string, size_t>        packageIds;
+#ifdef HAVE_MAT_EVT_TRACEID  
+        std::string                          traceId;
+#endif
         std::map<std::string, std::string>   recordIdsAndTenantIds;
         std::vector<int64_t>                 recordTimestamps;
         unsigned                             maxRetryCountSeen = 0;

Original file line number	Diff line number	Diff line change
`@@ -32,12 +32,21 @@ namespace MAT_NS_BEGIN {`
`32`	`32`	`{`
`33`	`33`	`}`
`34`	`34`
	`35`	`+#ifdef HAVE_MAT_EVT_TRACEID`
	`36`	`+ IncomingEventContext(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence, ::CsProtocol::Record* source)`
	`37`	`+ : source(source),`
	`38`	`+ record{ id, tenantToken, latency, persistence, (source != nullptr) ? source->cV : "" },`
	`39`	`+ policyBitFlags(0)`
	`40`	`+ {`
	`41`	`+ }`
	`42`	`+#else`
`35`	`43`	`IncomingEventContext(std::string const& id, std::string const& tenantToken, EventLatency latency, EventPersistence persistence, ::CsProtocol::Record* source)`
`36`	`44`	`: source(source),`
`37`	`45`	`record{ id, tenantToken, latency, persistence },`
`38`	`46`	`policyBitFlags(0)`
`39`	`47`	`{`
`40`	`48`	`}`
	`49`	`+#endif`
`41`	`50`
`42`	`51`	`virtual ~IncomingEventContext()`
`43`	`52`	`{`
`@@ -88,6 +97,9 @@ namespace MAT_NS_BEGIN {`
`88`	`97`	`unsigned maxUploadSize = 0;`
`89`	`98`	`EventLatency latency = EventLatency_Unspecified;`
`90`	`99`	`std::map<std::string, size_t> packageIds;`
	`100`	`+#ifdef HAVE_MAT_EVT_TRACEID`
	`101`	`+ std::string traceId;`
	`102`	`+#endif`
`91`	`103`	`std::map<std::string, std::string> recordIdsAndTenantIds;`
`92`	`104`	`std::vector<int64_t> recordTimestamps;`
`93`	`105`	`unsigned maxRetryCountSeen = 0;`