Add call to data collector

Signed-off-by: Thomas Poignant <[email protected]>

Author: thomaspoignant

libs/providers/go-feature-flag/src/lib/go-feature-flag-provider.ts

@@ -9,20 +9,22 @@ import {
   TypeMismatchError,
 } from '@openfeature/js-sdk';
 import axios from 'axios';
-import { transformContext } from './context-transformer';
-import { ProxyNotReady } from './errors/proxyNotReady';
-import { ProxyTimeout } from './errors/proxyTimeout';
-import { UnknownError } from './errors/unknownError';
-import { Unauthorized } from './errors/unauthorized';
+import {transformContext} from './context-transformer';
+import {ProxyNotReady} from './errors/proxyNotReady';
+import {ProxyTimeout} from './errors/proxyTimeout';
+import {UnknownError} from './errors/unknownError';
+import {Unauthorized} from './errors/unauthorized';
 import {
+  DataCollectorRequest,
+  DataCollectorResponse,
+  FeatureEvent,
   GoFeatureFlagProviderOptions,
   GoFeatureFlagProxyRequest,
   GoFeatureFlagProxyResponse,
   GoFeatureFlagUser,
 } from './model';
 import Receptacle from 'receptacle';
 
-
 // GoFeatureFlagProvider is the official Open-feature provider for GO Feature Flag.
 export class GoFeatureFlagProvider implements Provider {
   metadata = {
@@ -33,29 +35,90 @@ export class GoFeatureFlagProvider implements Provider {
   private readonly endpoint: string;
   // timeout in millisecond before we consider the request as a failure
   private readonly timeout: number;
+
+
   // cache contains the local cache used in the provider to avoid calling the relay-proxy for every evaluation
-  private cache?: Receptacle<ResolutionDetails<any>>;
+  private readonly cache?: Receptacle<ResolutionDetails<any>>;
+  // bgSchedulerId contains the id of the setInterval that is running.
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  private bgScheduler?: NodeJS.Timer;
+
+  // dataCollectorBuffer contains all the FeatureEvents that we need to send to the relay-proxy for data collection.
+  private dataCollectorBuffer?: FeatureEvent<any>[];
+  // dataCollectorMetadata are the metadata used when calling the data collector endpoint
+  private readonly dataCollectorMetadata: Record<string, string> = {
+    provider: 'open-feature-js-sdk',
+  };
 
   // cacheTTL is the time we keep the evaluation in the cache before we consider it as obsolete.
   // If you want to keep the value forever you can set the FlagCacheTTL field to -1
-  private readonly cacheTTL?: number
+  private readonly cacheTTL?: number;
+
+  // dataFlushInterval interval time (in millisecond) we use to call the relay proxy to collect data.
+  private readonly dataFlushInterval: number;
 
   constructor(options: GoFeatureFlagProviderOptions) {
     this.timeout = options.timeout || 0; // default is 0 = no timeout
     this.endpoint = options.endpoint;
     this.cacheTTL = options.flagCacheTTL !== undefined && options.flagCacheTTL !== 0 ? options.flagCacheTTL : 1000 * 60;
+    this.dataFlushInterval = options.dataFlushInterval || 1000 * 60;
 
     // Add API key to the headers
     if (options.apiKey) {
       axios.defaults.headers.common['Authorization'] = `Bearer ${options.apiKey}`;
     }
 
-    if(!options.disableCache) {
-      const cacheSize = options.flagCacheSize!== undefined && options.flagCacheSize !== 0 ? options.flagCacheSize : 10000;
+    if (!options.disableCache) {
+      const cacheSize = options.flagCacheSize !== undefined && options.flagCacheSize !== 0 ? options.flagCacheSize : 10000;
       this.cache = new Receptacle<ResolutionDetails<any>>({max: cacheSize})
     }
   }
 
+
+  async onClose() {
+    if (this.cache !== undefined && this.bgScheduler !== undefined) {
+      // we stop the background task to call the data collector endpoint
+      clearInterval(this.bgScheduler);
+      // We call the data collector with what is still in the buffer.
+      await this.callGoffDataCollection()
+    }
+  }
+
+  async initialize() {
+    if (this.cache !== undefined) {
+      this.bgScheduler = setInterval(async () => await this.callGoffDataCollection(), this.dataFlushInterval)
+      this.dataCollectorBuffer = []
+    }
+  }
+
+  async callGoffDataCollection() {
+    if (this.dataCollectorBuffer?.length === 0) {
+      return
+    }
+
+    const dataToSend = structuredClone(this.dataCollectorBuffer);
+    this.dataCollectorBuffer = []
+
+    const request: DataCollectorRequest<boolean> = {events: dataToSend, meta: this.dataCollectorMetadata,}
+    const endpointURL = new URL(this.endpoint);
+    endpointURL.pathname = 'v1/data/collector';
+
+    try {
+      await axios.post<DataCollectorResponse>(endpointURL.toString(), request, {
+        headers: {
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+        timeout: this.timeout,
+      });
+    } catch (e) {
+      // TODO : add a log here
+      // if we have an issue calling the collector we put the data back in the buffer
+      this.dataCollectorBuffer = [...this.dataCollectorBuffer, ...dataToSend]
+    }
+  }
+
   /**
    * resolveBooleanEvaluation is calling the GO Feature Flag relay-proxy API and return a boolean value.
    * @param flagKey - name of your feature flag key.
@@ -136,7 +199,7 @@ export class GoFeatureFlagProvider implements Provider {
    * @param flagKey - name of your feature flag key.
    * @param defaultValue - default value is used if we are not able to evaluate the flag for this user.
    * @param context - the context used for flag evaluation.
-   * @return {Promise<ResolutionDetails<U>>} An object containing the result of the flag evaluation by GO Feature Flag.
+   * @return {Promise<ResolutionDetails<U extends JsonValue>>} An object containing the result of the flag evaluation by GO Feature Flag.
    * @throws {ProxyNotReady} When we are not able to communicate with the relay-proxy
    * @throws {ProxyTimeout} When the HTTP call is timing out
    * @throws {UnknownError} When an unknown error occurs
@@ -182,12 +245,25 @@ export class GoFeatureFlagProvider implements Provider {
     if (this.cache !== undefined) {
       const cacheValue = this.cache.get(cacheKey);
       if (cacheValue !== null) {
-        // console.log(this.cache.get(cacheKey).)
+        // Building and inserting an event to the data collector buffer,
+        // so we will be able to bulk send these events to GO Feature Flag.
+        const dataCollectorEvent: FeatureEvent<T> = {
+          contextKind: user.anonymous ? 'anonymousUser' : 'user',
+          kind: 'feature',
+          creationDate: Math.round(Date.now() / 1000),
+          default: false,
+          key: flagKey,
+          value: cacheValue.value,
+          variation: cacheValue.variant || 'SdkDefault',
+          userKey: user.key,
+        }
+        this.dataCollectorBuffer?.push(dataCollectorEvent)
+
         return cacheValue;
       }
     }
 
-    const request: GoFeatureFlagProxyRequest<T> = { user, defaultValue };
+    const request: GoFeatureFlagProxyRequest<T> = {user, defaultValue};
     // build URL to access to the endpoint
     const endpointURL = new URL(this.endpoint);
     endpointURL.pathname = `v1/feature/${flagKey}/eval`;
@@ -249,12 +325,11 @@ export class GoFeatureFlagProvider implements Provider {
     if (apiResponseData.reason === StandardResolutionReasons.DISABLED) {
       // we don't set a variant since we are using the default value, and we are not able to know
       // which variant it is.
-      return { value: defaultValue, reason: apiResponseData.reason };
+      return {value: defaultValue, reason: apiResponseData.reason};
     }
 
     const sdkResponse: ResolutionDetails<T> = {
       value: apiResponseData.value,
-      reason: apiResponseData.reason?.toString() || 'UNKNOWN'
       variant: apiResponseData.variationType,
       reason: apiResponseData.reason?.toString() || 'UNKNOWN',
       flagMetadata: apiResponseData.metadata || undefined,
@@ -265,9 +340,8 @@ export class GoFeatureFlagProvider implements Provider {
       sdkResponse.errorCode = ErrorCode.GENERAL;
     }
 
-    if(this.cache!==undefined && apiResponseData.cacheable){
-      console.log('add cache', cacheKey)
-      if (this.cacheTTL === -1){
+    if (this.cache !== undefined && apiResponseData.cacheable) {
+      if (this.cacheTTL === -1) {
         this.cache.set(cacheKey, sdkResponse)
       } else {
         this.cache.set(cacheKey, sdkResponse, {ttl: this.cacheTTL, refresh: false})

libs/providers/go-feature-flag/src/lib/model.ts

@@ -67,10 +67,38 @@ export interface GoFeatureFlagProviderOptions {
   // If you want to keep the value forever you can set the FlagCacheTTL field to -1
   // default: 1 minute
   flagCacheTTL?: number
+
+  // dataFlushInterval (optional) interval time (in millisecond) we use to call the relay proxy to collect data.
+  // The parameter is used only if the cache is enabled, otherwise the collection of the data is done directly
+  // when calling the evaluation API.
+  // default: 1 minute
+  dataFlushInterval?: number
 }
 
 // GOFeatureFlagResolutionReasons allows to extends resolution reasons
 export declare enum GOFeatureFlagResolutionReasons {}
 
 // GOFeatureFlagErrorCode allows to extends error codes
 export declare enum GOFeatureFlagErrorCode {}
+
+
+export interface DataCollectorRequest<T> {
+  events: FeatureEvent<T>[];
+  meta: Record<string, string>;
+}
+
+export interface FeatureEvent<T> {
+  contextKind: string;
+  creationDate: number;
+  default: boolean;
+  key: string;
+  kind: string;
+  userKey: string;
+  value: T;
+  variation: string;
+  version?: string;
+}
+
+export interface DataCollectorResponse {
+  ingestedContentCount: number;
+}

libs/providers/go-feature-flag/src/lib/test.spec.ts

@@ -1,11 +1,25 @@
 import {OpenFeature} from '@openfeature/js-sdk';
-import { GoFeatureFlagProvider } from './go-feature-flag-provider';
+import {GoFeatureFlagProvider} from './go-feature-flag-provider';
+
+jest.setTimeout(1000 * 60 * 60);
 it('XXX', async () => {
-  const goff = new GoFeatureFlagProvider({endpoint:'http://localhost:1031', flagCacheTTL: 3000, flagCacheSize:1})
+  const goff = new GoFeatureFlagProvider({
+    endpoint: 'http://localhost:1031',
+    flagCacheTTL: 3000,
+    flagCacheSize: 1,
+    dataFlushInterval: 1000000
+  })
   OpenFeature.setProvider(goff)
   const cli = OpenFeature.getClient()
   console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
-  await new Promise((r) => setTimeout(r, 3100));
   console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  await new Promise((r) => setTimeout(r, 1000));
+  console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  console.log(await cli.getBooleanDetails('bool_targeting_match', false, {targetingKey: 'my-key'}))
+  await OpenFeature.close()
 
+  // await new Promise((r) => setTimeout(r, 1000 * 60));
 })

libs/providers/go-feature-flag/tsconfig.json

@@ -7,7 +7,10 @@
     "noImplicitOverride": true,
     "noPropertyAccessFromIndexSignature": true,
     "noImplicitReturns": true,
-    "noFallthroughCasesInSwitch": true
+    "noFallthroughCasesInSwitch": true,
+    "types": [
+      "node"
+    ]
   },
   "files": [],
   "include": [],
@@ -18,5 +21,5 @@
     {
       "path": "./tsconfig.spec.json"
     }
-  ]
+  ],
 }