feat: Add support for Payload Filtering

Author: keelerm84

contract-tests/index.js

@@ -27,6 +27,7 @@ app.get('/', (req, res) => {
       'all-flags-with-reasons',
       'tags',
       'big-segments',
+      'filtering',
       'user-type',
       'migrations',
       'event-sampling',

contract-tests/sdkClientEntity.js

@@ -26,11 +26,13 @@ export function makeSdkConfig(options, tag) {
   if (options.streaming) {
     cf.streamUri = options.streaming.baseUri;
     cf.streamInitialReconnectDelay = maybeTime(options.streaming.initialRetryDelayMs);
+    cf.payloadFilterKey = options.streaming.filter;
   }
   if (options.polling) {
     cf.stream = false;
     cf.baseUri = options.polling.baseUri;
     cf.pollInterface = options.polling.pollIntervalMs / 1000;
+    cf.payloadFilterKey = options.polling.filter;
   }
   if (options.events) {
     cf.allAttributesPrivate = options.events.allAttributesPrivate;

packages/shared/common/__tests__/internal/events/EventProcessor.test.ts

@@ -32,6 +32,9 @@ beforeEach(() => {
         diagnosticEventPath: '/diagnostic',
         analyticsEventPath: '/bulk',
         includeAuthorizationHeader: true,
+        getPollingUri: (_: string): string => '/sdk/latest-all',
+        getStreamingUri: (_: string): string => 'https://mockstream.ld.com/all',
+        getFilteredUri: (_: string): string => '',
       },
     },
     platform: mockPlatform,

packages/shared/common/__tests__/options/ServiceEndpoints.test.ts

@@ -33,3 +33,11 @@ describe.each([
     expect(endpoints.events).toEqual(expected.eventsUri);
   });
 });
+
+it('applies payload filter to polling and streaming endpoints', () => {
+  const endpoints = new ServiceEndpoints('https://stream.launchdarkly.com', 'https://sdk.launchdarkly.com', 'https://events.launchdarkly.com', '/bulk', '/diagnostic', true, 'filterKey');
+
+  expect(endpoints.getStreamingUri('')).toEqual('https://stream.launchdarkly.com/?filter=filterKey');
+  expect(endpoints.getPollingUri('')).toEqual('https://sdk.launchdarkly.com/?filter=filterKey');
+  expect(endpoints.events).toEqual('https://events.launchdarkly.com');
+});

packages/shared/common/src/internal/events/EventSender.test.ts

@@ -25,6 +25,9 @@ const basicConfig = {
     analyticsEventPath: '/bulk',
     diagnosticEventPath: '/diagnostic',
     includeAuthorizationHeader: true,
+    getPollingUri: (_: string): string => '/sdk/latest-all',
+    getStreamingUri: (_: string): string => '/all',
+    getFilteredUri: (_: string): string => '',
   },
 };
 const testEventData1 = { eventId: 'test-event-data-1' };

packages/shared/common/src/internal/stream/StreamingProcessor.test.ts

@@ -16,6 +16,9 @@ const serviceEndpoints = {
   diagnosticEventPath: '/diagnostic',
   analyticsEventPath: '/bulk',
   includeAuthorizationHeader: true,
+  getPollingUri: (_: string): string => '/sdk/latest-all',
+  getStreamingUri: (_: string): string => 'https://mockstream.ld.com/all',
+  getFilteredUri: (_: string): string => '',
 };
 
 function getBasicConfiguration(inLogger: LDLogger) {

packages/shared/common/src/internal/stream/StreamingProcessor.ts

@@ -49,7 +49,8 @@ class StreamingProcessor implements LDStreamProcessor {
     this.headers = defaultHeaders(sdkKey, info, tags);
     this.logger = logger;
     this.requests = requests;
-    this.streamUri = `${basicConfiguration.serviceEndpoints.streaming}${streamUriPath}`;
+
+    this.streamUri = basicConfiguration.serviceEndpoints.getStreamingUri(streamUriPath);
   }
 
   private logConnectionStarted() {

packages/shared/common/src/options/ServiceEndpoints.ts

@@ -11,6 +11,7 @@ export default class ServiceEndpoints {
   public readonly streaming: string;
   public readonly polling: string;
   public readonly events: string;
+  private readonly payloadFilterKey?: string;
 
   /** Valid paths are:
    * /bulk
@@ -36,12 +37,49 @@ export default class ServiceEndpoints {
     analyticsEventPath: string = '/bulk',
     diagnosticEventPath: string = '/diagnostic',
     includeAuthorizationHeader: boolean = true,
+    payloadFilterKey?: string,
   ) {
     this.streaming = canonicalizeUri(streaming);
     this.polling = canonicalizeUri(polling);
     this.events = canonicalizeUri(events);
     this.analyticsEventPath = analyticsEventPath;
     this.diagnosticEventPath = diagnosticEventPath;
     this.includeAuthorizationHeader = includeAuthorizationHeader;
+    this.payloadFilterKey = payloadFilterKey;
+  }
+
+  /**
+  * Constructs and returns the URI to be used for a streaming connection.
+  */
+  public getStreamingUri(path: string): string {
+    return this.getFilteredUri(`${this.streaming}${path}`);
+  }
+
+  /**
+  * Constructs and returns the URI to be used for a polling connection.
+  */
+  public getPollingUri(path: string): string {
+    return this.getFilteredUri(`${this.polling}${path}`);
+  }
+
+  /**
+   * If a payload filter was present in the SDK config, this function will
+   * apply that as a query parameter to the provided URI.
+   *
+   * If the provided uri cannot be parsed, this method will return that uri
+   * unmodified.
+   */
+  public getFilteredUri(uri: string): string {
+    if (!this.payloadFilterKey) {
+      return uri;
+    }
+
+    try {
+      let url = new URL(uri);
+      url.searchParams.set('filter', this.payloadFilterKey);
+      return url.toString();
+    } catch (e) {
+      return uri;
+    }
   }
 }

packages/shared/sdk-server/src/api/options/LDOptions.ts

@@ -268,6 +268,22 @@ export interface LDOptions {
      * ASCII digits, period, hyphen, underscore. A string containing any other characters will be ignored.
      */
     versionName?: string;
+
+    /**
+     * LaunchDarkly Server SDKs historically downloaded all flag configuration and segments for a particular environment
+     * during initialization.
+     *
+     * For some customers, this is an unacceptably large amount of data, and has contributed to performance issues
+     * within their products.
+     *
+     * Filtered environments aim to solve this problem. By allowing customers to specify subsets of an environment's
+     * flags using a filter key, SDKs will initialize faster and use less memory.
+     *
+     * This payload filter key only applies to the default streaming and polling data sources. It will not affect
+     * TestData or FileData data sources, nor will it be applied to any data source provided through the featureStore
+     * config property.
+     */
+    payloadFilterKey?: string;
   };
 
   /**

packages/shared/sdk-server/src/data_sources/Requestor.ts

@@ -33,7 +33,7 @@ export default class Requestor implements LDFeatureRequestor {
     private readonly requests: Requests,
   ) {
     this.headers = defaultHeaders(sdkKey, info, config.tags);
-    this.uri = `${config.serviceEndpoints.polling}/sdk/latest-all`;
+    this.uri = config.serviceEndpoints.getPollingUri('/sdk/latest-all');
   }
 
   /**