feat: hooks for dynamic schemas (cube-js#287). Thanks to @willhausman!

* hook for defining a custom context per request.

* use lru-cache for compilerApis

* docs for create options

* contextToDataSourceId option

* Revert "hook for defining a custom context per request."

This reverts commit 2a28077.

* revert requestToContext create option

Author: willhausman

docs/Cube.js-Backend/@cubejs-backend-server-core.md

@@ -49,11 +49,15 @@ Both [CubejsServerCore](@cubejs-backend-server-core) and [CubejsServer](@cubejs-
   driverFactory: (context: DriverContext) => BaseDriver,
   externalDriverFactory: (context: RequestContext) => BaseDriver,
   contextToAppId: (context: RequestContext) => String,
+  contextToDataSourceId: (context: RequestContext) => String,
   repositoryFactory: (context: RequestContext) => String,
   checkAuthMiddleware: (req: ExpressRequest, res: ExpressResponse, next: ExpressMiddleware) => any,
   queryTransformer: (query: Object, context: RequestContext) => Object,
   preAggregationsSchema: String | (context: RequestContext) => String,
   schemaVersion: (context: RequestContext) => String,
+  compilerCacheSize: Number,
+  maxCompilerCacheKeepAlive: Number,
+  updateCompilerCacheKeepAlive: Boolean,
   telemetry: Boolean,
   allowUngroupedWithoutPrimaryKey: Boolean,
   orchestratorOptions: {
@@ -175,6 +179,17 @@ CubejsServerCore.create({
 })
 ```
 
+### contextToDataSourceId
+
+`contextToDataSourceId` is a function to determine a DataSource Id which is used to override the `contextToAppId` caching key for managing connection pools.
+
+```javascript
+CubejsServerCore.create({
+  contextToAppId: ({ authInfo }) => `CUBEJS_APP_${authInfo.user_id}`,
+  contextToDataSourceId: ({ authInfo }) => `CUBEJS_APP_${authInfo.tenantId}`
+});
+```
+
 ### repositoryFactory
 
 This option allows to customize the repository for Cube.js data schema files. It
@@ -260,6 +275,18 @@ CubejsServerCore.create({
 });
 ```
 
+### compilerCacheSize
+
+Maximum number of compiled schemas to persist with in-memory cache.  Defaults to 250, but optimum value will depend on deployed environment. When the max is reached, will start dropping the least recently used schemas from the cache.
+
+### maxCompilerCacheKeepAlive
+
+Maximum length of time in ms to keep compiled schemas in memory.  Default keeps schemas in memory indefinitely.
+
+### updateCompilerCacheKeepAlive
+
+Providing `updateCompilerCacheKeepAlive: true` keeps frequently used schemas in memory by reseting their `maxCompilerCacheKeepAlive` every time they are accessed.
+
 ### allowUngroupedWithoutPrimaryKey
 
 Providing `allowUngroupedWithoutPrimaryKey: true` disables primary key inclusion check for `ungrouped` queries. 

packages/cubejs-server-core/core/index.d.ts

@@ -9,6 +9,7 @@ declare module '@cubejs-backend/server-core' {
     checkAuthMiddleware?: (req: any, res: any, next: any) => any;
     queryTransformer?: () => (query: any, context: any) => any;
     contextToAppId?: any;
+    contextToDataSourceId?: (context: any) => string;
     devServer?: boolean;
     dbType?: DatabaseType;
     driverFactory?: DriverFactory;
@@ -18,6 +19,9 @@ declare module '@cubejs-backend/server-core' {
     preAggregationsSchema?: string | (() => string)
     repositoryFactory?: any;
     schemaPath?: string;
+    compilerCacheSize?: number;
+    maxCompilerCacheKeepAlive?: number;
+    updateCompilerCacheKeepAlive?: boolean;
   }
 
   export interface DriverFactory {

packages/cubejs-server-core/core/index.js

@@ -3,6 +3,7 @@ const ApiGateway = require('@cubejs-backend/api-gateway');
 const crypto = require('crypto');
 const fs = require('fs-extra');
 const path = require('path');
+const LRUCache = require('lru-cache');
 const CompilerApi = require('./CompilerApi');
 const OrchestratorApi = require('./OrchestratorApi');
 const FileRepository = require('./FileRepository');
@@ -122,14 +123,24 @@ class CubejsServerCore {
       () => options.externalDbType;
     this.preAggregationsSchema =
       typeof options.preAggregationsSchema === 'function' ? options.preAggregationsSchema : () => options.preAggregationsSchema;
-    this.appIdToCompilerApi = {};
+    this.compilerCache = new LRUCache({
+      max: options.compilerCacheSize || 250,
+      maxAge: options.maxCompilerCacheKeepAlive,
+      updateAgeOnGet: options.updateCompilerCacheKeepAlive
+    });
     this.dataSourceIdToOrchestratorApi = {};
     this.contextToAppId = options.contextToAppId || (() => process.env.CUBEJS_APP || 'STANDALONE');
+    this.contextToDataSourceId = options.contextToDataSourceId || this.defaultContextToDataSourceId.bind(this);
     this.orchestratorOptions =
       typeof options.orchestratorOptions === 'function' ?
         options.orchestratorOptions :
         () => options.orchestratorOptions;
 
+    // proactively free up old cache values occassionally
+    if (options.maxCompilerCacheKeepAlive) {
+      setInterval(() => this.compilerCache.prune(), options.maxCompilerCacheKeepAlive);
+    }
+
     const { machineIdSync } = require('node-machine-id');
     let anonymousId = 'unknown';
     try {
@@ -263,20 +274,22 @@ class CubejsServerCore {
 
   getCompilerApi(context) {
     const appId = this.contextToAppId(context);
-    if (!this.appIdToCompilerApi[appId]) {
-      this.appIdToCompilerApi[appId] = this.createCompilerApi(
+    let compilerApi = this.compilerCache.get(appId);
+    if (!compilerApi) {
+      compilerApi = this.createCompilerApi(
         this.repositoryFactory(context), {
           dbType: (dataSourceContext) => this.contextToDbType({ ...context, ...dataSourceContext }),
           externalDbType: this.contextToExternalDbType(context),
           schemaVersion: this.options.schemaVersion && (() => this.options.schemaVersion(context)),
           preAggregationsSchema: this.preAggregationsSchema(context)
         }
       );
+      this.compilerCache.set(appId, compilerApi);
     }
-    return this.appIdToCompilerApi[appId];
+    return compilerApi;
   }
 
-  contextToDataSourceId(context) {
+  defaultContextToDataSourceId(context) {
     return `${this.contextToAppId(context)}_${context.dataSource}`;
   }
 

packages/cubejs-server-core/package.json

@@ -38,6 +38,7 @@
     "cross-spawn": "^6.0.5",
     "fs-extra": "^7.0.1",
     "jsonwebtoken": "^8.4.0",
+    "lru-cache": "^5.1.1",
     "node-fetch": "^2.6.0",
     "node-machine-id": "^1.1.10",
     "prettier": "^1.16.4",

packages/cubejs-server-core/yarn.lock

@@ -833,10 +833,10 @@
     moment-timezone "^0.5.27"
     ramda "^0.25.0"
 
-"@cubejs-backend/query-orchestrator@^0.12.1":
-  version "0.12.1"
-  resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.12.1.tgz#e8b7ea46f9a7e695b6d7a299616b68cb0ae42818"
-  integrity sha512-yINa38E+BWTrKS42ByqL0qpTDZAuWVNMWj1Ea+ajFtyInu2MDD7sUCF5HkIMD0H8m4nMtsXl/D4X0rZb0EthCw==
+"@cubejs-backend/query-orchestrator@^0.12.2":
+  version "0.12.2"
+  resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.12.2.tgz#13c3be7c3fa150a7a8047810143f0eb4bc5854bb"
+  integrity sha512-fy+8Mfhez1MF4eVpGFpBjYHxqf4AX9nRBqXKg+8BBrc6Rz7xLOoh3xvI7IGMrtahCo+HyuCgHpBqejryB24TEQ==
   dependencies:
     ramda "^0.24.1"
     redis "^2.8.0"
@@ -3620,6 +3620,13 @@ loose-envify@^1.0.0:
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
+lru-cache@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
+  integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==
+  dependencies:
+    yallist "^3.0.2"
+
 lz-string@^1.4.4:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
@@ -5566,7 +5573,7 @@ y18n@^4.0.0:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
   integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
 
-yallist@^3.0.0, yallist@^3.0.3:
+yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
   integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==