feat: new database connection options

Author: whoabuddy

.env

@@ -4,6 +4,11 @@ PG_USER=postgres
 PG_PASSWORD=postgres
 PG_DATABASE=stacks_blockchain_api
 PG_SCHEMA=public
+PG_SSL=false
+
+# The connection URI below can be used in place of the PG variables above,
+# but if enabled it must be defined without others or omitted.
+# PG_CONNECTION_URI=
 
 # Enable to have stacks-node events streamed to a file while the application is running
 # STACKS_EXPORT_EVENTS_FILE=/tmp/stacks-events.tsv

package.json

@@ -187,6 +187,7 @@
     "jest": "^26.6.3",
     "nodemon": "^2.0.3",
     "npm-api": "^1.0.0",
+    "pg-connection-string": "^2.5.0",
     "prettier": "2.2.1",
     "redoc-cli": "^0.9.12",
     "rimraf": "^3.0.2",

src/datastore/postgres-store.ts

@@ -8,6 +8,7 @@ import * as pgCopyStreams from 'pg-copy-streams';
 import * as PgCursor from 'pg-cursor';
 
 import {
+  parseArgBoolean,
   parsePort,
   APP_DIR,
   isTestEnv,
@@ -82,15 +83,51 @@ const MIGRATIONS_DIR = path.join(APP_DIR, 'migrations');
 
 type PgClientConfig = ClientConfig & { schema?: string };
 export function getPgClientConfig(): PgClientConfig {
-  const config: PgClientConfig = {
+  const pgEnvVars = {
     database: process.env['PG_DATABASE'],
     user: process.env['PG_USER'],
     password: process.env['PG_PASSWORD'],
     host: process.env['PG_HOST'],
-    port: parsePort(process.env['PG_PORT']),
+    port: process.env['PG_PORT'],
+    ssl: process.env['PG_SSL'],
     schema: process.env['PG_SCHEMA'],
   };
-  return config;
+  const pgConnectionUri = process.env['PG_CONNECTION_URI'];
+  const pgConfigEnvVar = Object.entries(pgEnvVars).find(([, v]) => typeof v === 'string')?.[0];
+  if (pgConfigEnvVar && pgConnectionUri) {
+    throw new Error(
+      `Both PG_CONNECTION_URI and ${pgConfigEnvVar} environmental variables are defined. PG_CONNECTION_URI must be defined without others or omitted.`
+    );
+  }
+  if (pgConnectionUri) {
+    const uri = new URL(pgConnectionUri);
+    const searchParams = Object.fromEntries(
+      [...uri.searchParams.entries()].map(([k, v]) => [k.toLowerCase(), v])
+    );
+    // Not really standardized
+    const schema: string | undefined =
+      searchParams['currentschema'] ??
+      searchParams['current_schema'] ??
+      searchParams['searchpath'] ??
+      searchParams['search_path'] ??
+      searchParams['schema'];
+    const config: PgClientConfig = {
+      connectionString: pgConnectionUri,
+      schema,
+    };
+    return config;
+  } else {
+    const config: PgClientConfig = {
+      database: pgEnvVars.database,
+      user: pgEnvVars.user,
+      password: pgEnvVars.password,
+      host: pgEnvVars.host,
+      port: parsePort(pgEnvVars.port),
+      ssl: parseArgBoolean(pgEnvVars.ssl),
+      schema: pgEnvVars.schema,
+    };
+    return config;
+  }
 }
 
 export async function runMigrations(

src/helpers.ts

@@ -347,6 +347,35 @@ export function httpPostRequest(
   });
 }
 
+/**
+ * Parses a boolean string using conventions from CLI arguments, URL query params, and environmental variables.
+ * If the input is defined but empty string then true is returned. If the input is undefined or null than false is returned.
+ * For example, if the input comes from a CLI arg like `--enable_thing` or URL query param like `?enable_thing`, then
+ * this function expects to receive a defined but empty string, and returns true.
+ * Otherwise, it checks or values like `true`, `1`, `on`, `yes` (and the inverses).
+ * Throws if an unexpected input value is provided.
+ */
+export function parseArgBoolean(val: string | undefined | null): boolean {
+  if (typeof val === 'undefined' || val === null) {
+    return false;
+  }
+  switch (val.trim().toLowerCase()) {
+    case '':
+    case 'true':
+    case '1':
+    case 'on':
+    case 'yes':
+      return true;
+    case 'false':
+    case '0':
+    case 'off':
+    case 'no':
+      return false;
+    default:
+      throw new Error(`Cannot parse boolean from "${val}"`);
+  }
+}
+
 export function parsePort(portVal: number | string | undefined): number | undefined {
   if (portVal === undefined) {
     return undefined;

src/tests/datastore-tests.ts

@@ -20,8 +20,14 @@ import {
   DbBnsSubdomain,
   DbTokenOfferingLocked,
 } from '../datastore/common';
-import { PgDataStore, cycleMigrations, runMigrations } from '../datastore/postgres-store';
+import {
+  PgDataStore,
+  cycleMigrations,
+  runMigrations,
+  getPgClientConfig,
+} from '../datastore/postgres-store';
 import { PoolClient } from 'pg';
+import * as pgConnectionString from 'pg-connection-string';
 import { parseDbEvent } from '../api/controllers/db-controller';
 import * as assert from 'assert';
 import { I32_MAX } from '../helpers';
@@ -55,6 +61,41 @@ describe('in-memory datastore', () => {
   });
 });
 
+function testEnvVars(envVars: Record<string, string | undefined>, use: () => void): void;
+function testEnvVars(
+  envVars: Record<string, string | undefined>,
+  use: () => Promise<void>
+): Promise<void>;
+function testEnvVars(
+  envVars: Record<string, string | undefined>,
+  use: () => void | Promise<void>
+): void | Promise<void> {
+  const existing = Object.fromEntries(
+    Object.keys(envVars)
+      .filter(k => k in process.env)
+      .map(k => [k, process.env[k]])
+  );
+  const added = Object.keys(envVars).filter(k => !(k in process.env));
+  Object.entries(envVars).forEach(([k, v]) => {
+    process.env[k] = v;
+    if (v === undefined) {
+      delete process.env[k];
+    }
+  });
+  const restoreEnvVars = () => {
+    added.forEach(k => delete process.env[k]);
+    Object.entries(existing).forEach(([k, v]) => (process.env[k] = v));
+  };
+  try {
+    const runFn = use();
+    if (runFn instanceof Promise) {
+      return runFn.finally(() => restoreEnvVars());
+    }
+  } finally {
+    restoreEnvVars();
+  }
+}
+
 describe('postgres datastore', () => {
   let db: PgDataStore;
   let client: PoolClient;
@@ -66,6 +107,81 @@ describe('postgres datastore', () => {
     client = await db.pool.connect();
   });
 
+  test('postgres uri config', () => {
+    const uri =
+      'postgresql://test_user:[email protected]:3211/test_db?ssl=true&currentSchema=test_schema';
+    testEnvVars(
+      {
+        PG_CONNECTION_URI: uri,
+        PG_DATABASE: undefined,
+        PG_USER: undefined,
+        PG_PASSWORD: undefined,
+        PG_HOST: undefined,
+        PG_PORT: undefined,
+        PG_SSL: undefined,
+        PG_SCHEMA: undefined,
+      },
+      () => {
+        const config = getPgClientConfig();
+        const parsedUrl = pgConnectionString.parse(uri);
+        expect(parsedUrl.database).toBe('test_db');
+        expect(parsedUrl.user).toBe('test_user');
+        expect(parsedUrl.password).toBe('secret_password');
+        expect(parsedUrl.host).toBe('database.server.com');
+        expect(parsedUrl.port).toBe('3211');
+        expect(parsedUrl.ssl).toBe(true);
+        expect(config.schema).toBe('test_schema');
+      }
+    );
+  });
+
+  test('postgres env var config', () => {
+    testEnvVars(
+      {
+        PG_CONNECTION_URI: undefined,
+        PG_DATABASE: 'pg_db_db1',
+        PG_USER: 'pg_user_user1',
+        PG_PASSWORD: 'pg_password_password1',
+        PG_HOST: 'pg_host_host1',
+        PG_PORT: '9876',
+        PG_SSL: 'true',
+        PG_SCHEMA: 'pg_schema_schema1',
+      },
+      () => {
+        const config = getPgClientConfig();
+        expect(config.database).toBe('pg_db_db1');
+        expect(config.user).toBe('pg_user_user1');
+        expect(config.password).toBe('pg_password_password1');
+        expect(config.host).toBe('pg_host_host1');
+        expect(config.port).toBe(9876);
+        expect(config.ssl).toBe(true);
+        expect(config.schema).toBe('pg_schema_schema1');
+      }
+    );
+  });
+
+  test('postgres conflicting config', () => {
+    const uri =
+      'postgresql://test_user:[email protected]:3211/test_db?ssl=true&currentSchema=test_schema';
+    testEnvVars(
+      {
+        PG_CONNECTION_URI: uri,
+        PG_DATABASE: 'pg_db_db1',
+        PG_USER: 'pg_user_user1',
+        PG_PASSWORD: 'pg_password_password1',
+        PG_HOST: 'pg_host_host1',
+        PG_PORT: '9876',
+        PG_SSL: 'true',
+        PG_SCHEMA: 'pg_schema_schema1',
+      },
+      () => {
+        expect(() => {
+          const config = getPgClientConfig();
+        }).toThrowError();
+      }
+    );
+  });
+
   test('pg address STX balances', async () => {
     const dbBlock: DbBlock = {
       block_hash: '0x9876',