Skip to content

Commit 40ce6e8

Browse files
authored
Merge pull request #60 from outerbase/invisal/snowflake-support
snowflake support
2 parents 6d03a81 + c01458f commit 40ce6e8

File tree

7 files changed

+4489
-1086
lines changed

7 files changed

+4489
-1086
lines changed

.github/workflows/test.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,3 +208,24 @@ jobs:
208208
MOTHERDUCK_PATH: ${{ secrets.MOTHERDUCK_PATH }}
209209
MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }}
210210
run: npm run test:connection
211+
212+
test_snowflake:
213+
name: 'Snowflake Connection'
214+
runs-on: ubuntu-latest
215+
needs: build
216+
217+
steps:
218+
- uses: actions/checkout@v4
219+
220+
- name: Install modules
221+
run: npm install
222+
223+
- name: Run tests
224+
env:
225+
CONNECTION_TYPE: snowflake
226+
SNOWFLAKE_ACCOUNT_ID: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
227+
SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
228+
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
229+
SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
230+
SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
231+
run: npm run test:connection

package-lock.json

Lines changed: 4233 additions & 1078 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@outerbase/sdk",
3-
"version": "2.0.0-rc.1",
3+
"version": "2.0.0-rc.2",
44
"description": "",
55
"main": "dist/index.js",
66
"module": "dist/index.js",
@@ -37,26 +37,27 @@
3737
"handlebars": "^4.7.8"
3838
},
3939
"devDependencies": {
40+
"@google-cloud/bigquery": "^7.9.0",
4041
"@jest/globals": "^29.7.0",
4142
"@libsql/client": "^0.14.0",
43+
"@neondatabase/serverless": "^0.9.3",
4244
"@types/jest": "^29.5.13",
4345
"@types/node": "^20.12.12",
4446
"@types/ws": "^8.5.10",
4547
"dotenv": "^16.4.5",
48+
"duckdb": "^1.1.1",
4649
"husky": "^9.0.11",
4750
"jest": "^29.7.0",
4851
"lint-staged": "^15.2.4",
52+
"mongodb": "^6.9.0",
4953
"mysql2": "^3.11.3",
5054
"pg": "^8.13.0",
5155
"prettier": "^3.2.5",
5256
"ts-jest": "^29.1.3",
5357
"ts-node": "^10.9.2",
5458
"tsconfig-paths": "^4.2.0",
5559
"typescript": "^5.4.5",
56-
"@google-cloud/bigquery": "^7.9.0",
57-
"@neondatabase/serverless": "^0.9.3",
58-
"duckdb": "^1.1.1",
5960
"ws": "^8.17.1",
60-
"mongodb": "^6.9.0"
61+
"snowflake-sdk": "^1.15.0"
6162
}
6263
}

src/connections/mysql.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ interface MySQLConstraintResult {
4949
CONSTRAINT_TYPE: string;
5050
}
5151

52-
interface MySQLConstraintColumnResult {
52+
export interface MySQLConstraintColumnResult {
5353
TABLE_SCHEMA: string;
5454
TABLE_NAME: string;
5555
COLUMN_NAME: string;
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
import snowflake from 'snowflake-sdk';
2+
import { Query } from '../../query';
3+
import { QueryResult } from '..';
4+
import {
5+
createErrorResult,
6+
transformArrayBasedResult,
7+
} from '../../utils/transformer';
8+
import { Database, TableColumn } from '../../models/database';
9+
import { PostgreBaseConnection } from './../postgre/base';
10+
import {
11+
buildMySQLDatabaseSchmea,
12+
MySQLConstraintColumnResult,
13+
} from '../mysql';
14+
15+
export class SnowflakeConnection extends PostgreBaseConnection {
16+
protected db: snowflake.Connection;
17+
18+
constructor(db: any) {
19+
super();
20+
this.db = db;
21+
}
22+
23+
async connect(): Promise<any> {
24+
await new Promise((resolve, reject) => {
25+
this.db.connectAsync((err, conn) => {
26+
if (err) reject(err.message);
27+
else resolve(conn);
28+
});
29+
});
30+
}
31+
32+
async disconnect(): Promise<any> {
33+
await new Promise((resolve) => this.db.destroy(resolve));
34+
}
35+
36+
async testConnection(): Promise<{ error?: string }> {
37+
try {
38+
await this.connect();
39+
const { data } = await this.query({
40+
query: 'SELECT CURRENT_DATABASE() AS DBNAME;',
41+
});
42+
43+
await this.disconnect();
44+
if (!data[0].DBNAME) return { error: 'Database does not exist' };
45+
46+
return {};
47+
} catch (e) {
48+
if (e instanceof Error) return { error: e.message };
49+
return { error: 'Unknown error' };
50+
}
51+
}
52+
53+
async fetchDatabaseSchema(): Promise<Database> {
54+
// Get the list of schema first
55+
const { data: schemaList } = await this.query<{ SCHEMA_NAME: string }>({
56+
query: `SELECT SCHEMA_NAME FROM information_schema.schemata WHERE schema_name NOT IN ('INFORMATION_SCHEMA');`,
57+
});
58+
59+
// Get the list of all tables
60+
const { data: tableList } = await this.query<{
61+
TABLE_NAME: string;
62+
TABLE_SCHEMA: string;
63+
}>({
64+
query: `SELECT TABLE_NAME, TABLE_SCHEMA FROM information_schema.tables WHERE table_schema NOT IN ('INFORMATION_SCHEMA');`,
65+
});
66+
67+
// Get the list of all columns
68+
const { data: columnList } = await this.query<{
69+
TABLE_SCHEMA: string;
70+
TABLE_NAME: string;
71+
COLUMN_NAME: string;
72+
DATA_TYPE: string;
73+
IS_NULLABLE: string;
74+
COLUMN_DEFAULT: string;
75+
ORDINAL_POSITION: number;
76+
}>({
77+
query: `SELECT * FROM information_schema.columns WHERE table_schema NOT IN ('INFORMATION_SCHEMA');`,
78+
});
79+
80+
// Get the list of all constraints
81+
const { data: constraintsList } = await this.query<{
82+
CONSTRAINT_SCHEMA: string;
83+
CONSTRAINT_NAME: string;
84+
TABLE_NAME: string;
85+
TABLE_SCHEMA: string;
86+
CONSTRAINT_TYPE: string;
87+
}>({
88+
query: `SELECT * FROM information_schema.table_constraints WHERE CONSTRAINT_SCHEMA NOT IN ('INFORMATION_SCHEMA') AND CONSTRAINT_TYPE IN ('FOREIGN KEY', 'PRIMARY KEY', 'UNIQUE');`,
89+
});
90+
91+
// Mamic the key usages table using SHOW PRIMARY KEY and SHOW FOREIGN KEYS
92+
const { data: primaryKeyConstraint } = await this.query<{
93+
schema_name: string;
94+
table_name: string;
95+
column_name: string;
96+
constraint_name: string;
97+
}>({ query: `SHOW PRIMARY KEYS;` });
98+
99+
const { data: foreignKeyConstraint } = await this.query<{
100+
pk_schema_name: string;
101+
pk_table_name: string;
102+
pk_column_name: string;
103+
fk_schema_name: string;
104+
fk_table_name: string;
105+
fk_column_name: string;
106+
fk_name: string;
107+
}>({ query: `SHOW IMPORTED KEYS;` });
108+
109+
// Postgres structure is similar to MySQL, so we can reuse the MySQL schema builder
110+
// by just mapping the column names
111+
return buildMySQLDatabaseSchmea({
112+
schemaList,
113+
tableList,
114+
columnList: columnList.map((column) => ({
115+
COLUMN_TYPE: column.DATA_TYPE,
116+
...column,
117+
COLUMN_KEY: '',
118+
EXTRA: '',
119+
})),
120+
constraintsList,
121+
constraintColumnsList: [
122+
...primaryKeyConstraint.map(
123+
(constraint): MySQLConstraintColumnResult => ({
124+
TABLE_SCHEMA: constraint.schema_name,
125+
TABLE_NAME: constraint.table_name,
126+
COLUMN_NAME: constraint.column_name,
127+
CONSTRAINT_NAME: constraint.constraint_name,
128+
REFERENCED_TABLE_SCHEMA: '',
129+
REFERENCED_TABLE_NAME: '',
130+
REFERENCED_COLUMN_NAME: '',
131+
})
132+
),
133+
...foreignKeyConstraint.map(
134+
(constraint): MySQLConstraintColumnResult => ({
135+
TABLE_SCHEMA: constraint.fk_schema_name,
136+
TABLE_NAME: constraint.fk_table_name,
137+
COLUMN_NAME: constraint.fk_column_name,
138+
CONSTRAINT_NAME: constraint.fk_name,
139+
REFERENCED_TABLE_SCHEMA: constraint.pk_schema_name,
140+
REFERENCED_TABLE_NAME: constraint.pk_table_name,
141+
REFERENCED_COLUMN_NAME: constraint.pk_column_name,
142+
})
143+
),
144+
],
145+
});
146+
}
147+
148+
createTable(
149+
schemaName: string | undefined,
150+
tableName: string,
151+
columns: TableColumn[]
152+
): Promise<QueryResult> {
153+
const tempColumns = structuredClone(columns);
154+
for (const column of tempColumns) {
155+
if (column.definition.references) {
156+
column.definition.references.table = schemaName
157+
? `${schemaName}.${column.definition.references.table}`
158+
: column.definition.references.table;
159+
}
160+
}
161+
162+
return super.createTable(schemaName, tableName, tempColumns);
163+
}
164+
165+
async renameTable(
166+
schemaName: string | undefined,
167+
tableName: string,
168+
newTableName: string
169+
): Promise<QueryResult> {
170+
// Schema is required for rename
171+
return super.renameTable(
172+
schemaName,
173+
tableName,
174+
schemaName ? `${schemaName}.${newTableName}` : newTableName
175+
);
176+
}
177+
178+
async query<T = Record<string, unknown>>(
179+
query: Query
180+
): Promise<QueryResult<T>> {
181+
try {
182+
const [err, headers, rows] = await new Promise<
183+
[snowflake.SnowflakeError | undefined, string[], unknown[][]]
184+
>((resolve) => {
185+
this.db.execute({
186+
sqlText: query.query,
187+
binds: query.parameters as snowflake.Binds,
188+
rowMode: 'array',
189+
complete: (err, stmt, rows) => {
190+
resolve([
191+
err,
192+
err
193+
? []
194+
: stmt.getColumns().map((col) => col.getName()),
195+
rows as unknown[][],
196+
]);
197+
},
198+
});
199+
});
200+
201+
if (err) return createErrorResult(err.message) as QueryResult<T>;
202+
return transformArrayBasedResult(
203+
headers,
204+
(header) => ({
205+
name: header,
206+
}),
207+
rows
208+
) as QueryResult<T>;
209+
} catch (e) {
210+
return createErrorResult('Unknown error') as QueryResult<T>;
211+
}
212+
}
213+
}

src/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ export * from './connections/mongodb';
77
export * from './connections/mysql';
88
export * from './connections/postgre/postgresql';
99
export * from './connections/sqlite/turso';
10+
export * from './connections/snowflake/snowflake';
1011
export * from './client';
1112
export * from './models/decorators';

tests/connections/create-test-connection.ts

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Client as PgClient } from 'pg';
22
import { BigQuery } from '@google-cloud/bigquery';
33
import duckDB from 'duckdb';
4+
import snowflake from 'snowflake-sdk';
45
import { createClient as createTursoConnection } from '@libsql/client';
56
import { createConnection as createMySqlConnection } from 'mysql2';
67
import {
@@ -13,6 +14,7 @@ import {
1314
MongoDBConnection,
1415
DuckDBConnection,
1516
StarbaseConnection,
17+
SnowflakeConnection,
1618
} from '../../src';
1719
import { MongoClient } from 'mongodb';
1820

@@ -86,8 +88,8 @@ export default function createTestClient(): {
8688
const client = new DuckDBConnection(
8789
process.env.MOTHERDUCK_PATH
8890
? new duckDB.Database(process.env.MOTHERDUCK_PATH, {
89-
motherduck_token: process.env.MOTHERDUCK_TOKEN as string,
90-
})
91+
motherduck_token: process.env.MOTHERDUCK_TOKEN as string,
92+
})
9193
: new duckDB.Database(':memory:')
9294
);
9395
return { client, defaultSchema: 'main' };
@@ -98,6 +100,16 @@ export default function createTestClient(): {
98100
});
99101

100102
return { client, defaultSchema: 'main' };
103+
} else if (process.env.CONNECTION_TYPE === 'snowflake') {
104+
const client = new SnowflakeConnection(snowflake.createConnection({
105+
database: process.env.SNOWFLAKE_DATABASE as string,
106+
username: process.env.SNOWFLAKE_USERNAME as string,
107+
password: process.env.SNOWFLAKE_PASSWORD as string,
108+
account: process.env.SNOWFLAKE_ACCOUNT_ID as string,
109+
warehouse: process.env.SNOKWFLAKE_WAREHOUSE as string,
110+
}));
111+
112+
return { client, defaultSchema: "PUBLIC" }
101113
}
102114

103115
throw new Error('Invalid connection type');

0 commit comments

Comments
 (0)