Skip to content

Commit d1a6645

Browse files
jeniiijeniii
authored
[FAI-14970] Copy faros api settings from dst to src config (#107)
1 parent 0d1da4b commit d1a6645

File tree

3 files changed

+163
-10
lines changed

3 files changed

+163
-10
lines changed

‎airbyte-local-cli-nodejs/src/utils.ts

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import readline from 'node:readline';
1515
import {pipeline, Transform, Writable} from 'node:stream';
1616
import {promisify} from 'node:util';
1717

18+
import {isNil, omitBy} from 'lodash';
1819
import pino from 'pino';
1920
import pretty from 'pino-pretty';
2021

@@ -202,6 +203,46 @@ export function overrideCatalog(
202203
return processedCatalog;
203204
}
204205

206+
/**
207+
* Copy Faros API settings from destination config to source config.
208+
* This is for users' convenience so that they don't have to provide the same settings
209+
* in both source and destination configs.
210+
*/
211+
export function updateSrcConfigWithFarosConfig(airbyteConfig: {src: AirbyteConfig; dst: AirbyteConfig}): void {
212+
const srcDockerImage = airbyteConfig.src?.image;
213+
const dstDockerImage = airbyteConfig.dst?.image;
214+
const dstConfig = airbyteConfig.dst?.config as any;
215+
216+
if (
217+
srcDockerImage?.startsWith('farosai/airbyte-faros-feeds-source') &&
218+
dstDockerImage?.startsWith('farosai/airbyte-faros-destination')
219+
) {
220+
// take Faros API settings from destination config
221+
const farosApiConfig: any = {
222+
api_key: dstConfig?.edition_configs?.api_key,
223+
api_url: dstConfig?.edition_configs?.api_url,
224+
graph: dstConfig?.edition_configs?.graph,
225+
graph_api: dstConfig?.edition_configs?.graph_api,
226+
};
227+
const compactFarosApiConfig = omitBy(farosApiConfig, isNil);
228+
if (Object.entries(compactFarosApiConfig).length === 0) {
229+
return;
230+
}
231+
232+
const debugLog = JSON.stringify({faros: compactFarosApiConfig}).replace(
233+
compactFarosApiConfig?.['api_key'],
234+
'REDACTED',
235+
);
236+
logger.debug(`Updating source config with Faros API settings from destination config: ${debugLog}`);
237+
238+
// merge Faros API config into source config
239+
airbyteConfig.src.config = {
240+
...airbyteConfig.src.config,
241+
faros: compactFarosApiConfig,
242+
};
243+
}
244+
}
245+
205246
/**
206247
* Write Airbyte config to temporary dir and a json file
207248
*/
@@ -227,12 +268,17 @@ export function writeConfig(tmpDir: string, config: FarosConfig): void {
227268
};
228269
}
229270

271+
// if not running source only, copy faros api settings from destination config to source config
272+
if (!config.srcOutputFile) {
273+
updateSrcConfigWithFarosConfig(airbyteConfig);
274+
}
275+
230276
// write config to temporary directory config files
231277
logger.debug(`Writing Airbyte config to files...`);
232278
const srcConfigFilePath = `${tmpDir}${sep}${SRC_CONFIG_FILENAME}`;
233279
const dstConfigFilePath = `${tmpDir}${sep}${FILENAME_PREFIX}_dst_config.json`;
234-
writeFileSync(srcConfigFilePath, JSON.stringify(airbyteConfig.src.config ?? {}, null, 2));
235-
writeFileSync(dstConfigFilePath, JSON.stringify(airbyteConfig.dst.config ?? {}, null, 2));
280+
writeFileSync(srcConfigFilePath, JSON.stringify(airbyteConfig.src.config ?? {}));
281+
writeFileSync(dstConfigFilePath, JSON.stringify(airbyteConfig.dst.config ?? {}));
236282
logger.debug(`Airbyte config files written to: ${srcConfigFilePath}, ${dstConfigFilePath}`);
237283
logger.debug(airbyteConfig.src.config ?? {}, `Source config: `);
238284
logger.debug(airbyteConfig.dst.config ?? {}, `Destination config: `);
@@ -265,8 +311,8 @@ export async function writeCatalog(tmpDir: string, config: FarosConfig): Promise
265311
});
266312

267313
logger.debug(`Writing Airbyte catalog to files...`);
268-
writeFileSync(srcCatalogFilePath, JSON.stringify(srcCatalog, null, 2));
269-
writeFileSync(dstCatalogFilePath, JSON.stringify(dstCatalog, null, 2));
314+
writeFileSync(srcCatalogFilePath, JSON.stringify(srcCatalog));
315+
writeFileSync(dstCatalogFilePath, JSON.stringify(dstCatalog));
270316
logger.debug(`Airbyte catalog files written to: ${srcCatalogFilePath}, ${dstCatalogFilePath}`);
271317
logger.debug(srcCatalog, `Source catalog: `);
272318
logger.debug(dstCatalog, `Destination catalog: `);

‎airbyte-local-cli-nodejs/test/utils.it.test.ts

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ const testConfig: FarosConfig = {
3838
dst: {
3939
image: 'farosai/airbyte-test-destination',
4040
config: {
41-
edition_config: {
41+
edition_configs: {
4242
graph: 'default',
4343
edition: 'cloud',
4444
api_url: 'https://test.api.faros.ai',
@@ -150,8 +150,8 @@ describe('write files to temporary dir', () => {
150150
expect(readFileSync(CONFIG_FILE, 'utf8')).toEqual(
151151
JSON.stringify({src: testConfig.src, dst: testConfig.dst}, null, 2),
152152
);
153-
expect(readFileSync(srcConfigPath, 'utf8')).toEqual(JSON.stringify(testConfig.src?.config, null, 2));
154-
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfig.dst?.config, null, 2));
153+
expect(readFileSync(srcConfigPath, 'utf8')).toEqual(JSON.stringify(testConfig.src?.config));
154+
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfig.dst?.config));
155155
});
156156

157157
it('should alter config if debug is enabled', () => {
@@ -166,9 +166,41 @@ describe('write files to temporary dir', () => {
166166
JSON.stringify({src: testConfigDebug.src, dst: testConfigDebug.dst}, null, 2),
167167
);
168168
expect(readFileSync(srcConfigPath, 'utf8')).toEqual(
169-
JSON.stringify({...testConfigDebug.src?.config, feed_cfg: {debug: true}}, null, 2),
169+
JSON.stringify({...testConfigDebug.src?.config, feed_cfg: {debug: true}}),
170170
);
171-
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfigDebug.dst?.config, null, 2));
171+
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfigDebug.dst?.config));
172+
});
173+
174+
it('should alter config with feeds source image', () => {
175+
const testConfigFeeds = structuredClone(testConfig);
176+
testConfigFeeds.src!.image = 'farosai/airbyte-faros-feeds-source';
177+
testConfigFeeds.dst!.image = 'farosai/airbyte-faros-destination';
178+
179+
expect(() => writeConfig(tmpDirPath, structuredClone(testConfigFeeds))).not.toThrow();
180+
expect(existsSync(CONFIG_FILE)).toBe(true);
181+
expect(existsSync(srcConfigPath)).toBe(true);
182+
expect(existsSync(dstConfigPath)).toBe(true);
183+
184+
expect(JSON.parse(readFileSync(srcConfigPath, 'utf8'))).toEqual({
185+
...testConfigFeeds.src?.config,
186+
faros: {graph: 'default', api_url: 'https://test.api.faros.ai'},
187+
});
188+
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfigFeeds.dst?.config));
189+
});
190+
191+
it('should succeed with empty dst config', () => {
192+
const testConfigFeeds = structuredClone(testConfig);
193+
testConfigFeeds.src!.image = 'farosai/airbyte-faros-feeds-source';
194+
testConfigFeeds.dst!.image = 'farosai/airbyte-faros-destination';
195+
testConfigFeeds.dst!.config = {};
196+
197+
expect(() => writeConfig(tmpDirPath, structuredClone(testConfigFeeds))).not.toThrow();
198+
expect(existsSync(CONFIG_FILE)).toBe(true);
199+
expect(existsSync(srcConfigPath)).toBe(true);
200+
expect(existsSync(dstConfigPath)).toBe(true);
201+
202+
expect(readFileSync(srcConfigPath, 'utf8')).toEqual(JSON.stringify(testConfigFeeds.src?.config));
203+
expect(readFileSync(dstConfigPath, 'utf8')).toEqual(JSON.stringify(testConfigFeeds.dst?.config));
172204
});
173205
});
174206

‎airbyte-local-cli-nodejs/test/utils.test.ts

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {spawnSync} from 'node:child_process';
22
import {readFileSync} from 'node:fs';
33

44
import {AirbyteCatalog} from '../src/types';
5-
import {checkDockerInstalled, overrideCatalog, parseConfigFile} from '../src/utils';
5+
import {checkDockerInstalled, overrideCatalog, parseConfigFile, updateSrcConfigWithFarosConfig} from '../src/utils';
66

77
jest.mock('node:fs');
88
jest.mock('node:child_process');
@@ -59,6 +59,81 @@ describe('checkDockerInstalled', () => {
5959
});
6060
});
6161

62+
describe('updateSrcConfigWithFarosConfig', () => {
63+
it('should succeed', () => {
64+
const testAirbyteConfig = {
65+
src: {
66+
image: 'farosai/airbyte-faros-feeds-source',
67+
config: {
68+
testKey: 'testValue',
69+
},
70+
},
71+
dst: {
72+
image: 'farosai/airbyte-faros-destination',
73+
config: {
74+
edition_configs: {
75+
api_url: 'api-url',
76+
api_key: 'api-key',
77+
},
78+
},
79+
},
80+
};
81+
updateSrcConfigWithFarosConfig(testAirbyteConfig);
82+
expect(testAirbyteConfig.src.config).toEqual({
83+
testKey: 'testValue',
84+
faros: {
85+
api_url: 'api-url',
86+
api_key: 'api-key',
87+
},
88+
});
89+
});
90+
91+
it('should succeed with no dst faros api settings', () => {
92+
const testAirbyteConfig = {
93+
src: {
94+
image: 'farosai/airbyte-faros-feeds-source',
95+
config: {
96+
testKey: 'testValue',
97+
},
98+
},
99+
dst: {
100+
image: 'farosai/airbyte-faros-destination',
101+
config: {
102+
testKey: {},
103+
},
104+
},
105+
};
106+
updateSrcConfigWithFarosConfig(testAirbyteConfig);
107+
expect(testAirbyteConfig.src.config).toEqual({
108+
testKey: 'testValue',
109+
});
110+
});
111+
112+
it('should skip with non feed src image', () => {
113+
const testAirbyteConfig = {
114+
src: {
115+
image: 'farosai/airbyte-example-source',
116+
config: {
117+
testKey: 'testValue',
118+
},
119+
},
120+
dst: {
121+
image: 'farosai/airbyte-faros-destination',
122+
config: {
123+
edition_configs: {
124+
api_url: 'api-url',
125+
api_key: 'api-key',
126+
},
127+
},
128+
},
129+
};
130+
updateSrcConfigWithFarosConfig(testAirbyteConfig);
131+
expect(testAirbyteConfig.src.config).toEqual({
132+
testKey: 'testValue',
133+
});
134+
});
135+
});
136+
62137
describe('overrideCatalog', () => {
63138
const testDefaultCatalog = {
64139
streams: [

0 commit comments

Comments
 (0)