Skip to content

feat(fdc): Data Connect Bulk Import #2905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions etc/firebase-admin.data-connect.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@ export class DataConnect {
executeGraphql<GraphqlResponse, Variables>(query: string, options?: GraphqlOptions<Variables>): Promise<ExecuteGraphqlResponse<GraphqlResponse>>;
// @beta
executeGraphqlRead<GraphqlResponse, Variables>(query: string, options?: GraphqlOptions<Variables>): Promise<ExecuteGraphqlResponse<GraphqlResponse>>;
// @beta
insert<GraphQlResponse, Variables extends object>(tableName: string, variables: Variables): Promise<ExecuteGraphqlResponse<GraphQlResponse>>;
// @beta
insertMany<GraphQlResponse, Variables extends Array<unknown>>(tableName: string, variables: Variables): Promise<ExecuteGraphqlResponse<GraphQlResponse>>;
// @beta
upsert<GraphQlResponse, Variables extends object>(tableName: string, variables: Variables): Promise<ExecuteGraphqlResponse<GraphQlResponse>>;
// @beta
upsertMany<GraphQlResponse, Variables extends Array<unknown>>(tableName: string, variables: Variables): Promise<ExecuteGraphqlResponse<GraphQlResponse>>;
}

// @public
Expand Down
197 changes: 197 additions & 0 deletions src/data-connect/data-connect-api-client-internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,203 @@ export class DataConnectApiClient {
const message = error.message || `Unknown server error: ${response.text}`;
return new FirebaseDataConnectError(code, message);
}

/**
* Converts JSON data into a GraphQL literal string.
* Handles nested objects, arrays, strings, numbers, and booleans.
* Ensures strings are properly escaped.
*/
private objectToString(data: any): string {
if (typeof data === 'string') {
const escapedString = data
.replace(/\\/g, '\\\\') // Replace \ with \\
.replace(/"/g, '\\"'); // Replace " with \"
return `"${escapedString}"`;
}
if (typeof data === 'number' || typeof data === 'boolean' || data === null) {
return String(data);
}
if (validator.isArray(data)) {
const elements = data.map(item => this.objectToString(item)).join(', ');
return `[${elements}]`;
}
if (typeof data === 'object' && data !== null) {
// Filter out properties where the value is undefined BEFORE mapping
const kvPairs = Object.entries(data)
.filter(([, val]) => val !== undefined)
.map(([key, val]) => {
// GraphQL object keys are typically unquoted.
return `${key}: ${this.objectToString(val)}`;
});

if (kvPairs.length === 0) {
return '{}'; // Represent an object with no defined properties as {}
}
return `{ ${kvPairs.join(', ')} }`;
}

// If value is undefined (and not an object property, which is handled above,
// e.g., if objectToString(undefined) is called directly or for an array element)
// it should be represented as 'null'.
if (typeof data === 'undefined') {
return 'null';
}

// Fallback for any other types (e.g., Symbol, BigInt - though less common in GQL contexts)
// Consider how these should be handled or if an error should be thrown.
// For now, simple string conversion.
return String(data);
}

private formatTableName(tableName: string): string {
// Format tableName: first character to lowercase
if (tableName && tableName.length > 0) {
return tableName.charAt(0).toLowerCase() + tableName.slice(1);
}
return tableName;
}

private handleBulkImportErrors(err: FirebaseDataConnectError): never {
if (err.code === `data-connect/${DATA_CONNECT_ERROR_CODE_MAPPING.QUERY_ERROR}`){
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.QUERY_ERROR,
`${err.message}. Make sure that your table name passed in matches the type name in your GraphQL schema file.`);
}
throw err;
}

/**
* Insert a single row into the specified table.
*/
public async insert<GraphQlResponse, Variables extends object>(
tableName: string,
data: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
if (!validator.isNonEmptyString(tableName)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`tableName` must be a non-empty string.');
}
if (validator.isArray(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be an object, not an array, for single insert.');
}
if (!validator.isNonNullObject(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be a non-null object.');
}

try {
tableName = this.formatTableName(tableName);
const gqlDataString = this.objectToString(data);
const mutation = `mutation { ${tableName}_insert(data: ${gqlDataString}) }`;
// Use internal executeGraphql
return this.executeGraphql<GraphQlResponse, Variables>(mutation).catch(this.handleBulkImportErrors);
} catch (e: any) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INTERNAL,
`Failed to construct insert mutation: ${e.message}`);
}
}

/**
* Insert multiple rows into the specified table.
*/
public async insertMany<GraphQlResponse, Variables extends Array<unknown>>(
tableName: string,
data: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
if (!validator.isNonEmptyString(tableName)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`tableName` must be a non-empty string.');
}
if (!validator.isNonEmptyArray(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be a non-empty array for insertMany.');
}

try {
tableName = this.formatTableName(tableName);
const gqlDataString = this.objectToString(data);
const mutation = `mutation { ${tableName}_insertMany(data: ${gqlDataString}) }`;
// Use internal executeGraphql
return this.executeGraphql<GraphQlResponse, Variables>(mutation).catch(this.handleBulkImportErrors);
} catch (e: any) {
throw new FirebaseDataConnectError(DATA_CONNECT_ERROR_CODE_MAPPING.INTERNAL,
`Failed to construct insertMany mutation: ${e.message}`);
}
}

/**
* Insert a single row into the specified table, or update it if it already exists.
*/
public async upsert<GraphQlResponse, Variables extends object>(
tableName: string,
data: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
if (!validator.isNonEmptyString(tableName)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`tableName` must be a non-empty string.');
}
if (validator.isArray(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be an object, not an array, for single upsert.');
}
if (!validator.isNonNullObject(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be a non-null object.');
}

try {
tableName = this.formatTableName(tableName);
const gqlDataString = this.objectToString(data);
const mutation = `mutation { ${tableName}_upsert(data: ${gqlDataString}) }`;
// Use internal executeGraphql
return this.executeGraphql<GraphQlResponse, Variables>(mutation).catch(this.handleBulkImportErrors);
} catch (e: any) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INTERNAL,
`Failed to construct upsert mutation: ${e.message}`);
}
}

/**
* Insert multiple rows into the specified table, or update them if they already exist.
*/
public async upsertMany<GraphQlResponse, Variables extends Array<unknown>>(
tableName: string,
data: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
if (!validator.isNonEmptyString(tableName)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`tableName` must be a non-empty string.');
}
if (!validator.isNonEmptyArray(data)) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INVALID_ARGUMENT,
'`data` must be a non-empty array for upsertMany.');
}

try {
tableName = this.formatTableName(tableName);
const gqlDataString = this.objectToString(data);
const mutation = `mutation { ${tableName}_upsertMany(data: ${gqlDataString}) }`;
// Use internal executeGraphql
return this.executeGraphql<GraphQlResponse, Variables>(mutation).catch(this.handleBulkImportErrors);
} catch (e: any) {
throw new FirebaseDataConnectError(
DATA_CONNECT_ERROR_CODE_MAPPING.INTERNAL,
`Failed to construct upsertMany mutation: ${e.message}`);
}
}
}

/**
Expand Down
94 changes: 77 additions & 17 deletions src/data-connect/data-connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ export class DataConnectService {
}

/**
* Returns the app associated with this `DataConnectService` instance.
*
* @returns The app associated with this `DataConnectService` instance.
*/
* Returns the app associated with this `DataConnectService` instance.
*
* @returns The app associated with this `DataConnectService` instance.
*/
get app(): App {
return this.appInternal;
}
Expand All @@ -63,24 +63,24 @@ export class DataConnect {
private readonly client: DataConnectApiClient;

/**
* @param connectorConfig - The connector configuration.
* @param app - The app for this `DataConnect` service.
* @constructor
* @internal
*/
* @param connectorConfig - The connector configuration.
* @param app - The app for this `DataConnect` service.
* @constructor
* @internal
*/
constructor(readonly connectorConfig: ConnectorConfig, readonly app: App) {
this.client = new DataConnectApiClient(connectorConfig, app);
}

/**
* Execute an arbitrary GraphQL query or mutation
*
* @param query - The GraphQL query or mutation.
* @param options - Optional {@link GraphqlOptions} when executing a GraphQL query or mutation.
*
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
* Execute an arbitrary GraphQL query or mutation
*
* @param query - The GraphQL query or mutation.
* @param options - Optional {@link GraphqlOptions} when executing a GraphQL query or mutation.
*
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
public executeGraphql<GraphqlResponse, Variables>(
query: string,
options?: GraphqlOptions<Variables>,
Expand All @@ -103,4 +103,64 @@ export class DataConnect {
): Promise<ExecuteGraphqlResponse<GraphqlResponse>> {
return this.client.executeGraphqlRead(query, options);
}

/**
* Insert a single row into the specified table.
*
* @param tableName - The name of the table to insert data into.
* @param variables - The data object to insert. The keys should correspond to the column names.
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
public insert<GraphQlResponse, Variables extends object>(
tableName: string,
variables: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
return this.client.insert(tableName, variables);
}

/**
* Insert multiple rows into the specified table.
*
* @param tableName - The name of the table to insert data into.
* @param variables - An array of data objects to insert. Each object's keys should correspond to the column names.
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
public insertMany<GraphQlResponse, Variables extends Array<unknown>>(
tableName: string,
variables: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
return this.client.insertMany(tableName, variables);
}

/**
* Insert a single row into the specified table, or update it if it already exists.
*
* @param tableName - The name of the table to upsert data into.
* @param variables - The data object to upsert. The keys should correspond to the column names.
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
public upsert<GraphQlResponse, Variables extends object>(
tableName: string,
variables: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
return this.client.upsert(tableName, variables);
}

/**
* Insert multiple rows into the specified table, or update them if they already exist.
*
* @param tableName - The name of the table to upsert data into.
* @param variables - An array of data objects to upsert. Each object's keys should correspond to the column names.
* @returns A promise that fulfills with a `ExecuteGraphqlResponse`.
* @beta
*/
public upsertMany<GraphQlResponse, Variables extends Array<unknown>>(
tableName: string,
variables: Variables,
): Promise<ExecuteGraphqlResponse<GraphQlResponse>> {
return this.client.upsertMany(tableName, variables);
}
}
Loading