Skip to content

Commit ca054b2

Browse files
jlooslijloosli
committed
Added support for documentReference, geopoint, and timestamp data types
1 parent d7b4f3d commit ca054b2

9 files changed

+281
-164
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ node_modules/
3333
*.json
3434
!package.json
3535
!tsconfig.json
36+
!tests/*.json
3637

3738
# Dist
3839
dist/**

‎src/bin/firestore-import.ts

+80-71
Original file line numberDiff line numberDiff line change
@@ -18,99 +18,108 @@ const backupFileParamDescription = 'Filename to store backup. (e.g. backups/full
1818

1919
const nodePathParamKey = 'nodePath';
2020
const nodePathParamDescription = 'Path to database node (has to be a collection) where import will to start (e.g. collectionA/docB/collectionC).' +
21-
' Imports at root level if missing.';
21+
' Imports at root level if missing.';
22+
23+
const yesToImportParamKey = 'yes';
24+
const yesToImportParamDescription = 'Unattended import without confirmation (like hitting "y" from the command line).';
2225

2326
commander.version(packageInfo.version)
24-
.option(`-a, --${accountCredentialsPathParamKey} <path>`, accountCredentialsPathParamDescription)
25-
.option(`-b, --${backupFileParamKey} <path>`, backupFileParamDescription)
26-
.option(`-n, --${nodePathParamKey} <path>`, nodePathParamDescription)
27-
.parse(process.argv);
27+
.option(`-a, --${accountCredentialsPathParamKey} <path>`, accountCredentialsPathParamDescription)
28+
.option(`-b, --${backupFileParamKey} <path>`, backupFileParamDescription)
29+
.option(`-n, --${nodePathParamKey} <path>`, nodePathParamDescription)
30+
.option(`-y, --${yesToImportParamKey}`, yesToImportParamDescription)
31+
.parse(process.argv);
2832

2933
const accountCredentialsPath = commander[accountCredentialsPathParamKey];
3034
if (!accountCredentialsPath) {
31-
console.log(colors.bold(colors.red('Missing: ')) + colors.bold(accountCredentialsPathParamKey) + ' - ' + accountCredentialsPathParamDescription);
32-
commander.help();
33-
process.exit(1);
35+
console.log(colors.bold(colors.red('Missing: ')) + colors.bold(accountCredentialsPathParamKey) + ' - ' + accountCredentialsPathParamDescription);
36+
commander.help();
37+
process.exit(1);
3438
}
3539

3640
if (!fs.existsSync(accountCredentialsPath)) {
37-
console.log(colors.bold(colors.red('Account credentials file does not exist: ')) + colors.bold(accountCredentialsPath));
38-
commander.help();
39-
process.exit(1)
41+
console.log(colors.bold(colors.red('Account credentials file does not exist: ')) + colors.bold(accountCredentialsPath));
42+
commander.help();
43+
process.exit(1)
4044
}
4145

4246
const backupFile = commander[backupFileParamKey];
4347
if (!backupFile) {
44-
console.log(colors.bold(colors.red('Missing: ')) + colors.bold(backupFileParamKey) + ' - ' + backupFileParamDescription);
45-
commander.help();
46-
process.exit(1);
48+
console.log(colors.bold(colors.red('Missing: ')) + colors.bold(backupFileParamKey) + ' - ' + backupFileParamDescription);
49+
commander.help();
50+
process.exit(1);
4751
}
4852

4953
if (!fs.existsSync(backupFile)) {
50-
console.log(colors.bold(colors.red('Backup file does not exist: ')) + colors.bold(backupFile));
51-
commander.help();
52-
process.exit(1)
54+
console.log(colors.bold(colors.red('Backup file does not exist: ')) + colors.bold(backupFile));
55+
commander.help();
56+
process.exit(1)
5357
}
5458

5559
const nodePath = commander[nodePathParamKey];
5660

5761
const importPathPromise = getCredentialsFromFile(accountCredentialsPath)
58-
.then(credentials => {
59-
const db = getFirestoreDBReference(credentials);
60-
return getDBReferenceFromPath(db, nodePath);
61-
});
62+
.then(credentials => {
63+
const db = getFirestoreDBReference(credentials);
64+
return getDBReferenceFromPath(db, nodePath);
65+
});
66+
67+
const unattendedConfirmation = commander[yesToImportParamKey];
6268

6369
Promise.all([loadJsonFile(backupFile), importPathPromise])
64-
.then((res) => {
65-
const [data, pathReference] = res;
66-
const nodeLocation = (<FirebaseFirestore.DocumentReference | FirebaseFirestore.CollectionReference>pathReference)
67-
.path || '[database root]';
68-
// For some reason, Firestore, DocumentReference, and CollectionReference interfaces
69-
// don't show a projectId property even though they do have them.
70-
// @todo: Remove any when that is fixed, or find the correct interface
71-
const projectID = (<any>pathReference).projectId ||
72-
(<any>pathReference).firestore.projectId;
73-
const importText = `About to import data ${backupFile} to the '${projectID}' firestore at '${nodeLocation}'.`;
74-
console.log(`\n\n${colors.bold(colors.blue(importText))}`);
75-
console.log(colors.bgYellow(colors.blue(' === Warning: This will overwrite existing data. Do you want to proceed? === ')));
76-
return new Promise((resolve, reject) => {
77-
prompt.message = 'firestore-import';
78-
prompt.start();
79-
prompt.get({
80-
properties: {
81-
response: {
82-
description: colors.red(`Proceed with import? [y/N] `)
83-
}
84-
}
85-
}, (err: Error, result: any) => {
86-
if (err) {
87-
return reject(err);
88-
}
89-
switch (result.response.trim().toLowerCase()) {
90-
case 'y':
91-
resolve(res);
92-
break;
93-
default:
94-
reject('Import aborted.');
95-
}
96-
})
97-
})
98-
})
99-
.then((res: any) => {
100-
const [data, pathReference] = res;
101-
return firestoreImport(data, pathReference);
102-
})
103-
.then(() => {
104-
console.log(colors.bold(colors.green('All done 🎉')));
105-
})
106-
.catch((error) => {
107-
if (error instanceof Error) {
108-
console.log(colors.red(`${error.name}: ${error.message}`));
109-
console.log(colors.red(error.stack as string));
110-
process.exit(1);
111-
} else {
112-
console.log(colors.red(error));
70+
.then((res) => {
71+
if (unattendedConfirmation) {
72+
return res;
73+
}
74+
const [data, pathReference] = res;
75+
const nodeLocation = (<FirebaseFirestore.DocumentReference | FirebaseFirestore.CollectionReference>pathReference)
76+
.path || '[database root]';
77+
// For some reason, Firestore, DocumentReference, and CollectionReference interfaces
78+
// don't show a projectId property even though they do have them.
79+
// @todo: Remove any when that is fixed, or find the correct interface
80+
const projectID = (<any>pathReference).projectId ||
81+
(<any>pathReference).firestore.projectId;
82+
const importText = `About to import data ${backupFile} to the '${projectID}' firestore at '${nodeLocation}'.`;
83+
console.log(`\n\n${colors.bold(colors.blue(importText))}`);
84+
console.log(colors.bgYellow(colors.blue(' === Warning: This will overwrite existing data. Do you want to proceed? === ')));
85+
return new Promise((resolve, reject) => {
86+
prompt.message = 'firestore-import';
87+
prompt.start();
88+
prompt.get({
89+
properties: {
90+
response: {
91+
description: colors.red(`Proceed with import? [y/N] `)
92+
}
93+
}
94+
}, (err: Error, result: any) => {
95+
if (err) {
96+
return reject(err);
11397
}
114-
});
98+
switch (result.response.trim().toLowerCase()) {
99+
case 'y':
100+
resolve(res);
101+
break;
102+
default:
103+
reject('Import aborted.');
104+
}
105+
})
106+
})
107+
})
108+
.then((res: any) => {
109+
const [data, pathReference] = res;
110+
return firestoreImport(data, pathReference);
111+
})
112+
.then(() => {
113+
console.log(colors.bold(colors.green('All done 🎉')));
114+
})
115+
.catch((error) => {
116+
if (error instanceof Error) {
117+
console.log(colors.red(`${error.name}: ${error.message}`));
118+
console.log(colors.red(error.stack as string));
119+
process.exit(1);
120+
} else {
121+
console.log(colors.red(error));
122+
}
123+
});
115124

116125

‎src/lib/export.ts

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
import * as admin from 'firebase-admin';
21
import {isLikeDocument, isRootOfDatabase, sleep} from "./firestore-helpers";
2+
import * as admin from "firebase-admin";
3+
import {serializeSpecialTypes} from "./helpers";
34

45
const SLEEP_TIME = 1000;
56

6-
//@todo: Add support for data types
7-
87
const exportData = async (startingRef: admin.firestore.Firestore |
98
FirebaseFirestore.DocumentReference |
109
FirebaseFirestore.CollectionReference) => {
@@ -17,7 +16,8 @@ const exportData = async (startingRef: admin.firestore.Firestore |
1716
dataPromise = (<FirebaseFirestore.DocumentReference>startingRef).get().then(snapshot => snapshot.data());
1817
}
1918
return await Promise.all([collectionsPromise, dataPromise]).then(res => {
20-
return Object.assign({}, {'__collections__': res[0]}, res[1]);
19+
// return Object.assign({}, {'__collections__': res[0]}, res[1]);
20+
return {'__collections__': res[0], ...res[1]};
2121
});
2222
}
2323
else {
@@ -79,7 +79,7 @@ const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference
7979
documentPromises.push(new Promise(async (resolve) => {
8080
const docDetails: any = {};
8181
// console.log(docSnapshot.id, '=>', docSnapshot.data());
82-
docDetails[docSnapshot.id] = docSnapshot.data();
82+
docDetails[docSnapshot.id] = serializeSpecialTypes(docSnapshot.data());
8383
const collections = await getCollections(docSnapshot.ref);
8484
docDetails[docSnapshot.id]['__collections__'] = collections;
8585
resolve(docDetails);
@@ -92,4 +92,5 @@ const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference
9292
return results;
9393
};
9494

95+
9596
export default exportData;

‎src/lib/helpers.ts

+56-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,59 @@
11
// From https://stackoverflow.com/questions/8495687/split-array-into-chunks
2+
import * as admin from "firebase-admin";
3+
import DocumentReference = admin.firestore.DocumentReference;
4+
import GeoPoint = admin.firestore.GeoPoint;
5+
import Firestore = FirebaseFirestore.Firestore;
6+
27
const array_chunks = (array: Array<any>, chunk_size: number): Array<Array<any>> => {
3-
return Array(Math.ceil(array.length / chunk_size))
4-
.fill(null)
5-
.map(
6-
(_: any, index: number) => index * chunk_size)
7-
.map((begin: number) => array.slice(begin, begin + chunk_size));
8+
return Array(Math.ceil(array.length / chunk_size))
9+
.fill(null)
10+
.map(
11+
(_: any, index: number) => index * chunk_size)
12+
.map((begin: number) => array.slice(begin, begin + chunk_size));
813
};
9-
export {array_chunks};
14+
15+
16+
const serializeSpecialTypes = (data: any) => {
17+
const cleaned: any = {};
18+
Object.keys(data).map(key => {
19+
let value = data[key];
20+
if (value instanceof Date) {
21+
value = {__datatype__: 'timestamp', value: value.toISOString()};
22+
} else if (value instanceof GeoPoint) {
23+
value = {__datatype__: 'geopoint', value: value};
24+
} else if (value instanceof DocumentReference) {
25+
value = {__datatype__: 'documentReference', value: value.path};
26+
} else if (value === Object(value)) {
27+
value = serializeSpecialTypes(value);
28+
}
29+
cleaned[key] = value;
30+
});
31+
return cleaned;
32+
};
33+
34+
const unserializeSpecialTypes = (data: any, fs: Firestore) => {
35+
const cleaned: any = {};
36+
Object.keys(data).map(key => {
37+
let value = data[key];
38+
if (value instanceof Object) {
39+
if ('__datatype__' in value && 'value' in value) {
40+
switch (value.__datatype__) {
41+
case 'timestamp':
42+
value = new Date(value.value);
43+
break;
44+
case 'geopoint':
45+
value = new admin.firestore.GeoPoint(value.value._latitude, value.value._longitude);
46+
break;
47+
case 'documentReference':
48+
value = fs.doc(value.value);
49+
break;
50+
}
51+
} else {
52+
value = unserializeSpecialTypes(value, fs);
53+
}
54+
}
55+
cleaned[key] = value;
56+
});
57+
return cleaned;
58+
};
59+
export {array_chunks, serializeSpecialTypes, unserializeSpecialTypes};

‎src/lib/import.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import * as admin from "firebase-admin";
22
import {isLikeDocument, isRootOfDatabase} from "./firestore-helpers";
33
import {ICollection} from "./interfaces";
4-
import {array_chunks} from "./helpers";
4+
import {array_chunks, unserializeSpecialTypes} from "./helpers";
55

66
const importData = (data: any,
77
startingRef: admin.firestore.Firestore |
88
FirebaseFirestore.DocumentReference |
99
FirebaseFirestore.CollectionReference): Promise<any> => {
1010

11-
const dataToImport = Object.assign({}, data);
11+
const dataToImport = {...data};
1212
if (isLikeDocument(startingRef)) {
1313
if (!dataToImport.hasOwnProperty('__collections__')) {
1414
throw new Error('Root or document reference doesn\'t contain a __collections__ property.');
@@ -56,10 +56,10 @@ const setDocuments = (data: ICollection, startingRef: FirebaseFirestore.Collecti
5656
});
5757
delete(data[documentKey]['__collections__']);
5858
}
59-
const documentData: any = {};
60-
Object.keys(data[documentKey]).map(field => {
61-
documentData[field] = data[documentKey][field];
62-
});
59+
const documentData: any = unserializeSpecialTypes(data[documentKey], startingRef.firestore);
60+
// Object.keys(data[documentKey]).map(field => {
61+
// documentData[field] = unserializeSpecialTypes(data[documentKey][field], startingRef.firestore);
62+
// });
6363
batch.set(startingRef.doc(documentKey), documentData, {merge: true});
6464
});
6565
return batch.commit();

‎tests/export.spec.ts

+1-33
Original file line numberDiff line numberDiff line change
@@ -3,40 +3,8 @@ import firebaseExport from '../src/lib/export';
33
import {expect} from 'chai';
44

55
const firebasemock: any = require('firebase-mock');
6-
// import * as firebasemock from 'firebase-mock';
76
const DocumentReference: any = require('firebase-mock/src/firestore-document');
8-
9-
const sampleRootData = {
10-
__collections__: {
11-
collectionA: {
12-
docA: {
13-
name: 'john',
14-
__collections__: {
15-
contacts: {
16-
contactDocId: {
17-
name: 'sam',
18-
__collections__: {}
19-
}
20-
}
21-
}
22-
},
23-
docB: {
24-
name: 'billy',
25-
__collections__: {}
26-
}
27-
},
28-
collectionB: {
29-
docC: {
30-
name: 'annie',
31-
__collections__: {}
32-
},
33-
docD: {
34-
name: 'jane',
35-
__collections__: {}
36-
}
37-
}
38-
}
39-
};
7+
const sampleRootData = require('./sampleRootData.json');
408

419
const getCollections = function (this: any): Promise<FirebaseFirestore.CollectionReference[]> {
4210
const self = this;

0 commit comments

Comments
 (0)