From 1af628f1e2307f3d6af1441b4c69d7c190a0e9d7 Mon Sep 17 00:00:00 2001 From: Akshay Gupta Date: Mon, 11 Nov 2024 12:18:10 +0530 Subject: [PATCH] Fix bitbucket sync (#3198) * increase number of sets returned to 100 * formatting * add changeset * update bitbucket storage test * update fetchJsonfiles to read more than 100 files * formatting * update bitbucket test * formatting * fix linting issues --- .changeset/healthy-rabbits-fetch.md | 5 ++ .../src/storage/BitbucketTokenStorage.ts | 76 ++++++++++--------- .../__tests__/BitbucketTokenStorage.test.ts | 56 ++++++-------- 3 files changed, 71 insertions(+), 66 deletions(-) create mode 100644 .changeset/healthy-rabbits-fetch.md diff --git a/.changeset/healthy-rabbits-fetch.md b/.changeset/healthy-rabbits-fetch.md new file mode 100644 index 000000000..e2dbaf509 --- /dev/null +++ b/.changeset/healthy-rabbits-fetch.md @@ -0,0 +1,5 @@ +--- +"@tokens-studio/figma-plugin": patch +--- + +fixed an issue with pulling token sets from Bitbucket when multi-file sync is enabled, wherein all the token sets were not being pulled. diff --git a/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts b/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts index bb4947f45..9c187c9af 100644 --- a/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts +++ b/packages/tokens-studio-for-figma/src/storage/BitbucketTokenStorage.ts @@ -1,6 +1,3 @@ -/* eslint-disable no-else-return */ -/* eslint-disable @typescript-eslint/indent */ -/* eslint "@typescript-eslint/no-unused-vars": off */ import { Bitbucket, Schema } from 'bitbucket'; import compact from 'just-compact'; import { @@ -145,31 +142,40 @@ export class BitbucketTokenStorage extends GitTokenStorage { */ private async fetchJsonFilesFromDirectory(url: string): Promise { - const response = await fetch(url, { - headers: { - Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`, - }, - cache: 'no-cache', + let allJsonFiles: any[] = []; + let nextPageUrl: string | null = `${url}?pagelen=100`; + + while (nextPageUrl) { + const response = await fetch(nextPageUrl, { + headers: { + Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`, + }, + cache: 'no-cache', }); - if (!response.ok) { - throw new Error(`Failed to read from Bitbucket: ${response.statusText}`); - } + if (!response.ok) { + throw new Error(`Failed to read from Bitbucket: ${response.statusText}`); + } - const data = await response.json(); - if (data.values && Array.isArray(data.values)) { - let jsonFiles = data.values.filter((file: any) => file.path.endsWith('.json')); + const data = await response.json(); + if (data.values && Array.isArray(data.values)) { + const jsonFiles = data.values.filter((file: any) => file.path.endsWith('.json')); + allJsonFiles = allJsonFiles.concat(jsonFiles); - const subDirectoryFiles = await Promise.all( - data.values - .filter((file: any) => file.type === 'commit_directory') - .map(async (directory: any) => await this.fetchJsonFilesFromDirectory(directory.links.self.href)), - ); + // Fetch files from subdirectories recursively + const subDirectoryFiles = await Promise.all( + data.values + .filter((file: any) => file.type === 'commit_directory') + .map(async (directory: any) => await this.fetchJsonFilesFromDirectory(directory.links.self.href)), + ); - jsonFiles = jsonFiles.concat(...subDirectoryFiles); - return jsonFiles; - } - return data; + allJsonFiles = allJsonFiles.concat(...subDirectoryFiles); + } + + nextPageUrl = data.next || null; + } + + return allJsonFiles; } public async read(): Promise { @@ -180,14 +186,14 @@ export class BitbucketTokenStorage extends GitTokenStorage { const jsonFiles = await this.fetchJsonFilesFromDirectory(url); if (Array.isArray(jsonFiles)) { - const jsonFileContents = await Promise.all( - jsonFiles.map((file: any) => fetch(file.links.self.href, { + const jsonFileContents = await Promise.all( + jsonFiles.map((file: any) => fetch(file.links.self.href, { headers: { Authorization: `Basic ${btoa(`${this.username}:${this.secret}`)}`, }, cache: 'no-cache', }).then((rsp) => rsp.text())), - ); + ); // Process the content of each JSON file return jsonFileContents.map((fileContent, index) => { const { path } = jsonFiles[index]; @@ -219,7 +225,7 @@ export class BitbucketTokenStorage extends GitTokenStorage { data: parsed as AnyTokenSet, }; }); - } else if (jsonFiles) { + } if (jsonFiles) { const parsed = jsonFiles as GitSingleFileObject; return [ { @@ -229,12 +235,12 @@ export class BitbucketTokenStorage extends GitTokenStorage { }, ...(parsed.$metadata ? [ - { - type: 'metadata' as const, - path: this.path, - data: parsed.$metadata, - }, - ] + { + type: 'metadata' as const, + path: this.path, + data: parsed.$metadata, + }, + ] : []), ...( Object.entries(parsed).filter(([key]) => !Object.values(SystemFilenames).includes(key)) as [ @@ -290,8 +296,8 @@ export class BitbucketTokenStorage extends GitTokenStorage { * const response = await createOrUpdateFiles(params); */ public async createOrUpdateFiles({ - owner, repo, branch, changes, -}: CreatedOrUpdatedFileType) { + owner, repo, branch, changes, + }: CreatedOrUpdatedFileType) { const { message, files } = changes[0]; const data = new FormData(); diff --git a/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts b/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts index 49f73870e..b1e369a10 100644 --- a/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts +++ b/packages/tokens-studio-for-figma/src/storage/__tests__/BitbucketTokenStorage.test.ts @@ -113,56 +113,50 @@ describe('BitbucketTokenStorage', () => { }); it('can read from Git in single file format', async () => { - mockFetch.mockImplementationOnce(() => Promise.resolve({ - ok: true, - json: () => Promise.resolve({ global: { red: { name: 'red', type: 'color', value: '#ff0000' } } }), - })); + mockFetch + .mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + values: [ + { path: '$themes.json', type: 'commit_file', links: { self: { href: 'https://api.bitbucket.org/file/$themes.json' } } }, + { path: 'global.json', type: 'commit_file', links: { self: { href: 'https://api.bitbucket.org/file/global.json' } } }, + ], + next: null, // No further pagination needed for this test + }), + })) + .mockImplementationOnce(() => Promise.resolve({ + ok: true, + text: () => Promise.resolve(JSON.stringify({ $themes: [] })), + })) + .mockImplementationOnce(() => Promise.resolve({ + ok: true, + text: () => Promise.resolve(JSON.stringify({ red: { name: 'red', type: 'color', value: '#ff0000' } })), + })); const result = await storageProvider.read(); - expect(result).toEqual([ - { - path: '/$themes.json', - type: 'themes', - data: [], - }, - { - path: '/global.json', - name: 'global', - type: 'tokenSet', - data: { - red: { - name: 'red', - type: 'color', - value: '#ff0000', - }, - }, - }, - ]); - - storageProvider.changePath('data/core.json'); expect(result).toEqual([ { - path: '/$themes.json', + path: '$themes.json', type: 'themes', - data: [], + data: { $themes: [] }, }, { - path: '/global.json', + path: 'global.json', name: 'global', type: 'tokenSet', data: { red: { - type: 'color', name: 'red', + type: 'color', value: '#ff0000', }, }, }, ]); - expect(mockFetch).toBeCalledWith( - `https://api.bitbucket.org/2.0/repositories/${storageProvider.owner}/${storageProvider.repository}/src/${storageProvider.branch}/`, + expect(mockFetch).toHaveBeenCalledWith( + `https://api.bitbucket.org/2.0/repositories/${storageProvider.owner}/${storageProvider.repository}/src/${storageProvider.branch}/?pagelen=100`, { headers: { Authorization: `Basic ${btoa('myusername:mock-secret')}`,