From b6801e6d70eff68c44a743e30940c14726b39f38 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Wed, 28 Jul 2021 23:27:11 +1000 Subject: [PATCH] Changed format for target location so that query can be passed directly to bitbucket api Signed-off-by: Bart Breen --- docs/integrations/bitbucket/discovery.md | 52 ++++--- .../BitbucketDiscoveryProcessor.test.ts | 131 +++++++++++++++++- .../processors/BitbucketDiscoveryProcessor.ts | 52 ++++--- .../ingestion/processors/bitbucket/client.ts | 10 +- 4 files changed, 201 insertions(+), 44 deletions(-) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 1a673fb78d..b851a88a3b 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -54,30 +54,44 @@ you can add a location target to the catalog configuration: catalog: locations: - type: bitbucket-discovery - target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml + target: https://bitbucket.org/workspaces/my-workspace ``` Note the `bitbucket-discovery` type, as this is not a regular `url` processor. -The target is composed of four parts: +The target is composed of the following parts: -- The base instance URL, `https://bitbucket.org` in this case -- The workspace name to scan, which must match a workspace accessible with the - username of your integration. -- The project key to scan, which accepts \* wildcard tokens. This can simply be - `*` to include all repositories. This example only returns repositories in the - `my-project` project. -- The repository blob to scan, which accepts \* wildcard tokens. This can simply - be `*` to scan all repositories in the workspace. This example only looks for - repositories prefixed with `service-`. -- The path within each repository to find the catalog YAML file. This will - usually be `/catalog-info.yaml` or a similar variation for catalog files - stored in the root directory of each repository. If omitted, the default value - `catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a` - exists, - `https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/` - will result in: - `https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`. +- The base URL for Bitbucket, `https://bitbucket.org` +- The workspace name to scan (following the `workspaces/` path part), which must + match a workspace accessible with the username of your integration. +- (Optional) The project key to scan (following the `projects/` path part), + which accepts \* wildcard tokens. If ommitted, repositories from all projects + in the workspace are included. +- (Optional) The repository blob to scan (following the `repos/` path part), + which accepts \* wildcard tokens. If ommitted, all repositories in the + workspace are included. +- (Optional) The `catalogPath` query argument to specify the location within + each repository to find the catalog YAML file. This will usually be + `/catalog-info.yaml` or a similar variation for catalog files stored in the + root directory of each repository. If omitted, the default value + `catalog-info.yaml` will be used. +- (Optional) The `q` query argument to be passed through to Bitbucket for + filtering results via the API. This is the most flexible option and will + reduce the amount of API calls if you have a large workspace. + [See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering) + for the query argument (will be passed as the `q` query parameter). + +Examples: + +- `https://bitbucket.org/workspaces/my-workspace/projects/my-project` will find + all repositories in the `my-project` project in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace/repos/service-*` will find all + repositories starting with `service-` in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*` + will find all repositories starting with `service-`, in all projects starting + with `apis-` in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace?q=project.key ~ "my-project"` + will find all repositories in a project containing `my-project` in its key. ## Custom repository processing diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 2c52bf81fd..d522eb3f53 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -88,6 +88,7 @@ function setupBitbucketCloudStubs( workspace: string, repositories: Pick[], ) { + const stubCallerFn = jest.fn(); function pagedResponse(values: any): PagedResponse20 { return { values: values, @@ -99,6 +100,7 @@ function setupBitbucketCloudStubs( rest.get( `https://api.bitbucket.org/2.0/repositories/${workspace}`, (_, res, ctx) => { + stubCallerFn(_); return res( ctx.json( pagedResponse( @@ -116,6 +118,7 @@ function setupBitbucketCloudStubs( }, ), ); + return stubCallerFn; } describe('BitbucketDiscoveryProcessor', () => { @@ -310,6 +313,77 @@ describe('BitbucketDiscoveryProcessor', () => { { logger: getVoidLogger() }, ); + it('output all repositories by default', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: 'https://bitbucket.org/workspaces/myworkspace', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('uses provided catalog path', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace?catalogPath=my/nested/path/catalog.yaml', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml', + }, + optional: true, + }); + }); + it('output all repositories', async () => { setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, @@ -318,7 +392,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -354,7 +428,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -371,6 +445,7 @@ describe('BitbucketDiscoveryProcessor', () => { optional: true, }); }); + it('filter unrelated repositories', async () => { setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, @@ -380,7 +455,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -398,13 +473,42 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); + it('submits query', async () => { + const mockCall = setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace?q=project.key ~ "prj-one"', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + expect(mockCall).toBeCalledTimes(1); + // it should be possible to do this via an `expect.objectContaining` check but seems to fail with some encoding issue. + expect(mockCall.mock.calls[0][0].url).toMatchInlineSnapshot( + `"https://api.bitbucket.org/2.0/repositories/myworkspace?page=1&pagelen=100&q=project.key+%7E+%22prj-one%22"`, + ); + }); + it.each` target ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'} ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'} ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'} `("target '$target' adds default path to catalog", async ({ target }) => { - setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]); setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, ]); @@ -428,6 +532,25 @@ describe('BitbucketDiscoveryProcessor', () => { optional: true, }); }); + + it.each` + target + ${'https://bitbucket.org/test'} + `("target '$target' is rejected", async ({ target }) => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: target, + }; + + const emitter = jest.fn(); + await expect( + processor.readLocation(location, false, emitter), + ).rejects.toThrow(/Failed to parse /); + }); }); describe('Custom repository parser', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 3521c98af5..4784aee2ea 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -196,11 +196,16 @@ export async function readBitbucketCloud( ): Promise> { const { workspacePath, + queryParam: q, projectSearchPath, repoSearchPath, } = parseBitbucketCloudUrl(target); - const repositories = paginated20(options => - client.listRepositoriesByWorkspace20(workspacePath, options), + + const repositories = paginated20( + options => client.listRepositoriesByWorkspace20(workspacePath, options), + { + q, + }, ); const result: Result = { scanned: 0, @@ -210,8 +215,8 @@ export async function readBitbucketCloud( for await (const repository of repositories) { result.scanned++; if ( - projectSearchPath.test(repository.project.key) && - repoSearchPath.test(repository.slug) + (!projectSearchPath || projectSearchPath.test(repository.project.key)) && + (!repoSearchPath || repoSearchPath.test(repository.slug)) ) { result.matches.push(repository); } @@ -237,28 +242,43 @@ function parseUrl( throw new Error(`Failed to parse ${urlString}`); } +function readPathParameters(pathParts: string[]): Map { + const vals: Record = {}; + for (let i = 0; i < pathParts.length; i += 2) { + if (i + 1 >= pathParts.length) continue; + vals[pathParts[i]] = decodeURIComponent(pathParts[i + 1]); + } + return new Map(Object.entries(vals)); +} + function parseBitbucketCloudUrl( urlString: string, ): { workspacePath: string; - projectSearchPath: RegExp; - repoSearchPath: RegExp; catalogPath: string; + projectSearchPath?: RegExp; + repoSearchPath?: RegExp; + queryParam?: string; } { const url = new URL(urlString); - const path = url.pathname.substr(1).split('/'); + const pathMap = readPathParameters(url.pathname.substr(1).split('/')); + const query = url.searchParams; - // workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml} - if (path.length > 5 && path[1].length && path[3].length) { - return { - workspacePath: decodeURIComponent(path[1]), - projectSearchPath: escapeRegExp(decodeURIComponent(path[3])), - repoSearchPath: escapeRegExp(decodeURIComponent(path[5])), - catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`, - }; + if (!pathMap.has('workspaces')) { + throw new Error(`Failed to parse workspace from ${urlString}`); } - throw new Error(`Failed to parse ${urlString}`); + return { + workspacePath: pathMap.get('workspaces')!, + projectSearchPath: pathMap.has('projects') + ? escapeRegExp(pathMap.get('projects')!) + : undefined, + repoSearchPath: pathMap.has('repos') + ? escapeRegExp(pathMap.get('repos')!) + : undefined, + catalogPath: `/${query.get('catalogPath') || ''}`, + queryParam: query.get('q') || undefined, + }; } function escapeRegExp(str: string): RegExp { diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 5cacd6d6e6..60a2e62bb7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -34,7 +34,7 @@ export class BitbucketClient { async listRepositoriesByWorkspace20( workspace: string, - options?: ListOptions, + options?: ListOptions20, ): Promise> { return this.pagedRequest20( `${this.config.apiBaseUrl}/repositories/${workspace}`, @@ -81,7 +81,7 @@ export class BitbucketClient { private async pagedRequest20( endpoint: string, - options?: ListOptions, + options?: ListOptions20, ): Promise> { const request = new URL(endpoint); for (const key in options) { @@ -123,7 +123,7 @@ export type PagedResponse = { }; export type ListOptions20 = { - [key: string]: number | undefined; + [key: string]: string | number | undefined; page?: number | undefined; pagelen?: number | undefined; }; @@ -155,11 +155,11 @@ export async function* paginated20( request: (options: ListOptions20) => Promise>, options?: ListOptions20, ) { - const opts = options || { page: 1, pagelen: 100 }; + const opts = { page: 1, pagelen: 100, ...options }; let res; do { res = await request(opts); - opts.page = (opts.page || 1) + 1; + opts.page = opts.page + 1; for (const item of res.values) { yield item; }