From c91cc32e306c53351147a64774e4a6da3bb5a206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Fri, 26 Mar 2021 23:02:32 +0000 Subject: [PATCH 1/5] Add Bitbucket Server discovery processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- .../BitbucketDiscoveryProcessor.test.ts | 217 ++++++++++++++++++ .../processors/BitbucketDiscoveryProcessor.ts | 147 ++++++++++++ .../ingestion/processors/bitbucket/client.ts | 129 +++++++++++ .../ingestion/processors/bitbucket/index.ts | 17 ++ 4 files changed, 510 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..4be408f475 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -0,0 +1,217 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { + BitbucketDiscoveryProcessor, + readBitbucketOrg, +} from './BitbucketDiscoveryProcessor'; +import { ConfigReader } from '@backstage/config'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, PagedResponse } from './bitbucket'; + +function pagedResponse(values: any): PagedResponse { + return { + values: values, + isLastPage: true, + } as PagedResponse; +} + +describe('BitbucketDiscoveryProcessor', () => { + const client: jest.Mocked = { + listProjects: jest.fn(), + listRepositories: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-bitbucket-discovery', + target: 'https://bitbucket.mycompany.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { host: 'bitbucket.org', token: 'blob' }, + { host: 'bitbucket.mycompany.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: 'https://not.bitbucket.mycompany.com/foobar', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/, + ); + }); + }); + + describe('handles repositories', () => { + it('output all repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'backstage', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', + }, + ], + }, + }, + ]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'demo', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + }); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + }); + }); + + it('output repositories with wildcards', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { slug: 'backstage' }, + { + slug: 'techdocs-cli', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', + }, + ], + }, + }, + { + slug: 'techdocs-container', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + }); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', + }); + }); + it('filter unrelated repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValue( + pagedResponse([ + { slug: 'abstest' }, + { slug: 'testxyz' }, + { + slug: 'test', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts new file mode 100644 index 0000000000..9c3f9e03b3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +import { ScmIntegrations } from '@backstage/integration'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, pageIterator } from './bitbucket'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { results } from './index'; + +export class BitbucketDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new BitbucketDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket-discovery') { + return false; + } + + const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target) + ?.config; + if (!bitbucketConfig) { + throw new Error( + `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, + ); + } else if (bitbucketConfig.host === 'bitbucket.org') { + throw new Error( + `Component discovery for Bitbucket Cloud is not yet supported`, + ); + } + + const client = new BitbucketClient({ + config: bitbucketConfig, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading Bitbucket repositories from ${location.target}`); + + const repositories = await readBitbucketOrg(client, location.target); + + for (const repository of repositories) { + emit( + results.location( + repository, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.info( + `Read ${repositories.length} Bitbucket repositories in ${duration} seconds`, + ); + + return true; + } +} + +export async function readBitbucketOrg( + client: BitbucketClient, + target: string, +): Promise { + const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); + const projectIterator = pageIterator(options => client.listProjects(options)); + let result: LocationSpec[] = []; + + for await (const page of projectIterator) { + for (const project of page.values) { + if (!projectSearchPath.test(project.key)) { + continue; + } + const repoIterator = pageIterator(options => + client.listRepositories(project.key, options), + ); + for await (const repoPage of repoIterator) { + result = result.concat( + repoPage.values + .filter(v => repoSearchPath.test(v.slug)) + .map(repo => { + return { + type: 'url', + target: `${repo.links.self[0].href}${catalogPath}`, + }; + }), + ); + } + } + } + return result; +} + +function parseUrl( + urlString: string, +): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + if (path.length > 3 && path[1].length && path[3].length) { + return { + projectSearchPath: escapeRegExp(decodeURIComponent(path[1])), + repoSearchPath: escapeRegExp(decodeURIComponent(path[3])), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts new file mode 100644 index 0000000000..4eb70cf3ff --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import fetch from 'cross-fetch'; + +import { + BitbucketIntegrationConfig, + getBitbucketRequestOptions, +} from '@backstage/integration'; + +export class BitbucketClient { + private readonly config: BitbucketIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: BitbucketIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + async listRepositories( + projectKey: string, + options?: ListOptions, + ): Promise> { + return this.pagedRequest( + `${this.config.apiBaseUrl}/projects/${projectKey}/repos`, + options, + ); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + if (options) { + (Object.keys(options) as Array).forEach(key => { + const value: any = options[key] as any; + if (value) { + request.searchParams.append(key, value); + } + }); + } + const response = await fetch( + request.toString(), + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(repositories => { + return repositories as PagedResponse; + }); + } +} + +export type ListOptions = { + limit?: number | undefined; + start?: number | undefined; +}; + +export type PagedResponse = { + size: number; + limit: number; + start: number; + isLastPage: boolean; + values: T[]; + nextPageStart: number; +}; + +export function pageIterator( + pagedRequest: (options: ListOptions) => Promise>, + options?: ListOptions, +): AsyncIterable> { + return { + [Symbol.asyncIterator]: () => { + const opts = options || { start: 0 }; + let finished = false; + return { + async next() { + if (!finished) { + try { + const response = await pagedRequest(opts); + finished = response.isLastPage; + opts.start = response.nextPageStart; + return Promise.resolve({ + value: response, + done: false, + }); + } catch (error) { + return Promise.reject({ + value: undefined, + done: true, + error: error, + }); + } + } else { + opts.start = 0; + finished = false; + return Promise.resolve({ + value: undefined, + done: true, + }); + } + }, + }; + }, + }; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts new file mode 100644 index 0000000000..a8d0a20627 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { BitbucketClient, pageIterator } from './client'; +export type { PagedResponse } from './client'; From 44590510dcc19cabfa0bc13b2c46847ea50531b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Fri, 26 Mar 2021 23:04:53 +0000 Subject: [PATCH 2/5] Add changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- .changeset/tricky-emus-wave.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tricky-emus-wave.md diff --git a/.changeset/tricky-emus-wave.md b/.changeset/tricky-emus-wave.md new file mode 100644 index 0000000000..45099d5fb2 --- /dev/null +++ b/.changeset/tricky-emus-wave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Add Bitbucket Server discovery processor. From ddaff02d11154ae4e8a5432f675489b92c0277e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Fri, 26 Mar 2021 23:23:28 +0000 Subject: [PATCH 3/5] Add documentation for Bitbucket Discovery processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- docs/integrations/bitbucket/discovery.md | 43 +++++++++++++++++++ .../processors/BitbucketDiscoveryProcessor.ts | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docs/integrations/bitbucket/discovery.md diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md new file mode 100644 index 0000000000..093dc82adc --- /dev/null +++ b/docs/integrations/bitbucket/discovery.md @@ -0,0 +1,43 @@ +--- +id: discovery +title: Bitbucket Server Discovery +sidebar_label: Discovery +description: + Automatically discovering catalog entities from repositories in a Bitbucket + Server instance +--- + +The Bitbucket integration has a special discovery processor for discovering +catalog entities within a Bitbucket Server instance. The processor will crawl +the Bitbucket Server instance and register entities matching the configured +path. This can be useful as an alternative to static locations or manually +adding things to the catalog. + +> Note: The Bitbucket Discovery Processor currently only supports a self-hosted +> Bitbucket Server, and not the hosted Bitbucket Cloud product. + +To use the discovery processor, you'll need a Bitbucket integration +[set up](locations.md) with a `BITBUCKET_TOKEN`. Then you can add a location +target to the catalog configuration: + +```yaml +catalog: + locations: + - type: bitbucket-discovery + target: https://bitbucket.mycompany.com/projects/my-project/repos/service-*/catalog-info.yaml +``` + +Note the `bitbucket-discovery` type, as this is not a regular `url` processor. + +The target is composed of four parts: + +- The base instance URL, `https://bitbucket.mycompany.com` in this case +- The project key to scan, which accepts \* wildcard tokens. This can simply be + `*` to scan repositories from all projects. This example only scans for + repositories in the `my-project` project. +- The repository blob to scan, which accepts \* wildcard tokens. This can simply + be `*` to scan all repositories in the organization. This example only looks + for repositories prefixed with `service-`. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml` or a similar variation for catalog files + stored in the root directory of each repository. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 9c3f9e03b3..15e51b6f9f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -130,7 +130,7 @@ function parseUrl( const url = new URL(urlString); const path = url.pathname.substr(1).split('/'); - // /backstage/techdocs-*/blob/master/catalog-info.yaml + // /projects/backstage/repos/techdocs-*/catalog-info.yaml if (path.length > 3 && path[1].length && path[3].length) { return { projectSearchPath: escapeRegExp(decodeURIComponent(path[1])), From 2d2b955ed11be827dbb1f8c6e42b91a7112456fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Mon, 29 Mar 2021 09:11:05 +0000 Subject: [PATCH 4/5] Add BitbucketDiscoveryProcessor to the list of enabled processors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- .changeset/tricky-emus-wave.md | 2 +- .../src/ingestion/processors/BitbucketDiscoveryProcessor.ts | 1 - .../src/ingestion/processors/bitbucket/client.ts | 5 +---- plugins/catalog-backend/src/ingestion/processors/index.ts | 1 + plugins/catalog-backend/src/service/CatalogBuilder.ts | 2 ++ 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.changeset/tricky-emus-wave.md b/.changeset/tricky-emus-wave.md index 45099d5fb2..69512381f3 100644 --- a/.changeset/tricky-emus-wave.md +++ b/.changeset/tricky-emus-wave.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Add Bitbucket Server discovery processor. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 15e51b6f9f..6c13a40770 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -63,7 +63,6 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const client = new BitbucketClient({ config: bitbucketConfig, - logger: this.logger, }); const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 4eb70cf3ff..b03dcd02bf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Logger } from 'winston'; import fetch from 'cross-fetch'; import { @@ -23,11 +22,9 @@ import { export class BitbucketClient { private readonly config: BitbucketIntegrationConfig; - private readonly logger: Logger; - constructor(options: { config: BitbucketIntegrationConfig; logger: Logger }) { + constructor(options: { config: BitbucketIntegrationConfig }) { this.config = options.config; - this.logger = options.logger; } async listProjects(options?: ListOptions): Promise> { diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index fe4fe1c1dd..8aedc8889b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,6 +19,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index e0b447c686..b8d8b3ee61 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,6 +38,7 @@ import { import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, + BitbucketDiscoveryProcessor, BuiltinKindsEntityProcessor, CatalogProcessor, CatalogProcessorParser, @@ -304,6 +305,7 @@ export class CatalogBuilder { if (!this.processorsReplace) { processors.push( new FileReaderProcessor(), + BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), From 13abbc15da7743005cfe6c9123499af73eddb728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20A=CC=8Ahsberg?= Date: Tue, 30 Mar 2021 10:38:32 +0000 Subject: [PATCH 5/5] Refactor the Bitbucket Discovery processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mathias Åhsberg --- docs/integrations/bitbucket/discovery.md | 20 +++--- docs/integrations/bitbucket/locations.md | 24 +++---- .../BitbucketDiscoveryProcessor.test.ts | 15 ++-- .../processors/BitbucketDiscoveryProcessor.ts | 70 +++++++++++-------- .../ingestion/processors/bitbucket/client.ts | 62 +++++----------- .../ingestion/processors/bitbucket/index.ts | 2 +- 6 files changed, 88 insertions(+), 105 deletions(-) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 093dc82adc..b31071be7f 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -1,24 +1,22 @@ --- id: discovery -title: Bitbucket Server Discovery +title: Bitbucket Discovery sidebar_label: Discovery description: - Automatically discovering catalog entities from repositories in a Bitbucket - Server instance + Automatically discovering catalog entities from repositories in Bitbucket --- The Bitbucket integration has a special discovery processor for discovering -catalog entities within a Bitbucket Server instance. The processor will crawl -the Bitbucket Server instance and register entities matching the configured -path. This can be useful as an alternative to static locations or manually -adding things to the catalog. +catalog entities located in Bitbucket. The processor will crawl your Bitbucket +account and register entities matching the configured path. This can be useful +as an alternative to static locations or manually adding things to the catalog. > Note: The Bitbucket Discovery Processor currently only supports a self-hosted > Bitbucket Server, and not the hosted Bitbucket Cloud product. To use the discovery processor, you'll need a Bitbucket integration -[set up](locations.md) with a `BITBUCKET_TOKEN`. Then you can add a location -target to the catalog configuration: +[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. +Then you can add a location target to the catalog configuration: ```yaml catalog: @@ -36,8 +34,8 @@ The target is composed of four parts: `*` to scan repositories from all projects. This example only scans for repositories in the `my-project` project. - The repository blob to scan, which accepts \* wildcard tokens. This can simply - be `*` to scan all repositories in the organization. This example only looks - for repositories prefixed with `service-`. + be `*` to scan all repositories in the project. This example only looks for + repositories prefixed with `service-`. - The path within each repository to find the catalog YAML file. This will usually be `/catalog-info.yaml` or a similar variation for catalog files stored in the root directory of each repository. diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucket/locations.md index bf7a6af57c..33490455d4 100644 --- a/docs/integrations/bitbucket/locations.md +++ b/docs/integrations/bitbucket/locations.md @@ -1,12 +1,12 @@ --- id: locations -title: BitBucket Locations +title: Bitbucket Locations sidebar_label: Locations description: - Integrating source code stored in BitBucket into the Backstage catalog + Integrating source code stored in Bitbucket into the Backstage catalog --- -The BitBucket integration supports loading catalog entities from bitbucket.com +The Bitbucket integration supports loading catalog entities from bitbucket.org or a self-hosted BitBucket. Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the @@ -21,21 +21,21 @@ integrations: token: ${BITBUCKET_TOKEN} ``` -> Note: A public BitBucket provider is added automatically at startup for +> Note: A public Bitbucket provider is added automatically at startup for > convenience, so you only need to list it if you want to supply a > [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html). Directly under the `bitbucket` key is a list of provider configurations, where -you can list the BitBucket providers you want to fetch data from. Each entry is +you can list the Bitbucket providers you want to fetch data from. Each entry is a structure with up to four elements: -- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`. -- `token` (optional): An personal access token as expected by BitBucket. Either +- `host`: The host of the Bitbucket instance, e.g. `bitbucket.company.com`. +- `token` (optional): An personal access token as expected by Bitbucket. Either an access token **or** a username + appPassword may be supplied. -- `username`: The BitBucket username to use in API requests. If neither a +- `username`: The Bitbucket username to use in API requests. If neither a username nor token are supplied, anonymous access will be used. -- `appPassword` (optional): The password for the BitBucket user. Only needed +- `appPassword` (optional): The password for the Bitbucket user. Only needed when using `username` instead of `token`. -- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted - installations, it is commonly at `https:///api/v4`. For gitlab.com, this - configuration is not needed as it can be inferred. +- `apiBaseUrl` (optional): The URL of the Bitbucket API. For self-hosted + installations, it is commonly at `https:///rest/api/1.0`. For + bitbucket.org, this configuration is not needed as it can be inferred. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 4be408f475..5197549d85 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -120,12 +120,13 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(2); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', }); - expect(actual).toContainEqual({ + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', @@ -168,13 +169,13 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', }); - expect(actual).toContainEqual({ + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', @@ -206,8 +207,8 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 6c13a40770..f3dae9c112 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -16,14 +16,17 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrationRegistry, + ScmIntegrations, +} from '@backstage/integration'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketClient, pageIterator } from './bitbucket'; +import { BitbucketClient, paginated } from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { results } from './index'; export class BitbucketDiscoveryProcessor implements CatalogProcessor { - private readonly integrations: ScmIntegrations; + private readonly integrations: ScmIntegrationRegistry; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { @@ -35,7 +38,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { }); } - constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + }) { this.integrations = options.integrations; this.logger = options.logger; } @@ -67,9 +73,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); - const repositories = await readBitbucketOrg(client, location.target); + const result = await readBitbucketOrg(client, location.target); - for (const repository of repositories) { + for (const repository of result.matches) { emit( results.location( repository, @@ -82,8 +88,8 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { } const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); - this.logger.info( - `Read ${repositories.length} Bitbucket repositories in ${duration} seconds`, + this.logger.debug( + `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`, ); return true; @@ -93,30 +99,29 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { export async function readBitbucketOrg( client: BitbucketClient, target: string, -): Promise { +): Promise { const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); - const projectIterator = pageIterator(options => client.listProjects(options)); - let result: LocationSpec[] = []; + const projects = paginated(options => client.listProjects(options)); + const result: Result = { + scanned: 0, + matches: [], + }; - for await (const page of projectIterator) { - for (const project of page.values) { - if (!projectSearchPath.test(project.key)) { - continue; - } - const repoIterator = pageIterator(options => - client.listRepositories(project.key, options), - ); - for await (const repoPage of repoIterator) { - result = result.concat( - repoPage.values - .filter(v => repoSearchPath.test(v.slug)) - .map(repo => { - return { - type: 'url', - target: `${repo.links.self[0].href}${catalogPath}`, - }; - }), - ); + for await (const project of projects) { + if (!projectSearchPath.test(project.key)) { + continue; + } + const repositories = paginated(options => + client.listRepositories(project.key, options), + ); + for await (const repository of repositories) { + result.scanned++; + + if (repoSearchPath.test(repository.slug)) { + result.matches.push({ + type: 'url', + target: `${repository.links.self[0].href}${catalogPath}`, + }); } } } @@ -144,3 +149,8 @@ function parseUrl( function escapeRegExp(str: string): RegExp { return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } + +type Result = { + scanned: number; + matches: LocationSpec[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index b03dcd02bf..c3c27aedfc 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -46,14 +46,12 @@ export class BitbucketClient { options?: ListOptions, ): Promise> { const request = new URL(endpoint); - if (options) { - (Object.keys(options) as Array).forEach(key => { - const value: any = options[key] as any; - if (value) { - request.searchParams.append(key, value); - } - }); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } } + const response = await fetch( request.toString(), getBitbucketRequestOptions(this.config), @@ -72,6 +70,7 @@ export class BitbucketClient { } export type ListOptions = { + [key: string]: number | undefined; limit?: number | undefined; start?: number | undefined; }; @@ -85,42 +84,17 @@ export type PagedResponse = { nextPageStart: number; }; -export function pageIterator( - pagedRequest: (options: ListOptions) => Promise>, +export async function* paginated( + request: (options: ListOptions) => Promise>, options?: ListOptions, -): AsyncIterable> { - return { - [Symbol.asyncIterator]: () => { - const opts = options || { start: 0 }; - let finished = false; - return { - async next() { - if (!finished) { - try { - const response = await pagedRequest(opts); - finished = response.isLastPage; - opts.start = response.nextPageStart; - return Promise.resolve({ - value: response, - done: false, - }); - } catch (error) { - return Promise.reject({ - value: undefined, - done: true, - error: error, - }); - } - } else { - opts.start = 0; - finished = false; - return Promise.resolve({ - value: undefined, - done: true, - }); - } - }, - }; - }, - }; +) { + const opts = options || { start: 0 }; + let res; + do { + res = await request(opts); + opts.start = res.nextPageStart; + for (const item of res.values) { + yield item; + } + } while (!res.isLastPage); } diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index a8d0a20627..7e70bcfe7a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -13,5 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { BitbucketClient, pageIterator } from './client'; +export { BitbucketClient, paginated } from './client'; export type { PagedResponse } from './client';