diff --git a/.changeset/tricky-emus-wave.md b/.changeset/tricky-emus-wave.md new file mode 100644 index 0000000000..69512381f3 --- /dev/null +++ b/.changeset/tricky-emus-wave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add Bitbucket Server discovery processor. diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md new file mode 100644 index 0000000000..b31071be7f --- /dev/null +++ b/docs/integrations/bitbucket/discovery.md @@ -0,0 +1,41 @@ +--- +id: discovery +title: Bitbucket Discovery +sidebar_label: Discovery +description: + Automatically discovering catalog entities from repositories in Bitbucket +--- + +The Bitbucket integration has a special discovery processor for discovering +catalog entities located in Bitbucket. The processor will crawl your Bitbucket +account and register entities matching the configured path. This can be useful +as an alternative to static locations or manually adding things to the catalog. + +> Note: The Bitbucket Discovery Processor currently only supports a self-hosted +> Bitbucket Server, and not the hosted Bitbucket Cloud product. + +To use the discovery processor, you'll need a Bitbucket integration +[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. +Then you can add a location target to the catalog configuration: + +```yaml +catalog: + locations: + - type: bitbucket-discovery + target: https://bitbucket.mycompany.com/projects/my-project/repos/service-*/catalog-info.yaml +``` + +Note the `bitbucket-discovery` type, as this is not a regular `url` processor. + +The target is composed of four parts: + +- The base instance URL, `https://bitbucket.mycompany.com` in this case +- The project key to scan, which accepts \* wildcard tokens. This can simply be + `*` to scan repositories from all projects. This example only scans for + repositories in the `my-project` project. +- The repository blob to scan, which accepts \* wildcard tokens. This can simply + be `*` to scan all repositories in the project. This example only looks for + repositories prefixed with `service-`. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml` or a similar variation for catalog files + stored in the root directory of each repository. diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucket/locations.md index bf7a6af57c..33490455d4 100644 --- a/docs/integrations/bitbucket/locations.md +++ b/docs/integrations/bitbucket/locations.md @@ -1,12 +1,12 @@ --- id: locations -title: BitBucket Locations +title: Bitbucket Locations sidebar_label: Locations description: - Integrating source code stored in BitBucket into the Backstage catalog + Integrating source code stored in Bitbucket into the Backstage catalog --- -The BitBucket integration supports loading catalog entities from bitbucket.com +The Bitbucket integration supports loading catalog entities from bitbucket.org or a self-hosted BitBucket. Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the @@ -21,21 +21,21 @@ integrations: token: ${BITBUCKET_TOKEN} ``` -> Note: A public BitBucket provider is added automatically at startup for +> Note: A public Bitbucket provider is added automatically at startup for > convenience, so you only need to list it if you want to supply a > [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html). Directly under the `bitbucket` key is a list of provider configurations, where -you can list the BitBucket providers you want to fetch data from. Each entry is +you can list the Bitbucket providers you want to fetch data from. Each entry is a structure with up to four elements: -- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`. -- `token` (optional): An personal access token as expected by BitBucket. Either +- `host`: The host of the Bitbucket instance, e.g. `bitbucket.company.com`. +- `token` (optional): An personal access token as expected by Bitbucket. Either an access token **or** a username + appPassword may be supplied. -- `username`: The BitBucket username to use in API requests. If neither a +- `username`: The Bitbucket username to use in API requests. If neither a username nor token are supplied, anonymous access will be used. -- `appPassword` (optional): The password for the BitBucket user. Only needed +- `appPassword` (optional): The password for the Bitbucket user. Only needed when using `username` instead of `token`. -- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted - installations, it is commonly at `https:///api/v4`. For gitlab.com, this - configuration is not needed as it can be inferred. +- `apiBaseUrl` (optional): The URL of the Bitbucket API. For self-hosted + installations, it is commonly at `https:///rest/api/1.0`. For + bitbucket.org, this configuration is not needed as it can be inferred. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..5197549d85 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { + BitbucketDiscoveryProcessor, + readBitbucketOrg, +} from './BitbucketDiscoveryProcessor'; +import { ConfigReader } from '@backstage/config'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, PagedResponse } from './bitbucket'; + +function pagedResponse(values: any): PagedResponse { + return { + values: values, + isLastPage: true, + } as PagedResponse; +} + +describe('BitbucketDiscoveryProcessor', () => { + const client: jest.Mocked = { + listProjects: jest.fn(), + listRepositories: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-bitbucket-discovery', + target: 'https://bitbucket.mycompany.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { host: 'bitbucket.org', token: 'blob' }, + { host: 'bitbucket.mycompany.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: 'https://not.bitbucket.mycompany.com/foobar', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/, + ); + }); + }); + + describe('handles repositories', () => { + it('output all repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'backstage', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', + }, + ], + }, + }, + ]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'demo', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + expect(actual.scanned).toBe(2); + expect(actual.matches).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + }); + expect(actual.matches).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + }); + }); + + it('output repositories with wildcards', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { slug: 'backstage' }, + { + slug: 'techdocs-cli', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', + }, + ], + }, + }, + { + slug: 'techdocs-container', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + }); + expect(actual.matches).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', + }); + }); + it('filter unrelated repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValue( + pagedResponse([ + { slug: 'abstest' }, + { slug: 'testxyz' }, + { + slug: 'test', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts new file mode 100644 index 0000000000..f3dae9c112 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +import { + ScmIntegrationRegistry, + ScmIntegrations, +} from '@backstage/integration'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, paginated } from './bitbucket'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { results } from './index'; + +export class BitbucketDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrationRegistry; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new BitbucketDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket-discovery') { + return false; + } + + const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target) + ?.config; + if (!bitbucketConfig) { + throw new Error( + `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, + ); + } else if (bitbucketConfig.host === 'bitbucket.org') { + throw new Error( + `Component discovery for Bitbucket Cloud is not yet supported`, + ); + } + + const client = new BitbucketClient({ + config: bitbucketConfig, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading Bitbucket repositories from ${location.target}`); + + const result = await readBitbucketOrg(client, location.target); + + for (const repository of result.matches) { + emit( + results.location( + repository, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`, + ); + + return true; + } +} + +export async function readBitbucketOrg( + client: BitbucketClient, + target: string, +): Promise { + const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); + const projects = paginated(options => client.listProjects(options)); + const result: Result = { + scanned: 0, + matches: [], + }; + + for await (const project of projects) { + if (!projectSearchPath.test(project.key)) { + continue; + } + const repositories = paginated(options => + client.listRepositories(project.key, options), + ); + for await (const repository of repositories) { + result.scanned++; + + if (repoSearchPath.test(repository.slug)) { + result.matches.push({ + type: 'url', + target: `${repository.links.self[0].href}${catalogPath}`, + }); + } + } + } + return result; +} + +function parseUrl( + urlString: string, +): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /projects/backstage/repos/techdocs-*/catalog-info.yaml + if (path.length > 3 && path[1].length && path[3].length) { + return { + projectSearchPath: escapeRegExp(decodeURIComponent(path[1])), + repoSearchPath: escapeRegExp(decodeURIComponent(path[3])), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} + +type Result = { + scanned: number; + matches: LocationSpec[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts new file mode 100644 index 0000000000..c3c27aedfc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -0,0 +1,100 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fetch from 'cross-fetch'; + +import { + BitbucketIntegrationConfig, + getBitbucketRequestOptions, +} from '@backstage/integration'; + +export class BitbucketClient { + private readonly config: BitbucketIntegrationConfig; + + constructor(options: { config: BitbucketIntegrationConfig }) { + this.config = options.config; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + async listRepositories( + projectKey: string, + options?: ListOptions, + ): Promise> { + return this.pagedRequest( + `${this.config.apiBaseUrl}/projects/${projectKey}/repos`, + options, + ); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + const response = await fetch( + request.toString(), + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(repositories => { + return repositories as PagedResponse; + }); + } +} + +export type ListOptions = { + [key: string]: number | undefined; + limit?: number | undefined; + start?: number | undefined; +}; + +export type PagedResponse = { + size: number; + limit: number; + start: number; + isLastPage: boolean; + values: T[]; + nextPageStart: number; +}; + +export async function* paginated( + request: (options: ListOptions) => Promise>, + options?: ListOptions, +) { + const opts = options || { start: 0 }; + let res; + do { + res = await request(opts); + opts.start = res.nextPageStart; + for (const item of res.values) { + yield item; + } + } while (!res.isLastPage); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts new file mode 100644 index 0000000000..7e70bcfe7a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { BitbucketClient, paginated } from './client'; +export type { PagedResponse } from './client'; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index fe4fe1c1dd..8aedc8889b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,6 +19,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index e0b447c686..b8d8b3ee61 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,6 +38,7 @@ import { import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, + BitbucketDiscoveryProcessor, BuiltinKindsEntityProcessor, CatalogProcessor, CatalogProcessorParser, @@ -304,6 +305,7 @@ export class CatalogBuilder { if (!this.processorsReplace) { processors.push( new FileReaderProcessor(), + BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }),