diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 1538806e39..1a673fb78d 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -11,12 +11,12 @@ catalog entities located in Bitbucket. The processor will crawl your Bitbucket account and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. -> Note: The Bitbucket Discovery Processor currently only supports a self-hosted -> Bitbucket Server, and not the hosted Bitbucket Cloud product. +## Self-hosted Bitbucket Server -To use the discovery processor, you'll need a Bitbucket integration -[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. -Then you can add a location target to the catalog configuration: +To use the discovery processor with a self-hosted Bitbucket Server, you'll need +a Bitbucket integration [set up](locations.md) with a `BITBUCKET_TOKEN` and a +`BITBUCKET_API_BASE_URL`. Then you can add a location target to the catalog +configuration: ```yaml catalog: @@ -44,6 +44,41 @@ The target is composed of four parts: will result in: `https://bitbucket.mycompany.com/projects/my-project/repos/service-a/catalog-info.yaml`. +## Bitbucket Cloud + +To use the discovery processor with Bitbucket Cloud, you'll need a Bitbucket +integration [set up](locations.md) with a `username` and an `appPassword`. Then +you can add a location target to the catalog configuration: + +```yaml +catalog: + locations: + - type: bitbucket-discovery + target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml +``` + +Note the `bitbucket-discovery` type, as this is not a regular `url` processor. + +The target is composed of four parts: + +- The base instance URL, `https://bitbucket.org` in this case +- The workspace name to scan, which must match a workspace accessible with the + username of your integration. +- The project key to scan, which accepts \* wildcard tokens. This can simply be + `*` to include all repositories. This example only returns repositories in the + `my-project` project. +- The repository blob to scan, which accepts \* wildcard tokens. This can simply + be `*` to scan all repositories in the workspace. This example only looks for + repositories prefixed with `service-`. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml` or a similar variation for catalog files + stored in the root directory of each repository. If omitted, the default value + `catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a` + exists, + `https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/` + will result in: + `https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`. + ## Custom repository processing The Bitbucket Discovery Processor will by default emit a location for each diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index e2b295e7ff..43efa8e5d6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -17,7 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketRepositoryParser, PagedResponse } from './bitbucket'; +import { + BitbucketRepository, + BitbucketRepository20, + BitbucketRepositoryParser, + PagedResponse, + PagedResponse20, +} from './bitbucket'; import { results } from './index'; import { RequestHandler, rest } from 'msw'; import { setupServer } from 'msw/node'; @@ -79,6 +85,40 @@ function setupStubs(projects: any[]) { } } +function setupBitbucketCloudStubs( + workspace: string, + repositories: Pick[], +) { + function pagedResponse(values: any): PagedResponse20 { + return { + values: values, + page: 1, + } as PagedResponse20; + } + + server.use( + rest.get( + `https://api.bitbucket.org/2.0/repositories/${workspace}`, + (_, res, ctx) => { + return res( + ctx.json( + pagedResponse( + repositories.map(r => ({ + ...r, + links: { + source: { + href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`, + }, + }, + })), + ), + ), + ); + }, + ), + ); +} + describe('BitbucketDiscoveryProcessor', () => { beforeAll(() => server.listen()); afterEach(() => server.resetHandlers()); @@ -129,7 +169,7 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); - describe('handles repositories', () => { + describe('handles organisation repositories', () => { const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { @@ -255,6 +295,142 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); + describe('handles cloud repositories', () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: 'myuser', + appPassword: 'blob', + }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + + it('output all repositories', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml', + }, + optional: true, + }); + }); + + it('output repositories with wildcards', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + }, + optional: true, + }); + }); + it('filter unrelated repositories', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-one' }, slug: 'repository-two' }, + { project: { key: 'prj-one' }, slug: 'repository-three' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml', + }, + optional: true, + }); + }); + + it.each` + target + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'} + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'} + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'} + `("target '$target' adds default path to catalog", async ({ target }) => { + setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]); + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: target, + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + }); + }); + describe('Custom repository parser', () => { const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) { yield results.location( diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 9d53b7e5b5..dd5ff0d4af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -17,6 +17,7 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { + BitbucketIntegration, ScmIntegrationRegistry, ScmIntegrations, } from '@backstage/integration'; @@ -26,10 +27,16 @@ import { BitbucketClient, defaultRepositoryParser, paginated, + paginated20, BitbucketRepository, + BitbucketRepository20, } from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +const DEFAULT_BRANCH = 'master'; +const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; +const EMPTY_CATALOG_LOCATION = '/'; + export class BitbucketDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; private readonly parser: BitbucketRepositoryParser; @@ -71,10 +78,6 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { throw new Error( `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, ); - } else if (integration.config.host === 'bitbucket.org') { - throw new Error( - `Component discovery for Bitbucket Cloud is not yet supported`, - ); } const client = new BitbucketClient({ @@ -83,38 +86,89 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); - const { catalogPath } = parseUrl(location.target); - const expandedCatalogPath = - catalogPath === '/' ? '/catalog-info.yaml' : catalogPath; + const isBitbucketCloud = integration.config.host === 'bitbucket.org'; - const result = await readBitbucketOrg(client, location.target); + const processOptions: ProcessOptions = { + client, + emit, + integration, + location, + }; + const { scanned, matches } = isBitbucketCloud + ? await this.processCloudRepositories(processOptions) + : await this.processOrganisationRepositories(processOptions); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${scanned} Bitbucket repositories (${matches} matching the pattern) in ${duration} seconds`, + ); + + return true; + } + + private async processCloudRepositories( + options: ProcessOptions, + ): Promise { + const { client, location, integration, emit } = options; + + const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl( + location.target, + ); + const catalogPath = + requestedCatalogPath === EMPTY_CATALOG_LOCATION + ? DEFAULT_CATALOG_LOCATION + : requestedCatalogPath; + const result = await readBitbucketCloud(client, location.target); for (const repository of result.matches) { + const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH; for await (const entity of this.parser({ - integration: integration, - target: `${repository.links.self[0].href}${expandedCatalogPath}`, + integration, + target: `${repository.links.source.href}/${mainbranch}${catalogPath}`, logger: this.logger, })) { emit(entity); } } + return { + matches: result.matches.length, + scanned: result.scanned, + }; + } - const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); - this.logger.debug( - `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`, - ); - - return true; + private async processOrganisationRepositories( + options: ProcessOptions, + ): Promise { + const { client, location, integration, emit } = options; + const { catalogPath: requestedCatalogPath } = parseUrl(location.target); + const catalogPath = + requestedCatalogPath === EMPTY_CATALOG_LOCATION + ? DEFAULT_CATALOG_LOCATION + : requestedCatalogPath; + const result = await readBitbucketOrg(client, location.target); + for (const repository of result.matches) { + for await (const entity of this.parser({ + integration, + target: `${repository.links.self[0].href}${catalogPath}`, + logger: this.logger, + })) { + emit(entity); + } + } + return { + matches: result.matches.length, + scanned: result.scanned, + }; } } export async function readBitbucketOrg( client: BitbucketClient, target: string, -): Promise { +): Promise> { const { projectSearchPath, repoSearchPath } = parseUrl(target); const projects = paginated(options => client.listProjects(options)); - const result: Result = { + const result: Result = { scanned: 0, matches: [], }; @@ -136,6 +190,35 @@ export async function readBitbucketOrg( return result; } +export async function readBitbucketCloud( + client: BitbucketClient, + target: string, +): Promise> { + const { + workspacePath, + projectSearchPath, + repoSearchPath, + } = parseBitbucketCloudUrl(target); + const repositories = paginated20(options => + client.listRepositoriesByWorkspace20(workspacePath, options), + ); + const result: Result = { + scanned: 0, + matches: [], + }; + + for await (const repository of repositories) { + result.scanned++; + if ( + projectSearchPath.test(repository.project.key) && + repoSearchPath.test(repository.slug) + ) { + result.matches.push(repository); + } + } + return result; +} + function parseUrl( urlString: string, ): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { @@ -154,11 +237,47 @@ function parseUrl( throw new Error(`Failed to parse ${urlString}`); } +function parseBitbucketCloudUrl( + urlString: string, +): { + workspacePath: string; + projectSearchPath: RegExp; + repoSearchPath: RegExp; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml} + if (path.length > 5 && path[1].length && path[3].length) { + return { + workspacePath: decodeURIComponent(path[1]), + projectSearchPath: escapeRegExp(decodeURIComponent(path[3])), + repoSearchPath: escapeRegExp(decodeURIComponent(path[5])), + catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + function escapeRegExp(str: string): RegExp { return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } -type Result = { - scanned: number; - matches: BitbucketRepository[]; +type ProcessOptions = { + client: BitbucketClient; + integration: BitbucketIntegration; + location: LocationSpec; + emit: CatalogProcessorEmit; +}; + +type Result = { + scanned: number; + matches: T[]; +}; + +type ResultSummary = { + scanned: number; + matches: number; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 5a1419dc27..bdea349180 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -19,6 +19,7 @@ import { BitbucketIntegrationConfig, getBitbucketRequestOptions, } from '@backstage/integration'; +import { BitbucketRepository, BitbucketRepository20 } from './types'; export class BitbucketClient { private readonly config: BitbucketIntegrationConfig; @@ -31,6 +32,16 @@ export class BitbucketClient { return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } + async listRepositoriesByWorkspace20( + workspace: string, + options?: ListOptions, + ): Promise> { + return this.pagedRequest20( + `${this.config.apiBaseUrl}/repositories/${workspace}`, + options, + ); + } + async listRepositories( projectKey: string, options?: ListOptions, @@ -67,6 +78,33 @@ export class BitbucketClient { return repositories as PagedResponse; }); } + + private async pagedRequest20( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + const response = await fetch( + request.toString(), + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(repositories => { + return repositories as PagedResponse20; + }); + } } export type ListOptions = { @@ -84,6 +122,19 @@ export type PagedResponse = { nextPageStart: number; }; +export type ListOptions20 = { + [key: string]: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse20 = { + page: number; + pagelen: number; + size: number; + values: T[]; + next: string; +}; + export async function* paginated( request: (options: ListOptions) => Promise>, options?: ListOptions, @@ -98,3 +149,18 @@ export async function* paginated( } } while (!res.isLastPage); } + +export async function* paginated20( + request: (options: ListOptions20) => Promise>, + options?: ListOptions20, +) { + const opts = options || { page: 1 }; + let res; + do { + res = await request(opts); + opts.page = (opts.page || 1) + 1; + for (const item of res.values) { + yield item; + } + } while (res.next); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index 7adab7746f..4b28c7f115 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { BitbucketClient, paginated } from './client'; +export { BitbucketClient, paginated, paginated20 } from './client'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; -export type { PagedResponse } from './client'; -export type { BitbucketRepository } from './types'; +export type { PagedResponse, PagedResponse20 } from './client'; +export type { BitbucketRepository, BitbucketRepository20 } from './types'; export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts index b273d26874..db8cf69688 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export type BitbucketRepository = { + +type BitbucketRepositoryBase = { project: { key: string; }; slug: string; +}; + +export type BitbucketRepository = BitbucketRepositoryBase & { links: Record< string, { @@ -25,3 +29,33 @@ export type BitbucketRepository = { }[] >; }; + +export type BitbucketRepository20 = BitbucketRepositoryBase & { + links: Record< + | 'self' + | 'source' + | 'html' + | 'avatar' + | 'pullrequests' + | 'commits' + | 'forks' + | 'watchers' + | 'downloads' + | 'hooks', + { + href: string; + name?: string; + } + > & + Record< + 'clone', + { + href: string; + name?: string; + }[] + >; + mainbranch?: { + type: string; + name: string; + }; +};