diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..4be408f475 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -0,0 +1,217 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { + BitbucketDiscoveryProcessor, + readBitbucketOrg, +} from './BitbucketDiscoveryProcessor'; +import { ConfigReader } from '@backstage/config'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, PagedResponse } from './bitbucket'; + +function pagedResponse(values: any): PagedResponse { + return { + values: values, + isLastPage: true, + } as PagedResponse; +} + +describe('BitbucketDiscoveryProcessor', () => { + const client: jest.Mocked = { + listProjects: jest.fn(), + listRepositories: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-bitbucket-discovery', + target: 'https://bitbucket.mycompany.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { host: 'bitbucket.org', token: 'blob' }, + { host: 'bitbucket.mycompany.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: 'https://not.bitbucket.mycompany.com/foobar', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/, + ); + }); + }); + + describe('handles repositories', () => { + it('output all repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'backstage', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', + }, + ], + }, + }, + ]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { + slug: 'demo', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + }); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + }); + }); + + it('output repositories with wildcards', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValueOnce( + pagedResponse([ + { slug: 'backstage' }, + { + slug: 'techdocs-cli', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', + }, + ], + }, + }, + { + slug: 'techdocs-container', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + }); + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', + }); + }); + it('filter unrelated repositories', async () => { + const target = + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml'; + + client.listProjects.mockResolvedValue( + pagedResponse([{ key: 'backstage' }]), + ); + client.listRepositories.mockResolvedValue( + pagedResponse([ + { slug: 'abstest' }, + { slug: 'testxyz' }, + { + slug: 'test', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test', + }, + ], + }, + }, + ]), + ); + + const actual = await readBitbucketOrg(client, target); + + expect(actual).toContainEqual({ + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts new file mode 100644 index 0000000000..9c3f9e03b3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +import { ScmIntegrations } from '@backstage/integration'; +import { LocationSpec } from '@backstage/catalog-model'; +import { BitbucketClient, pageIterator } from './bitbucket'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { results } from './index'; + +export class BitbucketDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new BitbucketDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket-discovery') { + return false; + } + + const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target) + ?.config; + if (!bitbucketConfig) { + throw new Error( + `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, + ); + } else if (bitbucketConfig.host === 'bitbucket.org') { + throw new Error( + `Component discovery for Bitbucket Cloud is not yet supported`, + ); + } + + const client = new BitbucketClient({ + config: bitbucketConfig, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading Bitbucket repositories from ${location.target}`); + + const repositories = await readBitbucketOrg(client, location.target); + + for (const repository of repositories) { + emit( + results.location( + repository, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.info( + `Read ${repositories.length} Bitbucket repositories in ${duration} seconds`, + ); + + return true; + } +} + +export async function readBitbucketOrg( + client: BitbucketClient, + target: string, +): Promise { + const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); + const projectIterator = pageIterator(options => client.listProjects(options)); + let result: LocationSpec[] = []; + + for await (const page of projectIterator) { + for (const project of page.values) { + if (!projectSearchPath.test(project.key)) { + continue; + } + const repoIterator = pageIterator(options => + client.listRepositories(project.key, options), + ); + for await (const repoPage of repoIterator) { + result = result.concat( + repoPage.values + .filter(v => repoSearchPath.test(v.slug)) + .map(repo => { + return { + type: 'url', + target: `${repo.links.self[0].href}${catalogPath}`, + }; + }), + ); + } + } + } + return result; +} + +function parseUrl( + urlString: string, +): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + if (path.length > 3 && path[1].length && path[3].length) { + return { + projectSearchPath: escapeRegExp(decodeURIComponent(path[1])), + repoSearchPath: escapeRegExp(decodeURIComponent(path[3])), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts new file mode 100644 index 0000000000..4eb70cf3ff --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import fetch from 'cross-fetch'; + +import { + BitbucketIntegrationConfig, + getBitbucketRequestOptions, +} from '@backstage/integration'; + +export class BitbucketClient { + private readonly config: BitbucketIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: BitbucketIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + async listRepositories( + projectKey: string, + options?: ListOptions, + ): Promise> { + return this.pagedRequest( + `${this.config.apiBaseUrl}/projects/${projectKey}/repos`, + options, + ); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + if (options) { + (Object.keys(options) as Array).forEach(key => { + const value: any = options[key] as any; + if (value) { + request.searchParams.append(key, value); + } + }); + } + const response = await fetch( + request.toString(), + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(repositories => { + return repositories as PagedResponse; + }); + } +} + +export type ListOptions = { + limit?: number | undefined; + start?: number | undefined; +}; + +export type PagedResponse = { + size: number; + limit: number; + start: number; + isLastPage: boolean; + values: T[]; + nextPageStart: number; +}; + +export function pageIterator( + pagedRequest: (options: ListOptions) => Promise>, + options?: ListOptions, +): AsyncIterable> { + return { + [Symbol.asyncIterator]: () => { + const opts = options || { start: 0 }; + let finished = false; + return { + async next() { + if (!finished) { + try { + const response = await pagedRequest(opts); + finished = response.isLastPage; + opts.start = response.nextPageStart; + return Promise.resolve({ + value: response, + done: false, + }); + } catch (error) { + return Promise.reject({ + value: undefined, + done: true, + error: error, + }); + } + } else { + opts.start = 0; + finished = false; + return Promise.resolve({ + value: undefined, + done: true, + }); + } + }, + }; + }, + }; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts new file mode 100644 index 0000000000..a8d0a20627 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { BitbucketClient, pageIterator } from './client'; +export type { PagedResponse } from './client';