diff --git a/.changeset/stale-carpets-poke.md b/.changeset/stale-carpets-poke.md new file mode 100644 index 0000000000..4f39e57b71 --- /dev/null +++ b/.changeset/stale-carpets-poke.md @@ -0,0 +1,30 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Externalize repository processing for BitbucketDiscoveryProcessor. + +Add an extension point where you can customize how a matched Bitbucket repository should +be processed. This can for example be used if you want to generate the catalog-info.yaml +automatically based on other files in a repository, while taking advantage of the +build-in repository crawling functionality. + +`BitbucketDiscoveryProcessor.fromConfig` now takes an optional parameter `options.parser` where +you can customize the logic for each repository found. The default parser has the same +behaviour as before, where it emits an optional location for the matched repository +and lets the other processors take care of further processing. + +```typescript +const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({ + client, + repository, +}) { + // Custom logic for interpret the matching repository. + // See defaultRepositoryParser for an example +}; + +const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, { + parser: customRepositoryParser, + logger: env.logger, +}); +``` diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index b24734abde..fadb7c9f06 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -39,3 +39,29 @@ The target is composed of four parts: - The path within each repository to find the catalog YAML file. This will usually be `/catalog-info.yaml` or a similar variation for catalog files stored in the root directory of each repository. + +## Custom repository processing + +The Bitbucket Discovery Processor will by default emit a location for each +matching repository for further processing by other processors. However, it is +possible to override this functionality and take full control of how each +matching repository is processed. + +`BitbucketDiscoveryProcessor.fromConfig` takes an optional parameter +`options.parser` where you can set your own parser to be used for each matched +repository. + +```typescript +const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({ + client, + repository, +}) { + // Custom logic for interpret the matching repository. + // See defaultRepositoryParser for an example +}; + +const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, { + parser: customRepositoryParser, + logger: env.logger, +}); +``` diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 5197549d85..73f4281a51 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -14,13 +14,15 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; -import { - BitbucketDiscoveryProcessor, - readBitbucketOrg, -} from './BitbucketDiscoveryProcessor'; +import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketClient, PagedResponse } from './bitbucket'; +import { + BitbucketClient, + BitbucketRepositoryParser, + PagedResponse, +} from './bitbucket'; +import { results } from './index'; function pagedResponse(values: any): PagedResponse { return { @@ -30,11 +32,6 @@ function pagedResponse(values: any): PagedResponse { } describe('BitbucketDiscoveryProcessor', () => { - const client: jest.Mocked = { - listProjects: jest.fn(), - listRepositories: jest.fn(), - } as any; - afterEach(() => jest.resetAllMocks()); describe('reject unrelated entries', () => { @@ -81,137 +78,236 @@ describe('BitbucketDiscoveryProcessor', () => { }); describe('handles repositories', () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + it('output all repositories', async () => { - const target = - 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml'; - - client.listProjects.mockResolvedValue( - pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), - ); - client.listRepositories.mockResolvedValueOnce( - pagedResponse([ - { - slug: 'backstage', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', - }, - ], - }, - }, - ]), - ); - client.listRepositories.mockResolvedValueOnce( - pagedResponse([ - { - slug: 'demo', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', - }, - ], - }, - }, - ]), - ); - - const actual = await readBitbucketOrg(client, target); - expect(actual.scanned).toBe(2); - expect(actual.matches).toContainEqual({ - type: 'url', + const location: LocationSpec = { + type: 'bitbucket-discovery', target: - 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml', + }; + + jest + .spyOn(BitbucketClient.prototype, 'listProjects') + .mockResolvedValue( + pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), + ); + jest + .spyOn(BitbucketClient.prototype, 'listRepositories') + .mockResolvedValueOnce( + pagedResponse([ + { + slug: 'backstage', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', + }, + ], + }, + }, + ]), + ); + jest + .spyOn(BitbucketClient.prototype, 'listRepositories') + .mockResolvedValueOnce( + pagedResponse([ + { + slug: 'demo', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', + }, + ], + }, + }, + ]), + ); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + }, + optional: true, }); - expect(actual.matches).toContainEqual({ - type: 'url', - target: - 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + }, + optional: true, }); }); it('output repositories with wildcards', async () => { - const target = - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml'; - - client.listProjects.mockResolvedValue( - pagedResponse([{ key: 'backstage' }]), - ); - client.listRepositories.mockResolvedValueOnce( - pagedResponse([ - { slug: 'backstage' }, - { - slug: 'techdocs-cli', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', - }, - ], - }, - }, - { - slug: 'techdocs-container', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', - }, - ], - }, - }, - ]), - ); - - const actual = await readBitbucketOrg(client, target); - expect(actual.scanned).toBe(3); - expect(actual.matches).toContainEqual({ - type: 'url', + const location: LocationSpec = { + type: 'bitbucket-discovery', target: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml', + }; + + jest + .spyOn(BitbucketClient.prototype, 'listProjects') + .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); + jest + .spyOn(BitbucketClient.prototype, 'listRepositories') + .mockResolvedValueOnce( + pagedResponse([ + { slug: 'backstage' }, + { + slug: 'techdocs-cli', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', + }, + ], + }, + }, + { + slug: 'techdocs-container', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', + }, + ], + }, + }, + ]), + ); + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + }, + optional: true, }); - expect(actual.matches).toContainEqual({ - type: 'url', - target: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', + }, + optional: true, }); }); it('filter unrelated repositories', async () => { - const target = - 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml'; - - client.listProjects.mockResolvedValue( - pagedResponse([{ key: 'backstage' }]), - ); - client.listRepositories.mockResolvedValue( - pagedResponse([ - { slug: 'abstest' }, - { slug: 'testxyz' }, - { - slug: 'test', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/test', - }, - ], - }, - }, - ]), - ); - - const actual = await readBitbucketOrg(client, target); - expect(actual.scanned).toBe(3); - expect(actual.matches).toContainEqual({ - type: 'url', + const location: LocationSpec = { + type: 'bitbucket-discovery', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }; + + jest + .spyOn(BitbucketClient.prototype, 'listProjects') + .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); + jest + .spyOn(BitbucketClient.prototype, 'listRepositories') + .mockResolvedValue( + pagedResponse([ + { slug: 'abstest' }, + { slug: 'testxyz' }, + { + slug: 'test', + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test', + }, + ], + }, + }, + ]), + ); + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }, + optional: true, + }); + }); + }); + + describe('Custom repository parser', () => { + const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) { + yield results.location( + { + type: 'custom-location-type', + target: 'custom-target', + }, + true, + ); + }; + + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + }, + }), + { parser: customRepositoryParser, logger: getVoidLogger() }, + ); + + it('use custom repository parser', async () => { + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + }; + + jest + .spyOn(BitbucketClient.prototype, 'listProjects') + .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); + jest + .spyOn(BitbucketClient.prototype, 'listRepositories') + .mockResolvedValue(pagedResponse([{ slug: 'test' }])); + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'custom-location-type', + target: 'custom-target', + }, + optional: true, }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index f3dae9c112..f92a5c6f7c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -21,15 +21,24 @@ import { ScmIntegrations, } from '@backstage/integration'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketClient, paginated } from './bitbucket'; +import { + Repository, + BitbucketRepositoryParser, + BitbucketClient, + defaultRepositoryParser, + paginated, +} from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { results } from './index'; export class BitbucketDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; + private readonly parser: BitbucketRepositoryParser; private readonly logger: Logger; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { parser?: BitbucketRepositoryParser; logger: Logger }, + ) { const integrations = ScmIntegrations.fromConfig(config); return new BitbucketDiscoveryProcessor({ @@ -40,9 +49,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrationRegistry; + parser?: BitbucketRepositoryParser; logger: Logger; }) { this.integrations = options.integrations; + this.parser = options.parser || defaultRepositoryParser; this.logger = options.logger; } @@ -73,18 +84,18 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); + const { catalogPath } = parseUrl(location.target); + const result = await readBitbucketOrg(client, location.target); for (const repository of result.matches) { - emit( - results.location( - repository, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - true, - ), - ); + for await (const entity of this.parser({ + client: client, + repository: repository, + path: catalogPath, + })) { + emit(entity); + } } const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); @@ -100,7 +111,7 @@ export async function readBitbucketOrg( client: BitbucketClient, target: string, ): Promise { - const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); + const { projectSearchPath, repoSearchPath } = parseUrl(target); const projects = paginated(options => client.listProjects(options)); const result: Result = { scanned: 0, @@ -116,12 +127,8 @@ export async function readBitbucketOrg( ); for await (const repository of repositories) { result.scanned++; - if (repoSearchPath.test(repository.slug)) { - result.matches.push({ - type: 'url', - target: `${repository.links.self[0].href}${catalogPath}`, - }); + result.matches.push(repository); } } } @@ -152,5 +159,5 @@ function escapeRegExp(str: string): RegExp { type Result = { scanned: number; - matches: LocationSpec[]; + matches: Repository[]; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts new file mode 100644 index 0000000000..ab5080f446 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { defaultRepositoryParser } from './BitbucketRepositoryParser'; +import { Project, Repository } from './types'; +import { BitbucketClient } from './client'; +import { results } from '../index'; + +describe('BitbucketRepositoryParser', () => { + describe('defaultRepositoryParser', () => { + it('emits location', async () => { + const browseUrl = + 'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse'; + const path = '/catalog-info.yaml'; + const expected = [ + results.location( + { + type: 'url', + target: `${browseUrl}${path}`, + }, + true, + ), + ]; + const actual = await defaultRepositoryParser({ + client: {} as BitbucketClient, + repository: { + project: {} as Project, + slug: 'repo-slug', + links: { + self: [{ href: browseUrl }], + }, + } as Repository, + path: path, + }); + + let i = 0; + for await (const entity of actual) { + expect(entity).toStrictEqual(expected[i]); + i++; + } + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts new file mode 100644 index 0000000000..f786b6dac8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Repository } from './types'; +import { CatalogProcessorResult } from '../types'; +import { results } from '../index'; +import { BitbucketClient } from './client'; + +export type BitbucketRepositoryParser = (options: { + client: BitbucketClient; + repository: Repository; + path: string; +}) => AsyncIterable; + +export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({ + repository, + path, +}) { + yield results.location( + { + type: 'url', + target: `${repository.links.self[0].href}${path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ); +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index c3c27aedfc..601461dcf5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -41,6 +41,15 @@ export class BitbucketClient { ); } + async getRaw( + projectKey: string, + repo: string, + path: string, + ): Promise { + const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`; + return fetch(request, getBitbucketRequestOptions(this.config)); + } + private async pagedRequest( endpoint: string, options?: ListOptions, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index 7e70bcfe7a..ba2a2b3afe 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -15,3 +15,6 @@ */ export { BitbucketClient, paginated } from './client'; export type { PagedResponse } from './client'; +export * from './types'; +export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser'; +export { defaultRepositoryParser } from './BitbucketRepositoryParser'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts new file mode 100644 index 0000000000..75dd372faa --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export type Project = { + key: string; +}; + +export type Repository = { + project: Project; + slug: string; + links: Record; +}; + +export type Link = { + href: string; +};