diff --git a/.changeset/khaki-rice-accept.md b/.changeset/khaki-rice-accept.md new file mode 100644 index 0000000000..72f1c635ed --- /dev/null +++ b/.changeset/khaki-rice-accept.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Expose `BitbucketRepositoryParser` introduced in [#5295](https://github.com/backstage/backstage/pull/5295) diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 73f4281a51..1f01b77f42 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -17,21 +17,73 @@ import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; -import { - BitbucketClient, - BitbucketRepositoryParser, - PagedResponse, -} from './bitbucket'; +import { BitbucketRepositoryParser, PagedResponse } from './bitbucket'; import { results } from './index'; +import { RequestHandler, rest } from 'msw'; +import { setupServer } from 'msw/node'; -function pagedResponse(values: any): PagedResponse { - return { - values: values, - isLastPage: true, - } as PagedResponse; +const server = setupServer(); + +function setupStubs(projects: any[]) { + function pagedResponse(values: any): PagedResponse { + return { + values: values, + isLastPage: true, + } as PagedResponse; + } + + function stubbedProject( + project: string, + repos: string[], + ): RequestHandler { + return rest.get( + `https://bitbucket.mycompany.com/api/rest/1.0/projects/${project}/repos`, + (_, res, ctx) => { + const response = []; + for (const repo of repos) { + response.push({ + slug: repo, + links: { + self: [ + { + href: `https://bitbucket.mycompany.com/projects/${project}/repos/${repo}/browse`, + }, + ], + }, + }); + } + return res(ctx.json(pagedResponse(response))); + }, + ); + } + + server.use( + rest.get( + `https://bitbucket.mycompany.com/api/rest/1.0/projects`, + (_, res, ctx) => { + return res( + ctx.json( + pagedResponse( + projects.map(p => { + return { key: p.key }; + }), + ), + ), + ); + }, + ), + ); + + for (const project of projects) { + server.use(stubbedProject(project.key, project.repos)); + } } describe('BitbucketDiscoveryProcessor', () => { + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + afterEach(() => jest.resetAllMocks()); describe('reject unrelated entries', () => { @@ -81,58 +133,29 @@ describe('BitbucketDiscoveryProcessor', () => { const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { - bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + bitbucket: [ + { + host: 'bitbucket.mycompany.com', + token: 'blob', + apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0', + }, + ], }, }), { logger: getVoidLogger() }, ); it('output all repositories', async () => { + setupStubs([ + { key: 'backstage', repos: ['backstage'] }, + { key: 'demo', repos: ['demo'] }, + ]); const location: LocationSpec = { type: 'bitbucket-discovery', target: 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml', }; - jest - .spyOn(BitbucketClient.prototype, 'listProjects') - .mockResolvedValue( - pagedResponse([{ key: 'backstage' }, { key: 'demo' }]), - ); - jest - .spyOn(BitbucketClient.prototype, 'listRepositories') - .mockResolvedValueOnce( - pagedResponse([ - { - slug: 'backstage', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse', - }, - ], - }, - }, - ]), - ); - jest - .spyOn(BitbucketClient.prototype, 'listRepositories') - .mockResolvedValueOnce( - pagedResponse([ - { - slug: 'demo', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse', - }, - ], - }, - }, - ]), - ); const emitter = jest.fn(); await processor.readLocation(location, false, emitter); @@ -158,44 +181,16 @@ describe('BitbucketDiscoveryProcessor', () => { }); it('output repositories with wildcards', async () => { + setupStubs([ + { key: 'backstage', repos: ['backstage', 'techdocs-cli'] }, + { key: 'demo', repos: ['demo'] }, + ]); const location: LocationSpec = { type: 'bitbucket-discovery', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml', }; - jest - .spyOn(BitbucketClient.prototype, 'listProjects') - .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); - jest - .spyOn(BitbucketClient.prototype, 'listRepositories') - .mockResolvedValueOnce( - pagedResponse([ - { slug: 'backstage' }, - { - slug: 'techdocs-cli', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse', - }, - ], - }, - }, - { - slug: 'techdocs-container', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse', - }, - ], - }, - }, - ]), - ); const emitter = jest.fn(); await processor.readLocation(location, false, emitter); @@ -208,46 +203,15 @@ describe('BitbucketDiscoveryProcessor', () => { }, optional: true, }); - expect(emitter).toHaveBeenCalledWith({ - type: 'location', - location: { - type: 'url', - target: - 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', - }, - optional: true, - }); }); it('filter unrelated repositories', async () => { + setupStubs([{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }]); const location: LocationSpec = { type: 'bitbucket-discovery', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', }; - jest - .spyOn(BitbucketClient.prototype, 'listProjects') - .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); - jest - .spyOn(BitbucketClient.prototype, 'listRepositories') - .mockResolvedValue( - pagedResponse([ - { slug: 'abstest' }, - { slug: 'testxyz' }, - { - slug: 'test', - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/backstage/repos/test', - }, - ], - }, - }, - ]), - ); - const emitter = jest.fn(); await processor.readLocation(location, false, emitter); @@ -256,7 +220,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', + 'https://bitbucket.mycompany.com/projects/backstage/repos/test/browse/catalog.yaml', }, optional: true, }); @@ -277,26 +241,27 @@ describe('BitbucketDiscoveryProcessor', () => { const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { - bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }], + bitbucket: [ + { + host: 'bitbucket.mycompany.com', + token: 'blob', + apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0', + }, + ], }, }), { parser: customRepositoryParser, logger: getVoidLogger() }, ); it('use custom repository parser', async () => { + setupStubs([{ key: 'backstage', repos: ['test'] }]); + const location: LocationSpec = { type: 'bitbucket-discovery', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', }; - jest - .spyOn(BitbucketClient.prototype, 'listProjects') - .mockResolvedValue(pagedResponse([{ key: 'backstage' }])); - jest - .spyOn(BitbucketClient.prototype, 'listRepositories') - .mockResolvedValue(pagedResponse([{ slug: 'test' }])); - const emitter = jest.fn(); await processor.readLocation(location, false, emitter); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index f92a5c6f7c..399adfbc14 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -22,11 +22,11 @@ import { } from '@backstage/integration'; import { LocationSpec } from '@backstage/catalog-model'; import { - Repository, BitbucketRepositoryParser, BitbucketClient, defaultRepositoryParser, paginated, + BitbucketRepository, } from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; @@ -66,20 +66,19 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { return false; } - const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target) - ?.config; - if (!bitbucketConfig) { + const integration = this.integrations.bitbucket.byUrl(location.target); + if (!integration) { throw new Error( `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, ); - } else if (bitbucketConfig.host === 'bitbucket.org') { + } else if (integration.config.host === 'bitbucket.org') { throw new Error( `Component discovery for Bitbucket Cloud is not yet supported`, ); } const client = new BitbucketClient({ - config: bitbucketConfig, + config: integration.config, }); const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); @@ -90,9 +89,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { for (const repository of result.matches) { for await (const entity of this.parser({ - client: client, - repository: repository, - path: catalogPath, + integration: integration, + target: `${repository.links.self[0].href}${catalogPath}`, + logger: this.logger, })) { emit(entity); } @@ -159,5 +158,5 @@ function escapeRegExp(str: string): RegExp { type Result = { scanned: number; - matches: Repository[]; + matches: BitbucketRepository[]; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts index ab5080f446..3d42d9c4ba 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ import { defaultRepositoryParser } from './BitbucketRepositoryParser'; -import { Project, Repository } from './types'; -import { BitbucketClient } from './client'; import { results } from '../index'; +import { getVoidLogger } from '@backstage/backend-common'; +import { BitbucketIntegration } from '@backstage/integration'; describe('BitbucketRepositoryParser', () => { describe('defaultRepositoryParser', () => { @@ -34,15 +34,9 @@ describe('BitbucketRepositoryParser', () => { ), ]; const actual = await defaultRepositoryParser({ - client: {} as BitbucketClient, - repository: { - project: {} as Project, - slug: 'repo-slug', - links: { - self: [{ href: browseUrl }], - }, - } as Repository, - path: path, + integration: {} as BitbucketIntegration, + target: `${browseUrl}${path}`, + logger: getVoidLogger(), }); let i = 0; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts index f786b6dac8..9ad038f9ec 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts @@ -13,25 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Repository } from './types'; import { CatalogProcessorResult } from '../types'; import { results } from '../index'; -import { BitbucketClient } from './client'; +import { Logger } from 'winston'; +import { BitbucketIntegration } from '@backstage/integration'; export type BitbucketRepositoryParser = (options: { - client: BitbucketClient; - repository: Repository; - path: string; + integration: BitbucketIntegration; + target: string; + logger: Logger; }) => AsyncIterable; export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({ - repository, - path, + target, }) { yield results.location( { type: 'url', - target: `${repository.links.self[0].href}${path}`, + target: target, }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 601461dcf5..c3c27aedfc 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -41,15 +41,6 @@ export class BitbucketClient { ); } - async getRaw( - projectKey: string, - repo: string, - path: string, - ): Promise { - const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`; - return fetch(request, getBitbucketRequestOptions(this.config)); - } - private async pagedRequest( endpoint: string, options?: ListOptions, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index ba2a2b3afe..06effffcd2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ export { BitbucketClient, paginated } from './client'; -export type { PagedResponse } from './client'; -export * from './types'; -export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; +export type { PagedResponse } from './client'; +export type { BitbucketRepository } from './types'; +export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts index 75dd372faa..32dc28eb98 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts @@ -13,16 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export type Project = { - key: string; -}; - -export type Repository = { - project: Project; +export type BitbucketRepository = { + project: { + key: string; + }; slug: string; - links: Record; -}; - -export type Link = { - href: string; + links: Record< + string, + { + href: string; + }[] + >; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 8aedc8889b..a92cc9ed3b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -35,3 +35,5 @@ export * from './types'; export { UrlReaderProcessor } from './UrlReaderProcessor'; export { parseEntityYaml } from './util/parse'; export { results }; + +export type { BitbucketRepositoryParser } from './bitbucket';