From 6fd70f8bc8a85a60e0c927c0cb6555552766398e Mon Sep 17 00:00:00 2001 From: Dhruval Darji Date: Fri, 7 Jan 2022 11:19:10 -0500 Subject: [PATCH] fix(catalog-backend): support Bitbucket servers with custom BaseURLs Previously, the `BitbucketDiscoveryProcessor` assumed that all Bitbucket servers would have paths formed in the pattern of `/projects/{project}/repos/{repo}`. However, since Bitbucket servers have the ability to configure custom baseUrls, this assumption does not always hold true. Instead of evaluating project and repo terms from the full target path, the processor will find the `/projects/` url segment, and start path evaluations from its index instead. This path substring is then used to determine the `projectSearchPath` and the `repoSearchPath`. Support is also added to accept search params for the `catalogPath` as well. Fixes #8771 Signed-off-by: Dhruval Darji --- .changeset/metal-worms-flash.md | 5 + .../BitbucketDiscoveryProcessor.test.ts | 227 ++++++++++++++++-- .../processors/BitbucketDiscoveryProcessor.ts | 8 +- 3 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 .changeset/metal-worms-flash.md diff --git a/.changeset/metal-worms-flash.md b/.changeset/metal-worms-flash.md new file mode 100644 index 0000000000..dd06abdb34 --- /dev/null +++ b/.changeset/metal-worms-flash.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Provide support for Bitbucket servers with custom BaseURLs. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 3f8d0ba0fb..7ace995fda 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -29,7 +29,10 @@ import { setupServer } from 'msw/node'; const server = setupServer(); -function setupStubs(projects: any[]) { +function setupStubs( + projects: any[], + bitbucketBaseUrl = `https://bitbucket.mycompany.com`, +) { function pagedResponse(values: any): PagedResponse { return { values: values, @@ -42,7 +45,7 @@ function setupStubs(projects: any[]) { repos: string[], ): RequestHandler { return rest.get( - `https://bitbucket.mycompany.com/api/rest/1.0/projects/${project}/repos`, + `${bitbucketBaseUrl}/api/rest/1.0/projects/${project}/repos`, (_, res, ctx) => { const response = []; for (const repo of repos) { @@ -51,7 +54,7 @@ function setupStubs(projects: any[]) { links: { self: [ { - href: `https://bitbucket.mycompany.com/projects/${project}/repos/${repo}/browse`, + href: `${bitbucketBaseUrl}/projects/${project}/repos/${repo}/browse`, }, ], }, @@ -63,20 +66,17 @@ function setupStubs(projects: any[]) { } server.use( - rest.get( - `https://bitbucket.mycompany.com/api/rest/1.0/projects`, - (_, res, ctx) => { - return res( - ctx.json( - pagedResponse( - projects.map(p => { - return { key: p.key }; - }), - ), + rest.get(`${bitbucketBaseUrl}/api/rest/1.0/projects`, (_, res, ctx) => { + return res( + ctx.json( + pagedResponse( + projects.map(p => { + return { key: p.key }; + }), ), - ); - }, - ), + ), + ); + }), ); for (const project of projects) { @@ -249,6 +249,30 @@ describe('BitbucketDiscoveryProcessor', () => { optional: true, }); }); + + it('output repositories by target search ref', async () => { + setupStubs([{ key: 'demo', repos: ['demo'] }]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/catalog.yaml?ref=branch-name', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml?ref=branch-name', + presence: 'optional', + }, + optional: true, + }); + }); + it('filter unrelated repositories', async () => { setupStubs([{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }]); const location: LocationSpec = { @@ -302,6 +326,177 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); + describe('handles organisation repositories with a custom baseURL', () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.mycompany.com', + token: 'blob', + apiBaseUrl: + 'https://bitbucket.mycompany.com/custom-path/api/rest/1.0', + }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + + it('output all repositories', async () => { + setupStubs( + [ + { key: 'backstage', repos: ['backstage'] }, + { key: 'demo', repos: ['demo'] }, + ], + 'https://bitbucket.mycompany.com/custom-path', + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/*/repos/*/catalog.yaml', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/backstage/browse/catalog.yaml', + presence: 'optional', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/browse/catalog.yaml', + presence: 'optional', + }, + optional: true, + }); + }); + + it('output repositories with wildcards', async () => { + setupStubs( + [ + { key: 'backstage', repos: ['backstage', 'techdocs-cli'] }, + { key: 'demo', repos: ['demo'] }, + ], + 'https://bitbucket.mycompany.com/custom-path', + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-*/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + presence: 'optional', + }, + optional: true, + }); + }); + + it('output repositories by target search ref', async () => { + setupStubs( + [{ key: 'demo', repos: ['demo'] }], + 'https://bitbucket.mycompany.com/custom-path', + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/catalog.yaml?ref=branch-name', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/demo/repos/demo/browse/catalog.yaml?ref=branch-name', + presence: 'optional', + }, + optional: true, + }); + }); + + it('filter unrelated repositories', async () => { + setupStubs( + [{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }], + 'https://bitbucket.mycompany.com/custom-path', + ); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/test/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/test/browse/catalog.yaml', + presence: 'optional', + }, + optional: true, + }); + }); + + it.each` + target + ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/*'} + ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/*/'} + ${'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-*/'} + `("target '$target' adds default path to catalog", async ({ target }) => { + setupStubs( + [{ key: 'backstage', repos: ['techdocs-cli'] }], + 'https://bitbucket.mycompany.com/custom-path', + ); + + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: target, + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.mycompany.com/custom-path/projects/backstage/repos/techdocs-cli/browse/catalog-info.yaml', + presence: 'optional', + }, + optional: true, + }); + }); + }); + describe('handles cloud repositories', () => { const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 6a09694ae8..f08888d1ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -230,14 +230,18 @@ function parseUrl(urlString: string): { catalogPath: string; } { const url = new URL(urlString); - const path = url.pathname.substr(1).split('/'); + const indexOfProjectSegment = + url.pathname.toLowerCase().indexOf('/projects/') + 1; + const path = url.pathname.substr(indexOfProjectSegment).split('/'); // /projects/backstage/repos/techdocs-*/catalog-info.yaml if (path.length > 3 && path[1].length && path[3].length) { return { projectSearchPath: escapeRegExp(decodeURIComponent(path[1])), repoSearchPath: escapeRegExp(decodeURIComponent(path[3])), - catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + catalogPath: `/${decodeURIComponent( + path.slice(4).join('/') + url.search, + )}`, }; }