From f7607f9d85b3922ef709e401deb59a394550bfdc Mon Sep 17 00:00:00 2001 From: Otto Nordander Date: Thu, 21 Jul 2022 14:23:29 +0200 Subject: [PATCH 1/3] feat: add BitbucketServerEntityProvider (new plugin) Adds a new entity provider `BitbucketServerEntityProvider` as a new plugin. Signed-off-by: Otto Nordander --- .changeset/stupid-carpets-remain.md | 51 +++ .../.eslintrc.js | 1 + .../CHANGELOG.md | 1 + .../README.md | 9 + .../api-report.md | 104 +++++ .../config.d.ts | 80 ++++ .../package.json | 56 +++ .../src/index.ts | 31 ++ .../src/lib/BitbucketServerClient.test.ts | 182 +++++++++ .../src/lib/BitbucketServerClient.ts | 158 ++++++++ .../src/lib/index.ts | 25 ++ .../src/lib/types.ts | 35 ++ .../BitbucketServerEntityProvider.test.ts | 368 ++++++++++++++++++ .../BitbucketServerEntityProvider.ts | 200 ++++++++++ ...itbucketServerEntityProviderConfig.test.ts | 122 ++++++ .../BitbucketServerEntityProviderConfig.ts | 72 ++++ .../BitbucketServerLocationParser.ts | 42 ++ .../src/providers/index.ts | 18 + .../src/setupTests.ts | 17 + 19 files changed, 1572 insertions(+) create mode 100644 .changeset/stupid-carpets-remain.md create mode 100644 plugins/catalog-backend-module-bitbucket-server/.eslintrc.js create mode 100644 plugins/catalog-backend-module-bitbucket-server/CHANGELOG.md create mode 100644 plugins/catalog-backend-module-bitbucket-server/README.md create mode 100644 plugins/catalog-backend-module-bitbucket-server/api-report.md create mode 100644 plugins/catalog-backend-module-bitbucket-server/config.d.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/package.json create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/index.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/lib/index.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/lib/types.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerLocationParser.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/providers/index.ts create mode 100644 plugins/catalog-backend-module-bitbucket-server/src/setupTests.ts diff --git a/.changeset/stupid-carpets-remain.md b/.changeset/stupid-carpets-remain.md new file mode 100644 index 0000000000..c0b4a54bfd --- /dev/null +++ b/.changeset/stupid-carpets-remain.md @@ -0,0 +1,51 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': minor +--- + +Add new plugin catalog-backend-module-bitbucket-server which adds the `BitbucketServerEntityProvider`. + +The entity provider is meant as a replacement for the `BitbucketDiscoveryProcessor` to be used with Bitbucket Server (Bitbucket Cloud already has a replacement). + +**Before:** + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + BitbucketDiscoveryProcessor.fromConfig(env.config, { logger: env.logger }), +); +``` + +```yaml +# app-config.yaml +catalog: + locations: + - type: bitbucket-discovery + target: 'https://bitbucket.mycompany.com/projects/*/repos/*/catalog-info.yaml +``` + +**After:** + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addEntityProvider( + BitbucketServerEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + }), +); +``` + +```yaml +# app-config.yaml +catalog: + providers: + bitbucketServer: + yourProviderId: # identifies your ingested dataset + catalogPath: /catalog-info.yaml # default value + filters: # optional + projectKey: '.*' # optional; RegExp + repoSlug: '.*' # optional; RegExp +``` diff --git a/plugins/catalog-backend-module-bitbucket-server/.eslintrc.js b/plugins/catalog-backend-module-bitbucket-server/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-bitbucket-server/CHANGELOG.md b/plugins/catalog-backend-module-bitbucket-server/CHANGELOG.md new file mode 100644 index 0000000000..721ebf9ec1 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/CHANGELOG.md @@ -0,0 +1 @@ +# @backstage/plugin-catalog-backend-module-bitbucket-server diff --git a/plugins/catalog-backend-module-bitbucket-server/README.md b/plugins/catalog-backend-module-bitbucket-server/README.md new file mode 100644 index 0000000000..aed8be25ec --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/README.md @@ -0,0 +1,9 @@ +# Catalog Backend Module for Bitbucket Server + +This is an extension module to the catalog-backend plugin, +providing extensions targeted at Bitbucket Server offerings. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/integrations/bitbucketServer/discovery) +for details on how to install and configure the plugin. diff --git a/plugins/catalog-backend-module-bitbucket-server/api-report.md b/plugins/catalog-backend-module-bitbucket-server/api-report.md new file mode 100644 index 0000000000..b087da6746 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/api-report.md @@ -0,0 +1,104 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-bitbucket-server" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BitbucketServerIntegrationConfig } from '@backstage/integration'; +import { Config } from '@backstage/config'; +import { Entity } from '@backstage/catalog-model'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { TaskRunner } from '@backstage/backend-tasks'; + +// @public +export class BitbucketServerClient { + constructor(options: { config: BitbucketServerIntegrationConfig }); + // (undocumented) + static fromConfig(options: { + config: BitbucketServerIntegrationConfig; + }): BitbucketServerClient; + // (undocumented) + getRepository(options: { + projectKey: string; + repo: string; + }): Promise; + // (undocumented) + listProjects(options: { + listOptions?: BitbucketServerListOptions; + }): Promise>; + // (undocumented) + listRepositories(options: { + projectKey: string; + listOptions?: BitbucketServerListOptions; + }): Promise>; + // (undocumented) + resolvePath(options: { projectKey: string; repo: string; path: string }): { + path: string; + }; +} + +// @public +export class BitbucketServerEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule: TaskRunner; + parser?: BitbucketServerLocationParser; + }, + ): BitbucketServerEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(logger: Logger): Promise; +} + +// @public (undocumented) +export type BitbucketServerListOptions = { + [key: string]: number | undefined; + limit?: number | undefined; + start?: number | undefined; +}; + +// @public +export type BitbucketServerLocationParser = (options: { + client: BitbucketServerClient; + location: LocationSpec; + logger: Logger; +}) => AsyncIterable; + +// @public (undocumented) +export type BitbucketServerPagedResponse = { + size: number; + limit: number; + start: number; + isLastPage: boolean; + values: T[]; + nextPageStart: number; +}; + +// @public (undocumented) +export type BitbucketServerProject = { + key: string; +}; + +// @public (undocumented) +export type BitbucketServerRepository = { + project: { + key: string; + }; + slug: string; + description: string; + links: Record< + string, + { + href: string; + }[] + >; +}; +``` diff --git a/plugins/catalog-backend-module-bitbucket-server/config.d.ts b/plugins/catalog-backend-module-bitbucket-server/config.d.ts new file mode 100644 index 0000000000..d8a09c92a4 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/config.d.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + catalog?: { + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * BitbucketServerEntityProvider configuration + * + * Uses "default" as default id for the single config variant. + */ + bitbucketServer?: + | { + /** + * (Optional) Path to the catalog file. Default to "/catalog-info.yaml". + * @visibility frontend + */ + catalogPath?: string; + /** + * (Optional) Filters applied to discovered catalog files in repositories. + * @visibility frontend + */ + filters?: { + /** + * (Optional) Filter for the repository slug. + * @visibility frontend + */ + repoSlug?: RegExp; + /** + * (Optional) Filter for the project key. + * @visibility frontend + */ + projectKey?: RegExp; + }; + } + | Record< + string, + { + /** + * (Optional) Path to the catalog file. Default to "/catalog-info.yaml". + * @visibility frontend + */ + catalogPath?: string; + /** + * (Optional) Filters applied to discovered catalog files in repositories. + * @visibility frontend + */ + filters?: { + /** + * (Optional) Filter for the repository slug. + * @visibility frontend + */ + repoSlug?: RegExp; + /** + * (Optional) Filter for the project key. + * @visibility frontend + */ + projectKey?: RegExp; + }; + } + >; + }; + }; +} diff --git a/plugins/catalog-backend-module-bitbucket-server/package.json b/plugins/catalog-backend-module-bitbucket-server/package.json new file mode 100644 index 0000000000..621f7cd87b --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/package.json @@ -0,0 +1,56 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-bitbucket-server", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-bitbucket-server" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-common": "^0.15.0-next.0", + "@backstage/backend-tasks": "^0.3.4-next.0", + "@backstage/catalog-model": "^1.0.1", + "@backstage/config": "^1.0.0", + "@backstage/errors": "^1.0.0", + "@backstage/integration": "^1.3.0-next.0", + "@backstage/plugin-catalog-backend": "^1.3.1-next.0", + "cross-fetch": "^3.1.5", + "uuid": "^8.0.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "^0.1.27-next.0", + "@backstage/cli": "^0.18.1-next.0", + "msw": "^0.35.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-bitbucket-server/src/index.ts b/plugins/catalog-backend-module-bitbucket-server/src/index.ts new file mode 100644 index 0000000000..34ac3a9966 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards Bitbucket Server + * + * @packageDocumentation + */ + +export { BitbucketServerClient } from './lib'; +export type { + BitbucketServerProject, + BitbucketServerRepository, + BitbucketServerPagedResponse, + BitbucketServerListOptions, +} from './lib'; +export { BitbucketServerEntityProvider } from './providers'; +export type { BitbucketServerLocationParser } from './providers'; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts new file mode 100644 index 0000000000..6b8043409f --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BitbucketServerIntegrationConfig } from '@backstage/integration'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { + BitbucketServerClient, + BitbucketServerPagedResponse, + paginated, +} from './BitbucketServerClient'; +import { BitbucketServerProject, BitbucketServerRepository } from './types'; + +const server = setupServer(); + +describe('BitbucketServerClient', () => { + const config: BitbucketServerIntegrationConfig = { + host: 'bitbucket.mycompany.com', + apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0', + username: 'test-user', + password: 'test-pw', + }; + const client = BitbucketServerClient.fromConfig({ + config: config, + }); + + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); + afterAll(() => server.close()); + afterEach(() => server.resetHandlers()); + + it('listProjects', async () => { + server.use( + rest.get(`${config.apiBaseUrl}/projects`, (req, res, ctx) => { + if ( + req.headers.get('authorization') !== 'Basic dGVzdC11c2VyOnRlc3QtcHc=' + ) { + return res(ctx.status(400)); + } + const response: BitbucketServerPagedResponse = { + size: 1, + limit: 25, + isLastPage: true, + start: 0, + nextPageStart: 0, + values: [ + { + key: 'test-project', + }, + ], + }; + return res(ctx.json(response)); + }), + ); + + const projects = paginated(options => + client.listProjects({ listOptions: options }), + ); + + const results = []; + for await (const project of projects) { + results.push(project); + } + + expect(results).toHaveLength(1); + expect(results[0].key).toEqual('test-project'); + }); + + it('listRepositories', async () => { + server.use( + rest.get( + `${config.apiBaseUrl}/projects/test-project/repos`, + (req, res, ctx) => { + if ( + req.headers.get('authorization') !== + 'Basic dGVzdC11c2VyOnRlc3QtcHc=' + ) { + return res(ctx.status(400)); + } + const response: BitbucketServerPagedResponse = + { + size: 1, + limit: 25, + isLastPage: true, + start: 0, + nextPageStart: 0, + values: [ + { + project: { + key: 'test-project', + }, + slug: 'test-repository', + description: 'A test repository.', + links: { + self: [ + { + href: 'https://bitbucket.mycompany.com/projects/test-project', + }, + ], + }, + }, + ], + }; + return res(ctx.json(response)); + }, + ), + ); + + const repos = paginated(options => + client.listRepositories({ + projectKey: 'test-project', + listOptions: options, + }), + ); + + const results = []; + for await (const repo of repos) { + results.push(repo); + } + + expect(results).toHaveLength(1); + expect(results[0].project.key).toEqual('test-project'); + expect(results[0].slug).toEqual('test-repository'); + expect(results[0].links.self[0].href).toEqual( + 'https://bitbucket.mycompany.com/projects/test-project', + ); + }); + + it('getRepository', async () => { + server.use( + rest.get( + `${config.apiBaseUrl}/projects/test-project/repos/test-repo`, + (req, res, ctx) => { + if ( + req.headers.get('authorization') !== + 'Basic dGVzdC11c2VyOnRlc3QtcHc=' + ) { + return res(ctx.status(400)); + } + const response: BitbucketServerRepository = { + project: { + key: 'test-project', + }, + slug: 'test-repo', + description: 'A test repository.', + links: { + self: [ + { + href: 'https://bitbucket.mycompany.com/projects/test-project', + }, + ], + }, + }; + + return res(ctx.json(response)); + }, + ), + ); + + const repo = await client.getRepository({ + projectKey: 'test-project', + repo: 'test-repo', + }); + expect(repo.project.key).toEqual('test-project'); + expect(repo.slug).toEqual('test-repo'); + expect(repo.links.self[0].href).toEqual( + 'https://bitbucket.mycompany.com/projects/test-project', + ); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts new file mode 100644 index 0000000000..fa233c6863 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -0,0 +1,158 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { fetch, Request } from 'cross-fetch'; +import { + BitbucketServerIntegrationConfig, + getBitbucketServerRequestOptions, +} from '@backstage/integration'; +import { BitbucketServerProject, BitbucketServerRepository } from './types'; + +/** + * A client for interacting with a Bitbucket Server instance + * + * @public + */ +export class BitbucketServerClient { + private readonly config: BitbucketServerIntegrationConfig; + + static fromConfig(options: { + config: BitbucketServerIntegrationConfig; + }): BitbucketServerClient { + return new BitbucketServerClient(options); + } + + constructor(options: { config: BitbucketServerIntegrationConfig }) { + this.config = options.config; + } + + async listProjects(options: { + listOptions?: BitbucketServerListOptions; + }): Promise> { + return this.pagedRequest( + `${this.config.apiBaseUrl}/projects`, + options.listOptions, + ); + } + + async listRepositories(options: { + projectKey: string; + listOptions?: BitbucketServerListOptions; + }): Promise> { + return this.pagedRequest( + `${this.config.apiBaseUrl}/projects/${encodeURIComponent( + options.projectKey, + )}/repos`, + options.listOptions, + ); + } + + async getRepository(options: { + projectKey: string; + repo: string; + }): Promise { + const request = `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}`; + const response = await fetch( + request, + getBitbucketServerRequestOptions(this.config), + ); + return response.json(); + } + + resolvePath(options: { projectKey: string; repo: string; path: string }): { + path: string; + } { + const base = new URL(this.config.apiBaseUrl || ''); + + return { + path: `${base.protocol}//${base.host}/projects/${options.projectKey}/repos/${options.repo}${options.path}`, + }; + } + + private async pagedRequest( + endpoint: string, + options?: BitbucketServerListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + return this.getTypeMapped(request); + } + + private async getTypeMapped(url: URL): Promise { + return this.get(url).then((response: Response) => { + return response.json() as Promise; + }); + } + + private async get(url: URL): Promise { + return this.request(new Request(url.toString(), { method: 'GET' })); + } + + private async request(req: Request): Promise { + return fetch(req, getBitbucketServerRequestOptions(this.config)).then( + (response: Response) => { + if (!response.ok) { + throw new Error( + `Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`, + ); + } + return response; + }, + ); + } +} + +/** + * @public + */ +export type BitbucketServerListOptions = { + [key: string]: number | undefined; + limit?: number | undefined; + start?: number | undefined; +}; + +/** + * @public + */ +export type BitbucketServerPagedResponse = { + size: number; + limit: number; + start: number; + isLastPage: boolean; + values: T[]; + nextPageStart: number; +}; + +export async function* paginated( + request: ( + options: BitbucketServerListOptions, + ) => Promise>, + options?: BitbucketServerListOptions, +) { + const opts = options || { start: 0 }; + let res; + do { + res = await request(opts); + opts.start = res.nextPageStart; + for (const item of res.values) { + yield item; + } + } while (!res.isLastPage); +} diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/index.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/index.ts new file mode 100644 index 0000000000..8b72c9335b --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BitbucketServerClient, paginated } from './BitbucketServerClient'; +export type { + BitbucketServerPagedResponse, + BitbucketServerListOptions, +} from './BitbucketServerClient'; +export type { + BitbucketServerRepository, + BitbucketServerProject, +} from './types'; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/types.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/types.ts new file mode 100644 index 0000000000..67a10abea7 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/types.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** @public */ +export type BitbucketServerRepository = { + project: { + key: string; + }; + slug: string; + description: string; + links: Record< + string, + { + href: string; + }[] + >; +}; + +/** @public */ +export type BitbucketServerProject = { + key: string; +}; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts new file mode 100644 index 0000000000..1ebf7c95a7 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts @@ -0,0 +1,368 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { BitbucketServerPagedResponse } from '../lib/BitbucketServerClient'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +type Project = { + key: string; + repos: [string]; +}; + +function pagedResponse(values: any): BitbucketServerPagedResponse { + return { + values: values, + isLastPage: true, + } as BitbucketServerPagedResponse; +} + +const logger = getVoidLogger(); + +const server = setupServer(); + +function setupStubs(projects: Project[], baseUrl: string) { + // Stub projects + server.use( + rest.get(`${baseUrl}/rest/api/1.0/projects`, (_, res, ctx) => { + return res( + ctx.json( + pagedResponse( + projects.map(p => { + return { key: p.key }; + }), + ), + ), + ); + }), + ); + + for (const project of projects) { + // Stub list repositories + server.use( + rest.get( + `${baseUrl}/rest/api/1.0/projects/${project.key}/repos`, + (_, res, ctx) => { + const response = []; + for (const repo of project.repos) { + response.push({ + slug: repo, + links: { + self: [ + { + href: `${baseUrl}/projects/${project.key}/repos/${repo}/browse`, + }, + ], + }, + }); + } + return res(ctx.json(pagedResponse(response))); + }, + ), + ); + } +} + +describe('BitbucketServerEntityProvider', () => { + setupRequestMockHandlers(server); + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('rejects no matching integration', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + }, + }); + expect(() => + BitbucketServerEntityProvider.fromConfig(config, { logger, schedule }), + ).toThrow(/bitbucket\.mycompany\.com/); + }); + + it('single simple provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + }); + const providers = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual( + 'bitbucketServer-provider:default', + ); + }); + + it('multiple provider configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: 'bitbucket.mycompany.com', + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: 'bitbucket.mycompany.com', + }, + secondary: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + }, + }); + const providers = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + expect(providers[1].getProviderName()).toEqual( + 'bitbucketServer-provider:secondary', + ); + }); + + it('apply full update on scheduled execution with filters', async () => { + const host = 'bitbucket.mycompany.com'; + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: host, + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: host, + filters: { + projectKey: 'project-.*', + repoSlug: 'repo-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + + setupStubs( + [ + { key: 'project-test', repos: ['repo-test'] }, + { key: 'other-project', repos: ['other-repo'] }, + ], + `https://${host}`, + ); + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); + + it('apply full update on scheduled execution without filters', async () => { + const host = 'bitbucket.mycompany.com'; + const config = new ConfigReader({ + integrations: { + bitbucketServer: [ + { + host: host, + }, + ], + }, + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: host, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + const provider = BitbucketServerEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual( + 'bitbucketServer-provider:mainProvider', + ); + + setupStubs( + [ + { key: 'project-test', repos: ['repo-test'] }, + { key: 'other-project', repos: ['other-repo'] }, + ], + `https://${host}`, + ); + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + }, + name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + 'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + }, + name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f', + }, + spec: { + presence: 'optional', + target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`, + type: 'url', + }, + }, + locationKey: 'bitbucketServer-provider:mainProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts new file mode 100644 index 0000000000..92cecf642a --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts @@ -0,0 +1,200 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; +import { TaskRunner } from '@backstage/backend-tasks'; +import * as uuid from 'uuid'; +import { + BitbucketServerLocationParser, + defaultBitbucketServerLocationParser, +} from './BitbucketServerLocationParser'; +import { + BitbucketServerIntegration, + ScmIntegrations, +} from '@backstage/integration'; +import { BitbucketServerClient, paginated } from '../lib'; +import { InputError } from '@backstage/errors'; +import { + BitbucketServerEntityProviderConfig, + readProviderConfigs, +} from './BitbucketServerEntityProviderConfig'; +import { Entity } from '@backstage/catalog-model'; + +/** + * Discovers catalog files located in Bitbucket Server. + * The provider will search your Bitbucket Server instance and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * + * @public + */ +export class BitbucketServerEntityProvider implements EntityProvider { + private readonly integration: BitbucketServerIntegration; + private readonly config: BitbucketServerEntityProviderConfig; + private readonly parser: BitbucketServerLocationParser; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { + logger: Logger; + schedule: TaskRunner; + parser?: BitbucketServerLocationParser; + }, + ): BitbucketServerEntityProvider[] { + const integrations = ScmIntegrations.fromConfig(config); + + return readProviderConfigs(config).map(providerConfig => { + const integration = integrations.bitbucketServer.byHost( + providerConfig.host, + ); + if (!integration) { + throw new InputError( + `No BitbucketServer integration found that matches host ${providerConfig.host}`, + ); + } + return new BitbucketServerEntityProvider( + providerConfig, + integration, + options.logger, + options.schedule, + options.parser, + ); + }); + } + + private constructor( + config: BitbucketServerEntityProviderConfig, + integration: BitbucketServerIntegration, + logger: Logger, + schedule: TaskRunner, + parser?: BitbucketServerLocationParser, + ) { + this.integration = integration; + this.config = config; + this.parser = parser || defaultBitbucketServerLocationParser; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: BitbucketServerEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `bitbucketServer-provider:${this.config.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + async refresh(logger: Logger) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering catalog files in Bitbucket Server repositories'); + + const entities = await this.findEntities(); + + await this.connection.applyMutation({ + type: 'full', + entities: entities.map(entity => ({ + locationKey: this.getProviderName(), + entity: entity, + })), + }); + + logger.info( + `Committed ${entities.length} entities for Bitbucket Server repositories`, + ); + } + + private async findEntities(): Promise { + const client = BitbucketServerClient.fromConfig({ + config: this.integration.config, + }); + const projects = paginated(options => + client.listProjects({ listOptions: options }), + ); + const result: Entity[] = []; + for await (const project of projects) { + if ( + this.config?.filters?.projectKey && + !this.config.filters.projectKey.test(project.key) + ) { + continue; + } + const repositories = paginated(options => + client.listRepositories({ + projectKey: project.key, + listOptions: options, + }), + ); + for await (const repository of repositories) { + if ( + this.config?.filters?.repoSlug && + !this.config.filters.repoSlug.test(repository.slug) + ) { + continue; + } + for await (const entity of this.parser({ + client, + logger: this.logger, + location: { + type: 'url', + target: `${repository.links.self[0].href}${this.config.catalogPath}`, + presence: 'optional', + }, + })) { + result.push(entity); + } + } + } + return result; + } +} diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts new file mode 100644 index 0000000000..712b26cbb0 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './BitbucketServerEntityProviderConfig'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + host: 'bitbucket.mycompany.com', + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0]).toEqual({ + id: 'default', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + }); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: 'bitbucket.mycompany.com', + }, + secondaryProvider: { + host: 'bitbucket2.mycompany.com', + catalogPath: 'custom/path/catalog-info.yaml', + }, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(2); + expect(providerConfigs[0]).toEqual({ + id: 'mainProvider', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + }); + expect(providerConfigs[1]).toEqual({ + id: 'secondaryProvider', + catalogPath: 'custom/path/catalog-info.yaml', + host: 'bitbucket2.mycompany.com', + filters: { + projectKey: undefined, + repoSlug: undefined, + }, + }); + }); + + it('single provider config with filters', () => { + const config = new ConfigReader({ + catalog: { + providers: { + bitbucketServer: { + mainProvider: { + host: 'bitbucket.mycompany.com', + filters: { + projectKey: 'project1', + repoSlug: '.*', + }, + }, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0]).toEqual({ + id: 'mainProvider', + catalogPath: '/catalog-info.yaml', + host: 'bitbucket.mycompany.com', + filters: { + projectKey: /project1/, + repoSlug: /.*/, + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts new file mode 100644 index 0000000000..8a3decab43 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; +const DEFAULT_PROVIDER_ID = 'default'; + +export type BitbucketServerEntityProviderConfig = { + id: string; + host: string; + catalogPath: string; + filters?: { + projectKey?: RegExp; + repoSlug?: RegExp; + }; +}; + +export function readProviderConfigs( + config: Config, +): BitbucketServerEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig( + 'catalog.providers.bitbucketServer', + ); + if (!providersConfig) { + return []; + } + if (providersConfig.has('host')) { + // simple/single config variant + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + const providerConfig = providersConfig.getConfig(id); + + return readProviderConfig(id, providerConfig); + }); +} + +function readProviderConfig( + id: string, + config: Config, +): BitbucketServerEntityProviderConfig { + const host = config.getString('host'); + const catalogPath = + config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; + const projectKeyPattern = config.getOptionalString('filters.projectKey'); + const repoSlugPattern = config.getOptionalString('filters.repoSlug'); + + return { + id, + host, + catalogPath, + filters: { + projectKey: projectKeyPattern ? new RegExp(projectKeyPattern) : undefined, + repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined, + }, + }; +} diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerLocationParser.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerLocationParser.ts new file mode 100644 index 0000000000..7f6dae8872 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerLocationParser.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + LocationSpec, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-backend'; +import { Entity } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import { BitbucketServerClient } from '../lib'; + +/** + * A custom callback that reacts to finding a location by yielding entities. + * Can be used for custom location/repository parsing logic. + * + * @public + */ +export type BitbucketServerLocationParser = (options: { + client: BitbucketServerClient; + location: LocationSpec; + logger: Logger; +}) => AsyncIterable; + +export const defaultBitbucketServerLocationParser = + async function* defaultBitbucketServerLocationParser(options: { + location: LocationSpec; + }) { + yield locationSpecToLocationEntity({ location: options.location }); + }; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/index.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/index.ts new file mode 100644 index 0000000000..8c6430a558 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider'; +export type { BitbucketServerLocationParser } from './BitbucketServerLocationParser'; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/setupTests.ts b/plugins/catalog-backend-module-bitbucket-server/src/setupTests.ts new file mode 100644 index 0000000000..d3232290a7 --- /dev/null +++ b/plugins/catalog-backend-module-bitbucket-server/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; From eadfa84a99a7ce8d7f501b467423ea4716fd7b71 Mon Sep 17 00:00:00 2001 From: Otto Nordander Date: Thu, 21 Jul 2022 14:46:19 +0200 Subject: [PATCH 2/3] Add docs for plugin-catalog-backend-module-bitbucket-server Signed-off-by: Otto Nordander --- docs/integrations/bitbucket/discovery.md | 4 +- .../integrations/bitbucketServer/discovery.md | 114 ++++++++++++++++++ .../locations.md | 23 ++-- microsite/sidebars.json | 13 +- mkdocs.yml | 4 +- 5 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 docs/integrations/bitbucketServer/discovery.md rename docs/integrations/{bitbucket => bitbucketServer}/locations.md (64%) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 285b7cc78b..15ede69a4a 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -41,7 +41,7 @@ And then add the processor to your catalog builder: ## Self-hosted Bitbucket Server To use the discovery processor with a self-hosted Bitbucket Server, you'll need -a Bitbucket integration [set up](locations.md) with a `BITBUCKET_TOKEN` and a +a Bitbucket integration [set up](../bitbucketServer/locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. Then you can add a location target to the catalog configuration: @@ -74,7 +74,7 @@ The target is composed of four parts: ## Bitbucket Cloud To use the discovery processor with Bitbucket Cloud, you'll need a Bitbucket -integration [set up](locations.md) with a `username` and an `appPassword`. Then +integration [set up](../bitbucketCloud/locations.md) with a `username` and an `appPassword`. Then you can add a location target to the catalog configuration: ```yaml diff --git a/docs/integrations/bitbucketServer/discovery.md b/docs/integrations/bitbucketServer/discovery.md new file mode 100644 index 0000000000..57e52d1dae --- /dev/null +++ b/docs/integrations/bitbucketServer/discovery.md @@ -0,0 +1,114 @@ +--- +id: discovery +title: Bitbucket Server Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in Bitbucket Server +--- + +The Bitbucket Server integration has a special entity provider for discovering +catalog files located in Bitbucket Server. +The provider will search your Bitbucket Server account and register catalog files matching the configured path +as Location entity and via following processing steps add all contained catalog entities. +This can be useful as an alternative to static locations or manually adding things to the catalog. + +## Installation + +You will have to add the entity provider in the catalog initialization code of your +backend. The provider is not installed by default, therefore you have to add a +dependency to `@backstage/plugin-catalog-backend-module-bitbucket-server` to your backend +package. + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-bitbucket-server +``` + +And then add the entity provider to your catalog builder: + +```diff + // In packages/backend/src/plugins/catalog.ts ++ import { BitbucketServerEntityProvider } from '@backstage/plugin-catalog-backend-module-bitbucket-server'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ builder.addEntityProvider( ++ BitbucketServerEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ }), ++ ); + + // [...] + } +``` + +## Configuration + +To use the entity provider, you'll need a [Bitbucket Server integration set up](locations.md). + +Additionally, you need to configure your entity provider instance(s): + +```yaml +# app-config.yaml + +catalog: + providers: + bitbucketServer: + yourProviderId: # identifies your ingested dataset + host: 'bitbucket.mycompany.com' + catalogPath: /catalog-info.yaml # default value + filters: # optional + projectKey: '^apis-.*$' # optional; RegExp + repoSlug: '^service-.*$' # optional; RegExp +``` + +- **host**: + The host of the Bitbucket Server instance, **note**: the host needs to registered as an integration as well, see [location](locations.md). +- **`catalogPath`** _(optional)_: + Default: `/catalog-info.yaml`. + Path where to look for `catalog-info.yaml` files. + When started with `/`, it is an absolute path from the repo root. +- **filters** _(optional)_: + - **`projectKey`** _(optional)_: + Regular expression used to filter results based on the project key. + - **repoSlug** _(optional)_: + Regular expression used to filter results based on the repo slug. + +## Custom location processing + +The Bitbucket Server Entity Provider will by default emit a location for each +matching repository. However, it is possible to override this functionality and take full control of how each +matching repository is processed. + +`BitbucketServerEntityProvider.fromConfig` takes an optional parameter +`options.parser` where you can set your own parser to be used for each matched +repository. + +```typescript +const provider = BitbucketServerEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + parser: async function* customLocationParser(options: { + location: LocationSpec, + client: BitbucketServerClient, + }) { + // Custom logic for interpreting the matching repository + // See defaultBitbucketServerLocationParser for an example + } +); +``` + +## Alternative + +_Deprecated!_ Please raise issues for use cases not covered by the entity provider. + +[You can use the `BitbucketDiscoveryProcessor`.](../bitbucket/discovery.md#self-hosted-bitbucket-server) diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucketServer/locations.md similarity index 64% rename from docs/integrations/bitbucket/locations.md rename to docs/integrations/bitbucketServer/locations.md index 4fbb96bb94..7ff19b8b27 100644 --- a/docs/integrations/bitbucket/locations.md +++ b/docs/integrations/bitbucketServer/locations.md @@ -1,28 +1,25 @@ --- id: locations -title: Bitbucket Locations +title: Bitbucket Server Locations sidebar_label: Locations # prettier-ignore -description: Integrating source code stored in Bitbucket into the Backstage catalog +description: Integrating source code stored in Bitbucket Server into the Backstage catalog --- -The Bitbucket integration supports loading catalog entities from bitbucket.org (Bitbucket Cloud) -or Bitbucket Server. Entities can be added to +The Bitbucket Server integration supports loading catalog entities from Bitbucket Server. +Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the [catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) plugin. -## Bitbucket Cloud - -Please see [the Bitbucket Cloud documentation](../bitbucketCloud/locations.md). - -## Bitbucket Server +## Configuration ```yaml integrations: bitbucketServer: - - host: bitbucket.company.com + - host: bitbucket.mycompany.com + apiBaseUrl: https://bitbucket.mycompany.com/rest/api/1.0 token: ${BITBUCKET_SERVER_TOKEN} ``` @@ -32,6 +29,7 @@ or with Basic Auth integrations: bitbucketServer: - host: bitbucket.company.com + apiBaseUrl: https://bitbucket.mycompany.com/rest/api/1.0 username: ${BITBUCKET_SERVER_USERNAME} password: ${BITBUCKET_SERVER_PASSWORD} ``` @@ -40,13 +38,14 @@ Directly under the `bitbucketServer` key is a list of provider configurations, w you can list the Bitbucket Server providers you want to fetch data from. Each entry is a structure with the following elements: -- `host`: The host of the Bitbucket Server instance, e.g. `bitbucket.company.com`. +- `host`: The host of the Bitbucket Server instance, e.g. `bitbucket.mycompany.com`. - `token` (optional): - An [personal access token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html) + A [personal access token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html) as expected by Bitbucket Server. - `username` (optional): use for [Basic Auth](https://developer.atlassian.com/server/bitbucket/how-tos/command-line-rest/#authentication) for Bitbucket Server. - `password` (optional): use for [Basic Auth](https://developer.atlassian.com/server/bitbucket/how-tos/command-line-rest/#authentication) for Bitbucket Server. + Note: a token can also be used as a substitute for the password, see [HTTP access tokens](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html). - `apiBaseUrl` (optional): The URL of the Bitbucket Server API. For self-hosted installations, it is commonly at `https:///rest/api/1.0`. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 314710b800..2ae7f5e81d 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -150,10 +150,7 @@ { "type": "subcategory", "label": "Bitbucket", - "ids": [ - "integrations/bitbucket/locations", - "integrations/bitbucket/discovery" - ] + "ids": ["integrations/bitbucket/discovery"] }, { "type": "subcategory", @@ -163,6 +160,14 @@ "integrations/bitbucketCloud/discovery" ] }, + { + "type": "subcategory", + "label": "Bitbucket Server", + "ids": [ + "integrations/bitbucketServer/locations", + "integrations/bitbucketServer/discovery" + ] + }, { "type": "subcategory", "label": "Datadog", diff --git a/mkdocs.yml b/mkdocs.yml index 03c9dee8d0..bf577e53a1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -95,11 +95,13 @@ nav: - Discovery: 'integrations/azure/discovery.md' - Org Data: 'integrations/azure/org.md' - Bitbucket: - - Locations: 'integrations/bitbucket/locations.md' - Discovery: 'integrations/bitbucket/discovery.md' - Bitbucket Cloud: - Locations: 'integrations/bitbucketCloud/locations.md' - Discovery: 'integrations/bitbucketCloud/discovery.md' + - Bitbucket Server: + - Locations: 'integrations/bitbucketServer/locations.md' + - Discovery: 'integrations/bitbucketServer/discovery.md' - Datadog: - Installation: 'integrations/datadog-rum/installation.md' - Gerrit: From d148022062dc1d9702855625c3395fbf49ec9228 Mon Sep 17 00:00:00 2001 From: Otto Nordander Date: Thu, 4 Aug 2022 12:56:20 +0200 Subject: [PATCH 3/3] Add utility function getFile in BitbucketServerClient Can be used to fetch arbitrary files given a project and a repository which is helpful when using custom repository parsing. Signed-off-by: Otto Nordander --- .../api-report.md | 6 +++ .../src/lib/BitbucketServerClient.test.ts | 37 +++++++++++++++++++ .../src/lib/BitbucketServerClient.ts | 12 ++++++ 3 files changed, 55 insertions(+) diff --git a/plugins/catalog-backend-module-bitbucket-server/api-report.md b/plugins/catalog-backend-module-bitbucket-server/api-report.md index b087da6746..1f40f22242 100644 --- a/plugins/catalog-backend-module-bitbucket-server/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-server/api-report.md @@ -20,6 +20,12 @@ export class BitbucketServerClient { config: BitbucketServerIntegrationConfig; }): BitbucketServerClient; // (undocumented) + getFile(options: { + projectKey: string; + repo: string; + path: string; + }): Promise; + // (undocumented) getRepository(options: { projectKey: string; repo: string; diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts index 6b8043409f..66ccdc8c9a 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.test.ts @@ -26,6 +26,18 @@ import { BitbucketServerProject, BitbucketServerRepository } from './types'; const server = setupServer(); +const catalogInfoFile = ` +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: "backstage" + description: "A Backstage service" +spec: + type: service + lifecycle: experimental + owner: "backstage" +`; + describe('BitbucketServerClient', () => { const config: BitbucketServerIntegrationConfig = { host: 'bitbucket.mycompany.com', @@ -138,6 +150,31 @@ describe('BitbucketServerClient', () => { ); }); + it('getFile', async () => { + server.use( + rest.get( + `https://${config.host}/projects/test-project/repos/test-repo/raw/catalog-info.yaml`, + (req, res, ctx) => { + if ( + req.headers.get('authorization') !== + 'Basic dGVzdC11c2VyOnRlc3QtcHc=' + ) { + return res(ctx.status(400)); + } + + return res(ctx.text(catalogInfoFile)); + }, + ), + ); + + const response = await client.getFile({ + projectKey: 'test-project', + repo: 'test-repo', + path: 'catalog-info.yaml', + }); + expect(await response.text()).toEqual(catalogInfoFile); + }); + it('getRepository', async () => { server.use( rest.get( diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index fa233c6863..025cc555ce 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -60,6 +60,18 @@ export class BitbucketServerClient { ); } + async getFile(options: { + projectKey: string; + repo: string; + path: string; + }): Promise { + const base = new URL(this.config.apiBaseUrl); + return fetch( + `${base.protocol}//${base.host}/projects/${options.projectKey}/repos/${options.repo}/raw/${options.path}`, + getBitbucketServerRequestOptions(this.config), + ); + } + async getRepository(options: { projectKey: string; repo: string;