diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 093dc82adc..b31071be7f 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -1,24 +1,22 @@ --- id: discovery -title: Bitbucket Server Discovery +title: Bitbucket Discovery sidebar_label: Discovery description: - Automatically discovering catalog entities from repositories in a Bitbucket - Server instance + Automatically discovering catalog entities from repositories in Bitbucket --- The Bitbucket integration has a special discovery processor for discovering -catalog entities within a Bitbucket Server instance. The processor will crawl -the Bitbucket Server instance and register entities matching the configured -path. This can be useful as an alternative to static locations or manually -adding things to the catalog. +catalog entities located in Bitbucket. The processor will crawl your Bitbucket +account and register entities matching the configured path. This can be useful +as an alternative to static locations or manually adding things to the catalog. > Note: The Bitbucket Discovery Processor currently only supports a self-hosted > Bitbucket Server, and not the hosted Bitbucket Cloud product. To use the discovery processor, you'll need a Bitbucket integration -[set up](locations.md) with a `BITBUCKET_TOKEN`. Then you can add a location -target to the catalog configuration: +[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. +Then you can add a location target to the catalog configuration: ```yaml catalog: @@ -36,8 +34,8 @@ The target is composed of four parts: `*` to scan repositories from all projects. This example only scans for repositories in the `my-project` project. - The repository blob to scan, which accepts \* wildcard tokens. This can simply - be `*` to scan all repositories in the organization. This example only looks - for repositories prefixed with `service-`. + be `*` to scan all repositories in the project. This example only looks for + repositories prefixed with `service-`. - The path within each repository to find the catalog YAML file. This will usually be `/catalog-info.yaml` or a similar variation for catalog files stored in the root directory of each repository. diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucket/locations.md index bf7a6af57c..33490455d4 100644 --- a/docs/integrations/bitbucket/locations.md +++ b/docs/integrations/bitbucket/locations.md @@ -1,12 +1,12 @@ --- id: locations -title: BitBucket Locations +title: Bitbucket Locations sidebar_label: Locations description: - Integrating source code stored in BitBucket into the Backstage catalog + Integrating source code stored in Bitbucket into the Backstage catalog --- -The BitBucket integration supports loading catalog entities from bitbucket.com +The Bitbucket integration supports loading catalog entities from bitbucket.org or a self-hosted BitBucket. Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the @@ -21,21 +21,21 @@ integrations: token: ${BITBUCKET_TOKEN} ``` -> Note: A public BitBucket provider is added automatically at startup for +> Note: A public Bitbucket provider is added automatically at startup for > convenience, so you only need to list it if you want to supply a > [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html). Directly under the `bitbucket` key is a list of provider configurations, where -you can list the BitBucket providers you want to fetch data from. Each entry is +you can list the Bitbucket providers you want to fetch data from. Each entry is a structure with up to four elements: -- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`. -- `token` (optional): An personal access token as expected by BitBucket. Either +- `host`: The host of the Bitbucket instance, e.g. `bitbucket.company.com`. +- `token` (optional): An personal access token as expected by Bitbucket. Either an access token **or** a username + appPassword may be supplied. -- `username`: The BitBucket username to use in API requests. If neither a +- `username`: The Bitbucket username to use in API requests. If neither a username nor token are supplied, anonymous access will be used. -- `appPassword` (optional): The password for the BitBucket user. Only needed +- `appPassword` (optional): The password for the Bitbucket user. Only needed when using `username` instead of `token`. -- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted - installations, it is commonly at `https:///api/v4`. For gitlab.com, this - configuration is not needed as it can be inferred. +- `apiBaseUrl` (optional): The URL of the Bitbucket API. For self-hosted + installations, it is commonly at `https:///rest/api/1.0`. For + bitbucket.org, this configuration is not needed as it can be inferred. diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 4be408f475..5197549d85 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -120,12 +120,13 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(2); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', }); - expect(actual).toContainEqual({ + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', @@ -168,13 +169,13 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', }); - expect(actual).toContainEqual({ + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml', @@ -206,8 +207,8 @@ describe('BitbucketDiscoveryProcessor', () => { ); const actual = await readBitbucketOrg(client, target); - - expect(actual).toContainEqual({ + expect(actual.scanned).toBe(3); + expect(actual.matches).toContainEqual({ type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml', diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 6c13a40770..f3dae9c112 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -16,14 +16,17 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrationRegistry, + ScmIntegrations, +} from '@backstage/integration'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketClient, pageIterator } from './bitbucket'; +import { BitbucketClient, paginated } from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { results } from './index'; export class BitbucketDiscoveryProcessor implements CatalogProcessor { - private readonly integrations: ScmIntegrations; + private readonly integrations: ScmIntegrationRegistry; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { @@ -35,7 +38,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { }); } - constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + constructor(options: { + integrations: ScmIntegrationRegistry; + logger: Logger; + }) { this.integrations = options.integrations; this.logger = options.logger; } @@ -67,9 +73,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); - const repositories = await readBitbucketOrg(client, location.target); + const result = await readBitbucketOrg(client, location.target); - for (const repository of repositories) { + for (const repository of result.matches) { emit( results.location( repository, @@ -82,8 +88,8 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { } const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); - this.logger.info( - `Read ${repositories.length} Bitbucket repositories in ${duration} seconds`, + this.logger.debug( + `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`, ); return true; @@ -93,30 +99,29 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { export async function readBitbucketOrg( client: BitbucketClient, target: string, -): Promise { +): Promise { const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target); - const projectIterator = pageIterator(options => client.listProjects(options)); - let result: LocationSpec[] = []; + const projects = paginated(options => client.listProjects(options)); + const result: Result = { + scanned: 0, + matches: [], + }; - for await (const page of projectIterator) { - for (const project of page.values) { - if (!projectSearchPath.test(project.key)) { - continue; - } - const repoIterator = pageIterator(options => - client.listRepositories(project.key, options), - ); - for await (const repoPage of repoIterator) { - result = result.concat( - repoPage.values - .filter(v => repoSearchPath.test(v.slug)) - .map(repo => { - return { - type: 'url', - target: `${repo.links.self[0].href}${catalogPath}`, - }; - }), - ); + for await (const project of projects) { + if (!projectSearchPath.test(project.key)) { + continue; + } + const repositories = paginated(options => + client.listRepositories(project.key, options), + ); + for await (const repository of repositories) { + result.scanned++; + + if (repoSearchPath.test(repository.slug)) { + result.matches.push({ + type: 'url', + target: `${repository.links.self[0].href}${catalogPath}`, + }); } } } @@ -144,3 +149,8 @@ function parseUrl( function escapeRegExp(str: string): RegExp { return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } + +type Result = { + scanned: number; + matches: LocationSpec[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index b03dcd02bf..c3c27aedfc 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -46,14 +46,12 @@ export class BitbucketClient { options?: ListOptions, ): Promise> { const request = new URL(endpoint); - if (options) { - (Object.keys(options) as Array).forEach(key => { - const value: any = options[key] as any; - if (value) { - request.searchParams.append(key, value); - } - }); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } } + const response = await fetch( request.toString(), getBitbucketRequestOptions(this.config), @@ -72,6 +70,7 @@ export class BitbucketClient { } export type ListOptions = { + [key: string]: number | undefined; limit?: number | undefined; start?: number | undefined; }; @@ -85,42 +84,17 @@ export type PagedResponse = { nextPageStart: number; }; -export function pageIterator( - pagedRequest: (options: ListOptions) => Promise>, +export async function* paginated( + request: (options: ListOptions) => Promise>, options?: ListOptions, -): AsyncIterable> { - return { - [Symbol.asyncIterator]: () => { - const opts = options || { start: 0 }; - let finished = false; - return { - async next() { - if (!finished) { - try { - const response = await pagedRequest(opts); - finished = response.isLastPage; - opts.start = response.nextPageStart; - return Promise.resolve({ - value: response, - done: false, - }); - } catch (error) { - return Promise.reject({ - value: undefined, - done: true, - error: error, - }); - } - } else { - opts.start = 0; - finished = false; - return Promise.resolve({ - value: undefined, - done: true, - }); - } - }, - }; - }, - }; +) { + const opts = options || { start: 0 }; + let res; + do { + res = await request(opts); + opts.start = res.nextPageStart; + for (const item of res.values) { + yield item; + } + } while (!res.isLastPage); } diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index a8d0a20627..7e70bcfe7a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -13,5 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { BitbucketClient, pageIterator } from './client'; +export { BitbucketClient, paginated } from './client'; export type { PagedResponse } from './client';