From e2ed9126e74f8e27161eb5026e9ff7995f2aa586 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Tue, 27 Jul 2021 11:15:49 +1000 Subject: [PATCH 001/116] Bitbucket Cloud Discovery When bitbucket.org is discovered as the target for the bitbucket-discovery catalog processor, a scan is performed in a workspace and finds repositories matching the any project and repository regexes in line with the existing bitbucket server implementation. Resolves #6216. Signed-off-by: Bart Breen --- docs/integrations/bitbucket/discovery.md | 45 ++++- .../BitbucketDiscoveryProcessor.test.ts | 180 +++++++++++++++++- .../processors/BitbucketDiscoveryProcessor.ts | 161 ++++++++++++++-- .../ingestion/processors/bitbucket/client.ts | 66 +++++++ .../ingestion/processors/bitbucket/index.ts | 6 +- .../ingestion/processors/bitbucket/types.ts | 36 +++- 6 files changed, 462 insertions(+), 32 deletions(-) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 1538806e39..1a673fb78d 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -11,12 +11,12 @@ catalog entities located in Bitbucket. The processor will crawl your Bitbucket account and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. -> Note: The Bitbucket Discovery Processor currently only supports a self-hosted -> Bitbucket Server, and not the hosted Bitbucket Cloud product. +## Self-hosted Bitbucket Server -To use the discovery processor, you'll need a Bitbucket integration -[set up](locations.md) with a `BITBUCKET_TOKEN` and a `BITBUCKET_API_BASE_URL`. -Then you can add a location target to the catalog configuration: +To use the discovery processor with a self-hosted Bitbucket Server, you'll need +a Bitbucket integration [set up](locations.md) with a `BITBUCKET_TOKEN` and a +`BITBUCKET_API_BASE_URL`. Then you can add a location target to the catalog +configuration: ```yaml catalog: @@ -44,6 +44,41 @@ The target is composed of four parts: will result in: `https://bitbucket.mycompany.com/projects/my-project/repos/service-a/catalog-info.yaml`. +## Bitbucket Cloud + +To use the discovery processor with Bitbucket Cloud, you'll need a Bitbucket +integration [set up](locations.md) with a `username` and an `appPassword`. Then +you can add a location target to the catalog configuration: + +```yaml +catalog: + locations: + - type: bitbucket-discovery + target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml +``` + +Note the `bitbucket-discovery` type, as this is not a regular `url` processor. + +The target is composed of four parts: + +- The base instance URL, `https://bitbucket.org` in this case +- The workspace name to scan, which must match a workspace accessible with the + username of your integration. +- The project key to scan, which accepts \* wildcard tokens. This can simply be + `*` to include all repositories. This example only returns repositories in the + `my-project` project. +- The repository blob to scan, which accepts \* wildcard tokens. This can simply + be `*` to scan all repositories in the workspace. This example only looks for + repositories prefixed with `service-`. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml` or a similar variation for catalog files + stored in the root directory of each repository. If omitted, the default value + `catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a` + exists, + `https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/` + will result in: + `https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`. + ## Custom repository processing The Bitbucket Discovery Processor will by default emit a location for each diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index e2b295e7ff..43efa8e5d6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -17,7 +17,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; -import { BitbucketRepositoryParser, PagedResponse } from './bitbucket'; +import { + BitbucketRepository, + BitbucketRepository20, + BitbucketRepositoryParser, + PagedResponse, + PagedResponse20, +} from './bitbucket'; import { results } from './index'; import { RequestHandler, rest } from 'msw'; import { setupServer } from 'msw/node'; @@ -79,6 +85,40 @@ function setupStubs(projects: any[]) { } } +function setupBitbucketCloudStubs( + workspace: string, + repositories: Pick[], +) { + function pagedResponse(values: any): PagedResponse20 { + return { + values: values, + page: 1, + } as PagedResponse20; + } + + server.use( + rest.get( + `https://api.bitbucket.org/2.0/repositories/${workspace}`, + (_, res, ctx) => { + return res( + ctx.json( + pagedResponse( + repositories.map(r => ({ + ...r, + links: { + source: { + href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`, + }, + }, + })), + ), + ), + ); + }, + ), + ); +} + describe('BitbucketDiscoveryProcessor', () => { beforeAll(() => server.listen()); afterEach(() => server.resetHandlers()); @@ -129,7 +169,7 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); - describe('handles repositories', () => { + describe('handles organisation repositories', () => { const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { @@ -255,6 +295,142 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); + describe('handles cloud repositories', () => { + const processor = BitbucketDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: 'myuser', + appPassword: 'blob', + }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + + it('output all repositories', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml', + }, + optional: true, + }); + }); + + it('output repositories with wildcards', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + }, + optional: true, + }); + }); + it('filter unrelated repositories', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-one' }, slug: 'repository-two' }, + { project: { key: 'prj-one' }, slug: 'repository-three' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml', + }, + optional: true, + }); + }); + + it.each` + target + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'} + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'} + ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'} + `("target '$target' adds default path to catalog", async ({ target }) => { + setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]); + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: target, + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + }); + }); + describe('Custom repository parser', () => { const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) { yield results.location( diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 9d53b7e5b5..dd5ff0d4af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -17,6 +17,7 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { + BitbucketIntegration, ScmIntegrationRegistry, ScmIntegrations, } from '@backstage/integration'; @@ -26,10 +27,16 @@ import { BitbucketClient, defaultRepositoryParser, paginated, + paginated20, BitbucketRepository, + BitbucketRepository20, } from './bitbucket'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +const DEFAULT_BRANCH = 'master'; +const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; +const EMPTY_CATALOG_LOCATION = '/'; + export class BitbucketDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; private readonly parser: BitbucketRepositoryParser; @@ -71,10 +78,6 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { throw new Error( `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`, ); - } else if (integration.config.host === 'bitbucket.org') { - throw new Error( - `Component discovery for Bitbucket Cloud is not yet supported`, - ); } const client = new BitbucketClient({ @@ -83,38 +86,89 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading Bitbucket repositories from ${location.target}`); - const { catalogPath } = parseUrl(location.target); - const expandedCatalogPath = - catalogPath === '/' ? '/catalog-info.yaml' : catalogPath; + const isBitbucketCloud = integration.config.host === 'bitbucket.org'; - const result = await readBitbucketOrg(client, location.target); + const processOptions: ProcessOptions = { + client, + emit, + integration, + location, + }; + const { scanned, matches } = isBitbucketCloud + ? await this.processCloudRepositories(processOptions) + : await this.processOrganisationRepositories(processOptions); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${scanned} Bitbucket repositories (${matches} matching the pattern) in ${duration} seconds`, + ); + + return true; + } + + private async processCloudRepositories( + options: ProcessOptions, + ): Promise { + const { client, location, integration, emit } = options; + + const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl( + location.target, + ); + const catalogPath = + requestedCatalogPath === EMPTY_CATALOG_LOCATION + ? DEFAULT_CATALOG_LOCATION + : requestedCatalogPath; + const result = await readBitbucketCloud(client, location.target); for (const repository of result.matches) { + const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH; for await (const entity of this.parser({ - integration: integration, - target: `${repository.links.self[0].href}${expandedCatalogPath}`, + integration, + target: `${repository.links.source.href}/${mainbranch}${catalogPath}`, logger: this.logger, })) { emit(entity); } } + return { + matches: result.matches.length, + scanned: result.scanned, + }; + } - const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); - this.logger.debug( - `Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`, - ); - - return true; + private async processOrganisationRepositories( + options: ProcessOptions, + ): Promise { + const { client, location, integration, emit } = options; + const { catalogPath: requestedCatalogPath } = parseUrl(location.target); + const catalogPath = + requestedCatalogPath === EMPTY_CATALOG_LOCATION + ? DEFAULT_CATALOG_LOCATION + : requestedCatalogPath; + const result = await readBitbucketOrg(client, location.target); + for (const repository of result.matches) { + for await (const entity of this.parser({ + integration, + target: `${repository.links.self[0].href}${catalogPath}`, + logger: this.logger, + })) { + emit(entity); + } + } + return { + matches: result.matches.length, + scanned: result.scanned, + }; } } export async function readBitbucketOrg( client: BitbucketClient, target: string, -): Promise { +): Promise> { const { projectSearchPath, repoSearchPath } = parseUrl(target); const projects = paginated(options => client.listProjects(options)); - const result: Result = { + const result: Result = { scanned: 0, matches: [], }; @@ -136,6 +190,35 @@ export async function readBitbucketOrg( return result; } +export async function readBitbucketCloud( + client: BitbucketClient, + target: string, +): Promise> { + const { + workspacePath, + projectSearchPath, + repoSearchPath, + } = parseBitbucketCloudUrl(target); + const repositories = paginated20(options => + client.listRepositoriesByWorkspace20(workspacePath, options), + ); + const result: Result = { + scanned: 0, + matches: [], + }; + + for await (const repository of repositories) { + result.scanned++; + if ( + projectSearchPath.test(repository.project.key) && + repoSearchPath.test(repository.slug) + ) { + result.matches.push(repository); + } + } + return result; +} + function parseUrl( urlString: string, ): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { @@ -154,11 +237,47 @@ function parseUrl( throw new Error(`Failed to parse ${urlString}`); } +function parseBitbucketCloudUrl( + urlString: string, +): { + workspacePath: string; + projectSearchPath: RegExp; + repoSearchPath: RegExp; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml} + if (path.length > 5 && path[1].length && path[3].length) { + return { + workspacePath: decodeURIComponent(path[1]), + projectSearchPath: escapeRegExp(decodeURIComponent(path[3])), + repoSearchPath: escapeRegExp(decodeURIComponent(path[5])), + catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + function escapeRegExp(str: string): RegExp { return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } -type Result = { - scanned: number; - matches: BitbucketRepository[]; +type ProcessOptions = { + client: BitbucketClient; + integration: BitbucketIntegration; + location: LocationSpec; + emit: CatalogProcessorEmit; +}; + +type Result = { + scanned: number; + matches: T[]; +}; + +type ResultSummary = { + scanned: number; + matches: number; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 5a1419dc27..bdea349180 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -19,6 +19,7 @@ import { BitbucketIntegrationConfig, getBitbucketRequestOptions, } from '@backstage/integration'; +import { BitbucketRepository, BitbucketRepository20 } from './types'; export class BitbucketClient { private readonly config: BitbucketIntegrationConfig; @@ -31,6 +32,16 @@ export class BitbucketClient { return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } + async listRepositoriesByWorkspace20( + workspace: string, + options?: ListOptions, + ): Promise> { + return this.pagedRequest20( + `${this.config.apiBaseUrl}/repositories/${workspace}`, + options, + ); + } + async listRepositories( projectKey: string, options?: ListOptions, @@ -67,6 +78,33 @@ export class BitbucketClient { return repositories as PagedResponse; }); } + + private async pagedRequest20( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + const response = await fetch( + request.toString(), + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(repositories => { + return repositories as PagedResponse20; + }); + } } export type ListOptions = { @@ -84,6 +122,19 @@ export type PagedResponse = { nextPageStart: number; }; +export type ListOptions20 = { + [key: string]: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse20 = { + page: number; + pagelen: number; + size: number; + values: T[]; + next: string; +}; + export async function* paginated( request: (options: ListOptions) => Promise>, options?: ListOptions, @@ -98,3 +149,18 @@ export async function* paginated( } } while (!res.isLastPage); } + +export async function* paginated20( + request: (options: ListOptions20) => Promise>, + options?: ListOptions20, +) { + const opts = options || { page: 1 }; + let res; + do { + res = await request(opts); + opts.page = (opts.page || 1) + 1; + for (const item of res.values) { + yield item; + } + } while (res.next); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts index 7adab7746f..4b28c7f115 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { BitbucketClient, paginated } from './client'; +export { BitbucketClient, paginated, paginated20 } from './client'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; -export type { PagedResponse } from './client'; -export type { BitbucketRepository } from './types'; +export type { PagedResponse, PagedResponse20 } from './client'; +export type { BitbucketRepository, BitbucketRepository20 } from './types'; export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts index b273d26874..db8cf69688 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export type BitbucketRepository = { + +type BitbucketRepositoryBase = { project: { key: string; }; slug: string; +}; + +export type BitbucketRepository = BitbucketRepositoryBase & { links: Record< string, { @@ -25,3 +29,33 @@ export type BitbucketRepository = { }[] >; }; + +export type BitbucketRepository20 = BitbucketRepositoryBase & { + links: Record< + | 'self' + | 'source' + | 'html' + | 'avatar' + | 'pullrequests' + | 'commits' + | 'forks' + | 'watchers' + | 'downloads' + | 'hooks', + { + href: string; + name?: string; + } + > & + Record< + 'clone', + { + href: string; + name?: string; + }[] + >; + mainbranch?: { + type: string; + name: string; + }; +}; From 501ce92f9c0ff7db911ac6afc30fcfbd71493eec Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Tue, 27 Jul 2021 11:18:07 +1000 Subject: [PATCH 002/116] Added changeset Signed-off-by: Bart Breen --- .changeset/pretty-sloths-rush.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/pretty-sloths-rush.md diff --git a/.changeset/pretty-sloths-rush.md b/.changeset/pretty-sloths-rush.md new file mode 100644 index 0000000000..85b5b4341c --- /dev/null +++ b/.changeset/pretty-sloths-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Bitbucket Cloud Discovery support From 407aefba221b4974d5ab37cd68b844b273230587 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Tue, 27 Jul 2021 15:17:30 +1000 Subject: [PATCH 003/116] Fixed paging and authenticated requests Signed-off-by: Bart Breen --- .../processors/BitbucketDiscoveryProcessor.test.ts | 14 +++++++------- .../processors/BitbucketDiscoveryProcessor.ts | 2 +- .../src/ingestion/processors/bitbucket/client.ts | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 43efa8e5d6..559174054f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -106,8 +106,8 @@ function setupBitbucketCloudStubs( repositories.map(r => ({ ...r, links: { - source: { - href: `https://api.bitbucket.org/2.0/repositories/${workspace}/${r.slug}/src`, + html: { + href: `https://bitbucket.org/${workspace}/${r.slug}`, }, }, })), @@ -332,7 +332,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', }, optional: true, }); @@ -341,7 +341,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-two/src/master/catalog.yaml', + 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml', }, optional: true, }); @@ -367,7 +367,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog.yaml', + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', }, optional: true, }); @@ -393,7 +393,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-three/src/master/catalog.yaml', + 'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml', }, optional: true, }); @@ -424,7 +424,7 @@ describe('BitbucketDiscoveryProcessor', () => { location: { type: 'url', target: - 'https://api.bitbucket.org/2.0/repositories/myworkspace/repository-one/src/master/catalog-info.yaml', + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index dd5ff0d4af..3521c98af5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -124,7 +124,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH; for await (const entity of this.parser({ integration, - target: `${repository.links.source.href}/${mainbranch}${catalogPath}`, + target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`, logger: this.logger, })) { emit(entity); diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index bdea349180..485e248a76 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -125,6 +125,7 @@ export type PagedResponse = { export type ListOptions20 = { [key: string]: number | undefined; page?: number | undefined; + pagelen?: number | undefined; }; export type PagedResponse20 = { @@ -154,7 +155,7 @@ export async function* paginated20( request: (options: ListOptions20) => Promise>, options?: ListOptions20, ) { - const opts = options || { page: 1 }; + const opts = options || { page: 1, pagelen: 100 }; let res; do { res = await request(opts); From d02ecd3eb39f00f539fee71e8a7a1a1a2460be36 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Wed, 28 Jul 2021 12:10:51 +1000 Subject: [PATCH 004/116] Fixed unused imports Signed-off-by: Bart Breen --- .../ingestion/processors/BitbucketDiscoveryProcessor.test.ts | 1 - .../src/ingestion/processors/bitbucket/client.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 559174054f..2c52bf81fd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -18,7 +18,6 @@ import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; import { - BitbucketRepository, BitbucketRepository20, BitbucketRepositoryParser, PagedResponse, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 485e248a76..5cacd6d6e6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -19,7 +19,7 @@ import { BitbucketIntegrationConfig, getBitbucketRequestOptions, } from '@backstage/integration'; -import { BitbucketRepository, BitbucketRepository20 } from './types'; +import { BitbucketRepository20 } from './types'; export class BitbucketClient { private readonly config: BitbucketIntegrationConfig; From b6801e6d70eff68c44a743e30940c14726b39f38 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Wed, 28 Jul 2021 23:27:11 +1000 Subject: [PATCH 005/116] Changed format for target location so that query can be passed directly to bitbucket api Signed-off-by: Bart Breen --- docs/integrations/bitbucket/discovery.md | 52 ++++--- .../BitbucketDiscoveryProcessor.test.ts | 131 +++++++++++++++++- .../processors/BitbucketDiscoveryProcessor.ts | 52 ++++--- .../ingestion/processors/bitbucket/client.ts | 10 +- 4 files changed, 201 insertions(+), 44 deletions(-) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 1a673fb78d..b851a88a3b 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -54,30 +54,44 @@ you can add a location target to the catalog configuration: catalog: locations: - type: bitbucket-discovery - target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml + target: https://bitbucket.org/workspaces/my-workspace ``` Note the `bitbucket-discovery` type, as this is not a regular `url` processor. -The target is composed of four parts: +The target is composed of the following parts: -- The base instance URL, `https://bitbucket.org` in this case -- The workspace name to scan, which must match a workspace accessible with the - username of your integration. -- The project key to scan, which accepts \* wildcard tokens. This can simply be - `*` to include all repositories. This example only returns repositories in the - `my-project` project. -- The repository blob to scan, which accepts \* wildcard tokens. This can simply - be `*` to scan all repositories in the workspace. This example only looks for - repositories prefixed with `service-`. -- The path within each repository to find the catalog YAML file. This will - usually be `/catalog-info.yaml` or a similar variation for catalog files - stored in the root directory of each repository. If omitted, the default value - `catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a` - exists, - `https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/` - will result in: - `https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`. +- The base URL for Bitbucket, `https://bitbucket.org` +- The workspace name to scan (following the `workspaces/` path part), which must + match a workspace accessible with the username of your integration. +- (Optional) The project key to scan (following the `projects/` path part), + which accepts \* wildcard tokens. If ommitted, repositories from all projects + in the workspace are included. +- (Optional) The repository blob to scan (following the `repos/` path part), + which accepts \* wildcard tokens. If ommitted, all repositories in the + workspace are included. +- (Optional) The `catalogPath` query argument to specify the location within + each repository to find the catalog YAML file. This will usually be + `/catalog-info.yaml` or a similar variation for catalog files stored in the + root directory of each repository. If omitted, the default value + `catalog-info.yaml` will be used. +- (Optional) The `q` query argument to be passed through to Bitbucket for + filtering results via the API. This is the most flexible option and will + reduce the amount of API calls if you have a large workspace. + [See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering) + for the query argument (will be passed as the `q` query parameter). + +Examples: + +- `https://bitbucket.org/workspaces/my-workspace/projects/my-project` will find + all repositories in the `my-project` project in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace/repos/service-*` will find all + repositories starting with `service-` in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*` + will find all repositories starting with `service-`, in all projects starting + with `apis-` in the `my-workspace` workspace. +- `https://bitbucket.org/workspaces/my-workspace?q=project.key ~ "my-project"` + will find all repositories in a project containing `my-project` in its key. ## Custom repository processing diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 2c52bf81fd..d522eb3f53 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -88,6 +88,7 @@ function setupBitbucketCloudStubs( workspace: string, repositories: Pick[], ) { + const stubCallerFn = jest.fn(); function pagedResponse(values: any): PagedResponse20 { return { values: values, @@ -99,6 +100,7 @@ function setupBitbucketCloudStubs( rest.get( `https://api.bitbucket.org/2.0/repositories/${workspace}`, (_, res, ctx) => { + stubCallerFn(_); return res( ctx.json( pagedResponse( @@ -116,6 +118,7 @@ function setupBitbucketCloudStubs( }, ), ); + return stubCallerFn; } describe('BitbucketDiscoveryProcessor', () => { @@ -310,6 +313,77 @@ describe('BitbucketDiscoveryProcessor', () => { { logger: getVoidLogger() }, ); + it('output all repositories by default', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: 'https://bitbucket.org/workspaces/myworkspace', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('uses provided catalog path', async () => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + { project: { key: 'prj-two' }, slug: 'repository-two' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace?catalogPath=my/nested/path/catalog.yaml', + }; + + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml', + }, + optional: true, + }); + }); + it('output all repositories', async () => { setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, @@ -318,7 +392,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -354,7 +428,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -371,6 +445,7 @@ describe('BitbucketDiscoveryProcessor', () => { optional: true, }); }); + it('filter unrelated repositories', async () => { setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, @@ -380,7 +455,7 @@ describe('BitbucketDiscoveryProcessor', () => { const location: LocationSpec = { type: 'bitbucket-discovery', target: - 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml', + 'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?catalogPath=catalog.yaml', }; const emitter = jest.fn(); @@ -398,13 +473,42 @@ describe('BitbucketDiscoveryProcessor', () => { }); }); + it('submits query', async () => { + const mockCall = setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: + 'https://bitbucket.org/workspaces/myworkspace?q=project.key ~ "prj-one"', + }; + + const emitter = jest.fn(); + await processor.readLocation(location, false, emitter); + + expect(emitter).toBeCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + }, + optional: true, + }); + expect(mockCall).toBeCalledTimes(1); + // it should be possible to do this via an `expect.objectContaining` check but seems to fail with some encoding issue. + expect(mockCall.mock.calls[0][0].url).toMatchInlineSnapshot( + `"https://api.bitbucket.org/2.0/repositories/myworkspace?page=1&pagelen=100&q=project.key+%7E+%22prj-one%22"`, + ); + }); + it.each` target ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'} ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'} ${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'} `("target '$target' adds default path to catalog", async ({ target }) => { - setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]); setupBitbucketCloudStubs('myworkspace', [ { project: { key: 'prj-one' }, slug: 'repository-one' }, ]); @@ -428,6 +532,25 @@ describe('BitbucketDiscoveryProcessor', () => { optional: true, }); }); + + it.each` + target + ${'https://bitbucket.org/test'} + `("target '$target' is rejected", async ({ target }) => { + setupBitbucketCloudStubs('myworkspace', [ + { project: { key: 'prj-one' }, slug: 'repository-one' }, + ]); + + const location: LocationSpec = { + type: 'bitbucket-discovery', + target: target, + }; + + const emitter = jest.fn(); + await expect( + processor.readLocation(location, false, emitter), + ).rejects.toThrow(/Failed to parse /); + }); }); describe('Custom repository parser', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 3521c98af5..4784aee2ea 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -196,11 +196,16 @@ export async function readBitbucketCloud( ): Promise> { const { workspacePath, + queryParam: q, projectSearchPath, repoSearchPath, } = parseBitbucketCloudUrl(target); - const repositories = paginated20(options => - client.listRepositoriesByWorkspace20(workspacePath, options), + + const repositories = paginated20( + options => client.listRepositoriesByWorkspace20(workspacePath, options), + { + q, + }, ); const result: Result = { scanned: 0, @@ -210,8 +215,8 @@ export async function readBitbucketCloud( for await (const repository of repositories) { result.scanned++; if ( - projectSearchPath.test(repository.project.key) && - repoSearchPath.test(repository.slug) + (!projectSearchPath || projectSearchPath.test(repository.project.key)) && + (!repoSearchPath || repoSearchPath.test(repository.slug)) ) { result.matches.push(repository); } @@ -237,28 +242,43 @@ function parseUrl( throw new Error(`Failed to parse ${urlString}`); } +function readPathParameters(pathParts: string[]): Map { + const vals: Record = {}; + for (let i = 0; i < pathParts.length; i += 2) { + if (i + 1 >= pathParts.length) continue; + vals[pathParts[i]] = decodeURIComponent(pathParts[i + 1]); + } + return new Map(Object.entries(vals)); +} + function parseBitbucketCloudUrl( urlString: string, ): { workspacePath: string; - projectSearchPath: RegExp; - repoSearchPath: RegExp; catalogPath: string; + projectSearchPath?: RegExp; + repoSearchPath?: RegExp; + queryParam?: string; } { const url = new URL(urlString); - const path = url.pathname.substr(1).split('/'); + const pathMap = readPathParameters(url.pathname.substr(1).split('/')); + const query = url.searchParams; - // workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml} - if (path.length > 5 && path[1].length && path[3].length) { - return { - workspacePath: decodeURIComponent(path[1]), - projectSearchPath: escapeRegExp(decodeURIComponent(path[3])), - repoSearchPath: escapeRegExp(decodeURIComponent(path[5])), - catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`, - }; + if (!pathMap.has('workspaces')) { + throw new Error(`Failed to parse workspace from ${urlString}`); } - throw new Error(`Failed to parse ${urlString}`); + return { + workspacePath: pathMap.get('workspaces')!, + projectSearchPath: pathMap.has('projects') + ? escapeRegExp(pathMap.get('projects')!) + : undefined, + repoSearchPath: pathMap.has('repos') + ? escapeRegExp(pathMap.get('repos')!) + : undefined, + catalogPath: `/${query.get('catalogPath') || ''}`, + queryParam: query.get('q') || undefined, + }; } function escapeRegExp(str: string): RegExp { diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts index 5cacd6d6e6..60a2e62bb7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts @@ -34,7 +34,7 @@ export class BitbucketClient { async listRepositoriesByWorkspace20( workspace: string, - options?: ListOptions, + options?: ListOptions20, ): Promise> { return this.pagedRequest20( `${this.config.apiBaseUrl}/repositories/${workspace}`, @@ -81,7 +81,7 @@ export class BitbucketClient { private async pagedRequest20( endpoint: string, - options?: ListOptions, + options?: ListOptions20, ): Promise> { const request = new URL(endpoint); for (const key in options) { @@ -123,7 +123,7 @@ export type PagedResponse = { }; export type ListOptions20 = { - [key: string]: number | undefined; + [key: string]: string | number | undefined; page?: number | undefined; pagelen?: number | undefined; }; @@ -155,11 +155,11 @@ export async function* paginated20( request: (options: ListOptions20) => Promise>, options?: ListOptions20, ) { - const opts = options || { page: 1, pagelen: 100 }; + const opts = { page: 1, pagelen: 100, ...options }; let res; do { res = await request(opts); - opts.page = (opts.page || 1) + 1; + opts.page = opts.page + 1; for (const item of res.values) { yield item; } From 8e22cbede896931c9e587b31f471a40d81258faa Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Wed, 28 Jul 2021 23:35:37 +1000 Subject: [PATCH 006/116] Fixed typos Signed-off-by: Bart Breen --- docs/integrations/bitbucket/discovery.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index b851a88a3b..2b04490cbe 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -65,10 +65,10 @@ The target is composed of the following parts: - The workspace name to scan (following the `workspaces/` path part), which must match a workspace accessible with the username of your integration. - (Optional) The project key to scan (following the `projects/` path part), - which accepts \* wildcard tokens. If ommitted, repositories from all projects + which accepts \* wildcard tokens. If omitted, repositories from all projects in the workspace are included. - (Optional) The repository blob to scan (following the `repos/` path part), - which accepts \* wildcard tokens. If ommitted, all repositories in the + which accepts \* wildcard tokens. If omitted, all repositories in the workspace are included. - (Optional) The `catalogPath` query argument to specify the location within each repository to find the catalog YAML file. This will usually be @@ -92,6 +92,9 @@ Examples: with `apis-` in the `my-workspace` workspace. - `https://bitbucket.org/workspaces/my-workspace?q=project.key ~ "my-project"` will find all repositories in a project containing `my-project` in its key. +- `https://bitbucket.org/workspaces/my-workspace?catalogPath=my/nested/path/catalog.yaml` + will find all repositories in the `my-workspace` workspace and use the catalog + file at `my/nested/path/catalog.yaml`. ## Custom repository processing From 4dea23c3781445e0efa359fcda01f4ec04c80e49 Mon Sep 17 00:00:00 2001 From: Bart Breen Date: Thu, 2 Sep 2021 21:51:05 +1000 Subject: [PATCH 007/116] Fixed prettier error Signed-off-by: Bart Breen --- .../processors/BitbucketDiscoveryProcessor.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts index 4784aee2ea..fc3898f066 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts @@ -224,9 +224,11 @@ export async function readBitbucketCloud( return result; } -function parseUrl( - urlString: string, -): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } { +function parseUrl(urlString: string): { + projectSearchPath: RegExp; + repoSearchPath: RegExp; + catalogPath: string; +} { const url = new URL(urlString); const path = url.pathname.substr(1).split('/'); @@ -251,9 +253,7 @@ function readPathParameters(pathParts: string[]): Map { return new Map(Object.entries(vals)); } -function parseBitbucketCloudUrl( - urlString: string, -): { +function parseBitbucketCloudUrl(urlString: string): { workspacePath: string; catalogPath: string; projectSearchPath?: RegExp; From 046410ee42c53bf06b04baa30c085aa0766ad0c5 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Mon, 13 Sep 2021 14:58:59 -0400 Subject: [PATCH 008/116] Include example of catalog.ts update for discovery interval change. Signed-off-by: Kyle Smith --- docs/integrations/github/discovery.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index a578d0893d..db4952f904 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -38,17 +38,28 @@ The target is composed of three parts: ## GitHub API Rate Limits -GitHub -[rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) -API requests to 5,000 per hour (or more for Enterprise accounts). The default -Backstage catalog backend refreshes data every 100 seconds, which issues an API -request for each discovered location. +GitHub [rate limits] API requests to 5,000 per hour (or more for Enterprise +accounts). The default Backstage catalog backend refreshes data every 100 +seconds, which issues an API request for each discovered location. This means if you have more than ~140 catalog entities, you may get throttled by rate limiting. This will soon be resolved once catalog refreshes make use of ETags; to work around this in the meantime, you can change the refresh rate of -the catalog in your `packages/backend/src/plugins/catalog.ts` file, or configure -Backstage to use the [github-apps plugin](../../plugins/github-apps.md). +the catalog in your `packages/backend/src/plugins/catalog.ts` file: + +```typescript +const builder = await CatalogBuilder.create(env); + +// For example, to refresh every 5 minutes (300 seconds). +builder.setRefreshIntervalSeconds(300); +``` + +Alternatively, or additionally, you can use the [github-apps plugin] which +carries a much higher rate limit at GitHub. This is true for any method of adding GitHub entities to the catalog, but especially easy to hit with automatic discovery. + +[rate limits]: + https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting +[github-apps plugin]: ../../plugins/github-apps.md From 038b9763d17c9b1bcd326f0b50fcad5f54de5142 Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 17 Sep 2021 10:08:06 +0200 Subject: [PATCH 009/116] add search to featureFlag Signed-off-by: Samira Mokaram --- .changeset/shaggy-beds-relax.md | 5 ++++ .../FeatureFlags/UserSettingsFeatureFlags.tsx | 26 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 .changeset/shaggy-beds-relax.md diff --git a/.changeset/shaggy-beds-relax.md b/.changeset/shaggy-beds-relax.md new file mode 100644 index 0000000000..caa6b90924 --- /dev/null +++ b/.changeset/shaggy-beds-relax.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-user-settings': patch +--- + +Add search to FeatureFlags diff --git a/plugins/user-settings/src/components/FeatureFlags/UserSettingsFeatureFlags.tsx b/plugins/user-settings/src/components/FeatureFlags/UserSettingsFeatureFlags.tsx index 5a3f077920..fd1520d926 100644 --- a/plugins/user-settings/src/components/FeatureFlags/UserSettingsFeatureFlags.tsx +++ b/plugins/user-settings/src/components/FeatureFlags/UserSettingsFeatureFlags.tsx @@ -15,7 +15,7 @@ */ import React, { useCallback, useState } from 'react'; -import { List } from '@material-ui/core'; +import { List, TextField, ListItem } from '@material-ui/core'; import { EmptyFlags } from './EmptyFlags'; import { FlagItem } from './FeatureFlagsItem'; @@ -35,6 +35,7 @@ export const UserSettingsFeatureFlags = () => { ); const [state, setState] = useState>(initialFlagState); + const [searchInput, setSearchInput] = useState(''); const toggleFlag = useCallback( (flagName: string) => { @@ -62,16 +63,25 @@ export const UserSettingsFeatureFlags = () => { return ( - {featureFlags.map(featureFlag => { + + setSearchInput(e.target.value)} + value={searchInput} + /> + + {featureFlags.map((featureFlag, index) => { const enabled = Boolean(state[featureFlag.name]); return ( - + featureFlag.name.includes(searchInput) && ( + + ) ); })} From 8acca63fa9cab7d67cd319cf5c0e99b36a82ba73 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Tue, 21 Sep 2021 14:01:53 +0700 Subject: [PATCH 010/116] Allow url with extension .yml in gitlab processor Signed-off-by: Dede Hamzah --- packages/integration/src/gitlab/core.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts index 83ab29c62e..b30d44e2db 100644 --- a/packages/integration/src/gitlab/core.ts +++ b/packages/integration/src/gitlab/core.ts @@ -76,7 +76,7 @@ export function buildRawUrl(target: string): URL { userOrOrg === '' || repoName === '' || blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) + !restOfPath.join('/').match(/\.(yaml|yml)$/) ) { throw new Error('Wrong GitLab URL'); } From 8113ba5ebbdee128869621793fb9d0972dadbc27 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Tue, 21 Sep 2021 14:58:03 +0700 Subject: [PATCH 011/116] add changest Signed-off-by: Dede Hamzah --- .changeset/great-squids-deliver.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/great-squids-deliver.md diff --git a/.changeset/great-squids-deliver.md b/.changeset/great-squids-deliver.md new file mode 100644 index 0000000000..1d31e3b651 --- /dev/null +++ b/.changeset/great-squids-deliver.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Allow file extension .yml to be ingested in gitlab processor From 14e01923b4d6c122d97eae15e5b26565c6279584 Mon Sep 17 00:00:00 2001 From: Dede Hamzah Date: Tue, 21 Sep 2021 15:38:49 +0700 Subject: [PATCH 012/116] add test Signed-off-by: Dede Hamzah --- packages/integration/src/gitlab/core.test.ts | 47 +++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/packages/integration/src/gitlab/core.test.ts b/packages/integration/src/gitlab/core.test.ts index f07d65e02a..4f39fb2fe7 100644 --- a/packages/integration/src/gitlab/core.test.ts +++ b/packages/integration/src/gitlab/core.test.ts @@ -47,7 +47,7 @@ describe('gitlab core', () => { baseUrl: '', }; - describe('getGitLabFileFetchUrl', () => { + describe('getGitLabFileFetchUrl with .yaml extension', () => { it.each([ // Project URLs { @@ -91,4 +91,49 @@ describe('gitlab core', () => { await expect(getGitLabFileFetchUrl(url, config)).resolves.toBe(result); }); }); + + describe('getGitLabFileFetchUrl with .yml extension', () => { + it.each([ + // Project URLs + { + config: configWithNoToken, + url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', + }, + { + config: configWithNoToken, + // Works with non URI encoded link + url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yml', + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', + }, + { + config: configWithNoToken, + url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yml', + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%20with%20spaces%2Fto%2Ffile.yml/raw?ref=branch', + }, + { + config: configWithToken, + url: 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yml', + result: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%20with%20spaces%2Fto%2Ffile.yml/raw?ref=branch', + }, + { + config: configWithNoToken, + url: 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yml', // Repo not in subgroup + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%20with%20spaces%2Fto%2Ffile.yml/raw?ref=branch', + }, + // Raw URLs + { + config: configWithNoToken, + url: 'https://gitlab.example.com/a/b/blob/master/c.yml', + result: 'https://gitlab.example.com/a/b/raw/master/c.yml', + }, + ])('should handle happy path %#', async ({ config, url, result }) => { + await expect(getGitLabFileFetchUrl(url, config)).resolves.toBe(result); + }); + }); }); From 537bd040057063da83c785eecdd94e7738c46c5e Mon Sep 17 00:00:00 2001 From: Chris Carpita Date: Mon, 20 Sep 2021 21:07:18 -0400 Subject: [PATCH 013/116] fix(SignInPage): Ensure click handler and login popup occur in same tick The current SignIn implementation is broken on Safari iOS by default, due to the combination of useEffect and the extra asynchronous login check that occurs before the authentication routine that causes the popup (instantPopup: true). This change refactors the internals of SignIn page to avoid useEffect, which was arguably confusing for implementors, and also avoid overloading the use of the poorly scoped autoSignIn variable. Signed-off-by: Chris Carpita --- .changeset/hot-shirts-shake.md | 5 + .../src/layout/SignInPage/SignInPage.tsx | 91 ++++++++++--------- 2 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 .changeset/hot-shirts-shake.md diff --git a/.changeset/hot-shirts-shake.md b/.changeset/hot-shirts-shake.md new file mode 100644 index 0000000000..68b133cfdf --- /dev/null +++ b/.changeset/hot-shirts-shake.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': minor +--- + +Fixed a popup-blocking bug affecting iOS Safari in SignInPage.tsx by ensuring that the popup occurs in the same tick as the tap/click diff --git a/packages/core-components/src/layout/SignInPage/SignInPage.tsx b/packages/core-components/src/layout/SignInPage/SignInPage.tsx index e0d4f6ed0d..84998b6ba6 100644 --- a/packages/core-components/src/layout/SignInPage/SignInPage.tsx +++ b/packages/core-components/src/layout/SignInPage/SignInPage.tsx @@ -15,6 +15,7 @@ */ import { + BackstageIdentity, configApiRef, SignInPageProps, useApi, @@ -91,62 +92,63 @@ export const SingleSignInPage = ({ const authApi = useApi(provider.apiRef); const configApi = useApi(configApiRef); - const [autoShowPopup, setAutoShowPopup] = useState(auto ?? false); - // Defaults to true so that an initial check for existing user session is made - const [retry, setRetry] = useState<{} | boolean | undefined>(undefined); const [error, setError] = useState(); + const [loginCount, setLoginCount] = useState(0); // The SignIn component takes some time to decide whether the user is logged-in or not. // showLoginPage is used to prevent a glitch-like experience where the sign-in page is // displayed for a split second when the user is already logged-in. const [showLoginPage, setShowLoginPage] = useState(false); - useEffect(() => { - const login = async () => { - try { - let identity; + type LoginOpts = { checkExisting?: boolean; showPopup?: boolean }; + const login = async ({ checkExisting, showPopup }: LoginOpts) => { + setLoginCount(prev => prev + 1); + try { + let identity: BackstageIdentity | undefined; + if (checkExisting) { // Do an initial check if any logged-in session exists identity = await authApi.getBackstageIdentity({ optional: true, }); - - // If no session exists, show the sign-in page - if (!identity && autoShowPopup) { - // Unless auto is set to true, this step should not happen. - // When user intentionally clicks the Sign In button, autoShowPopup is set to true - setShowLoginPage(true); - identity = await authApi.getBackstageIdentity({ - instantPopup: true, - }); - } - - if (!identity) { - setShowLoginPage(true); - return; - } - - const profile = await authApi.getProfile(); - onResult({ - userId: identity!.id, - profile: profile!, - getIdToken: () => { - return authApi - .getBackstageIdentity() - .then(i => i!.token ?? i!.idToken); - }, - signOut: async () => { - await authApi.signOut(); - }, - }); - } catch (err) { - // User closed the sign-in modal - setError(err); - setShowLoginPage(true); } - }; - login(); - }, [onResult, authApi, retry, autoShowPopup]); + // If no session exists, show the sign-in page + if (!identity && (showPopup || auto)) { + // Unless auto is set to true, this step should not happen. + // When user intentionally clicks the Sign In button, autoShowPopup is set to true + setShowLoginPage(true); + identity = await authApi.getBackstageIdentity({ + instantPopup: true, + }); + } + + if (!identity) { + setShowLoginPage(true); + return; + } + + const profile = await authApi.getProfile(); + onResult({ + userId: identity!.id, + profile: profile!, + getIdToken: () => { + return authApi + .getBackstageIdentity() + .then(i => i!.token ?? i!.idToken); + }, + signOut: async () => { + await authApi.signOut(); + }, + }); + } catch (err: any) { + // User closed the sign-in modal + setError(err); + setShowLoginPage(true); + } + }; + if (loginCount === 0) { + login({ checkExisting: true }); + } return showLoginPage ? ( @@ -168,8 +170,7 @@ export const SingleSignInPage = ({ color="primary" variant="outlined" onClick={() => { - setRetry({}); - setAutoShowPopup(true); + login({ showPopup: true }); }} > Sign In From 641f5e8ba39fec72c33ac724de0c24ee94491e48 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 22 Sep 2021 09:13:37 +0200 Subject: [PATCH 014/116] Add demo videos to catalog graph readme Signed-off-by: Oliver Sand --- plugins/catalog-graph/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-graph/README.md b/plugins/catalog-graph/README.md index 0e06e15f4c..0034cf0c3b 100644 --- a/plugins/catalog-graph/README.md +++ b/plugins/catalog-graph/README.md @@ -9,11 +9,13 @@ The plugin comes with these features: A card that displays the directly related entities to the current entity. This card is for use on the entity page. The card can be customized, for example filtering for specific relations. +