diff --git a/.changeset/angry-starfishes-confess.md b/.changeset/angry-starfishes-confess.md new file mode 100644 index 0000000000..5cb4395d9a --- /dev/null +++ b/.changeset/angry-starfishes-confess.md @@ -0,0 +1,41 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch +--- + +Handle Bitbucket Cloud `repo:push` events at the `BitbucketCloudEntityProvider` +by subscribing to the topic `bitbucketCloud.repo:push.` + +Implements `EventSubscriber` to receive events for the topic `bitbucketCloud.repo:push`. + +On `repo:push`, the affected repository will be refreshed. +This includes adding new Location entities, refreshing existing ones, +and removing obsolete ones. + +To support this, a new annotation `bitbucket.org/repo-url` was added +to Location entities. + +A full refresh will require 1 API call to Bitbucket Cloud to discover all catalog files. +When we handle one `repo:push` event, we also need 1 API call in order to know +which catalog files exist. +This may lead to more discovery-related API calls (code search). +The main cause for hitting the rate limits are Locations refresh-related operations. + +A reduction of total API calls to reduce the rate limit issues can only be achieved in +combination with + +1. reducing the full refresh frequency (e.g., to monthly) +2. reducing the frequency of general Location refresh operations by the processing loop + +For (2.), it is not possible to reduce the frequency only for Bitbucket Cloud-related +Locations though. + +Further optimizations might be required to resolve the rate limit issue. + +**Installation and Migration** + +Please find more information at +https://backstage.io/docs/integrations/bitbucketCloud/discovery, +in particular the section about "_Installation with Events Support_". + +In case of the new backend-plugin-api _(alpha)_ the module will take care of +registering itself at both. diff --git a/.changeset/small-chefs-tell.md b/.changeset/small-chefs-tell.md new file mode 100644 index 0000000000..46d8a33f3c --- /dev/null +++ b/.changeset/small-chefs-tell.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-bitbucket-cloud-common': patch +--- + +Add interfaces for Bitbucket Cloud (webhook) events. diff --git a/docs/integrations/bitbucketCloud/discovery.md b/docs/integrations/bitbucketCloud/discovery.md index 9b9bc4ecde..f6ceab9ff7 100644 --- a/docs/integrations/bitbucketCloud/discovery.md +++ b/docs/integrations/bitbucketCloud/discovery.md @@ -24,10 +24,12 @@ package. yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-bitbucket-cloud ``` +### Installation without Events Support + And then add the entity provider to your catalog builder: ```diff - // In packages/backend/src/plugins/catalog.ts +// packages/backend/src/plugins/catalog.ts + import { BitbucketCloudEntityProvider } from '@backstage/plugin-catalog-backend-module-bitbucket-cloud'; export default async function createPlugin( @@ -37,11 +39,7 @@ And then add the entity provider to your catalog builder: + builder.addEntityProvider( + BitbucketCloudEntityProvider.fromConfig(env.config, { + logger: env.logger, -+ // optional: alternatively, configure via app-config.yaml -+ schedule: env.scheduler.createScheduledTaskRunner({ -+ frequency: { minutes: 30 }, -+ timeout: { minutes: 3 }, -+ }), ++ scheduler: env.scheduler, + }), + ); @@ -49,6 +47,62 @@ And then add the entity provider to your catalog builder: } ``` +Alternatively to the config-based schedule, you can use + +```diff +- scheduler: env.scheduler, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), +``` + +### Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-bitbucket-cloud/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { CatalogClient } from '@backstage/catalog-client'; ++import { BitbucketCloudEntityProvider } from '@backstage/plugin-catalog-backend-module-bitbucket-cloud'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ BitbucketCloudEntityProvider.fromConfig(env.config, { ++ catalogApi: new CatalogClient({ discoveryApi: env.discovery }), ++ logger: env.logger, ++ scheduler: env.scheduler, ++ tokenManager: env.tokenManager, ++ }), ++ ); + return providers.flat(); + } +``` + +**Attention:** +`catalogApi` and `tokenManager` are required at this variant +compared to the one without events support. + ## Configuration To use the entity provider, you'll need a [Bitbucket Cloud integration set up](locations.md). diff --git a/plugins/bitbucket-cloud-common/api-report.md b/plugins/bitbucket-cloud-common/api-report.md index d66b8b9a70..5182579a4b 100644 --- a/plugins/bitbucket-cloud-common/api-report.md +++ b/plugins/bitbucket-cloud-common/api-report.md @@ -24,6 +24,57 @@ export class BitbucketCloudClient { ): WithPagination; } +// @public (undocumented) +export namespace Events { + // (undocumented) + export interface Change { + // (undocumented) + closed: boolean; + // (undocumented) + commits: Models.Commit[]; + // (undocumented) + created: boolean; + // (undocumented) + forced: boolean; + // (undocumented) + links: ChangeLinks; + // (undocumented) + new: Models.Branch; + // (undocumented) + old: Models.Branch; + // (undocumented) + truncated: boolean; + } + // (undocumented) + export interface ChangeLinks { + // (undocumented) + commits: Models.Link; + // (undocumented) + diff: Models.Link; + // (undocumented) + html: Models.Link; + } + // (undocumented) + export interface RepoEvent { + // (undocumented) + actor: Models.Account; + // (undocumented) + repository: Models.Repository & { + workspace: Models.Workspace; + }; + } + // (undocumented) + export interface RepoPush { + // (undocumented) + changes: Change[]; + } + // (undocumented) + export interface RepoPushEvent extends RepoEvent { + // (undocumented) + push: RepoPush; + } +} + // @public (undocumented) export type FilterAndSortOptions = { q?: string; @@ -340,6 +391,37 @@ export namespace Models { // (undocumented) self?: Link; } + export interface Workspace extends ModelObject { + // (undocumented) + created_on?: string; + is_private?: boolean; + // (undocumented) + links?: WorkspaceLinks; + name?: string; + slug?: string; + // (undocumented) + updated_on?: string; + uuid?: string; + } + // (undocumented) + export interface WorkspaceLinks { + // (undocumented) + avatar?: Link; + // (undocumented) + html?: Link; + // (undocumented) + members?: Link; + // (undocumented) + owners?: Link; + // (undocumented) + projects?: Link; + // (undocumented) + repositories?: Link; + // (undocumented) + self?: Link; + // (undocumented) + snippets?: Link; + } } // @public (undocumented) diff --git a/plugins/bitbucket-cloud-common/scripts/reduce-models.js b/plugins/bitbucket-cloud-common/scripts/reduce-models.js index 9d458f4e74..115020493c 100755 --- a/plugins/bitbucket-cloud-common/scripts/reduce-models.js +++ b/plugins/bitbucket-cloud-common/scripts/reduce-models.js @@ -30,6 +30,14 @@ const modelsModule = modelsFile.getModuleOrThrow('Models'); const clientFile = project.getSourceFile('src/BitbucketCloudClient.ts'); const clientClass = clientFile.getClassOrThrow('BitbucketCloudClient'); +const eventsFile = project.getSourceFile('src/events/index.ts'); +const eventsModule = eventsFile.getModuleOrThrow('Events'); +const eventsStmts = [ + ...eventsModule.getClasses(), + ...eventsModule.getInterfaces(), + ...eventsModule.getTypeAliases(), +]; + /** * Returns an array of the unique items of the provided array. * @@ -79,7 +87,11 @@ function referencedModelsIdentifiers(stmt, processed) { } // all directly or transitively referenced/used `Models.[...]` are allowed to stay -const allowed = referencedModelsIdentifiers(clientClass); +const processed = []; +const allowed = referencedModelsIdentifiers(clientClass, processed); +allowed.push( + ...eventsStmts.flatMap(stmt => referencedModelsIdentifiers(stmt, processed)), +); // remove everything not part of the "allow list" modelsModule diff --git a/plugins/bitbucket-cloud-common/src/events/index.ts b/plugins/bitbucket-cloud-common/src/events/index.ts new file mode 100644 index 0000000000..53658be032 --- /dev/null +++ b/plugins/bitbucket-cloud-common/src/events/index.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Models } from '../models'; + +// source: https://support.atlassian.com/bitbucket-cloud/docs/event-payloads + +/** @public */ +export namespace Events { + /** @public */ + export interface RepoEvent { + repository: Models.Repository & { workspace: Models.Workspace }; + actor: Models.Account; + } + + /** @public */ + export interface RepoPushEvent extends RepoEvent { + push: RepoPush; + } + + /** @public */ + export interface RepoPush { + changes: Change[]; + } + + /** @public */ + export interface Change { + old: Models.Branch; + new: Models.Branch; + truncated: boolean; + created: boolean; + forced: boolean; + closed: boolean; + links: ChangeLinks; + commits: Models.Commit[]; + } + + /** @public */ + export interface ChangeLinks { + commits: Models.Link; + diff: Models.Link; + html: Models.Link; + } +} diff --git a/plugins/bitbucket-cloud-common/src/index.ts b/plugins/bitbucket-cloud-common/src/index.ts index 4edf148240..4a9b00e89b 100644 --- a/plugins/bitbucket-cloud-common/src/index.ts +++ b/plugins/bitbucket-cloud-common/src/index.ts @@ -21,6 +21,7 @@ */ export * from './BitbucketCloudClient'; +export * from './events'; export * from './models'; export * from './pagination'; export * from './types'; diff --git a/plugins/bitbucket-cloud-common/src/models/index.ts b/plugins/bitbucket-cloud-common/src/models/index.ts index 580f23a13f..9d25bbb873 100644 --- a/plugins/bitbucket-cloud-common/src/models/index.ts +++ b/plugins/bitbucket-cloud-common/src/models/index.ts @@ -518,4 +518,46 @@ export namespace Models { repositories?: Link; self?: Link; } + + /** + * A Bitbucket workspace. + * Workspaces are used to organize repositories. + * @public + */ + export interface Workspace extends ModelObject { + created_on?: string; + /** + * Indicates whether the workspace is publicly accessible, or whether it is + * private to the members and consequently only visible to members. + */ + is_private?: boolean; + links?: WorkspaceLinks; + /** + * The name of the workspace. + */ + name?: string; + /** + * The short label that identifies this workspace. + */ + slug?: string; + updated_on?: string; + /** + * The workspace's immutable id. + */ + uuid?: string; + } + + /** + * @public + */ + export interface WorkspaceLinks { + avatar?: Link; + html?: Link; + members?: Link; + owners?: Link; + projects?: Link; + repositories?: Link; + self?: Link; + snippets?: Link; + } } diff --git a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md index 162ff8f67e..480395e477 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/api-report.md +++ b/plugins/catalog-backend-module-bitbucket-cloud/api-report.md @@ -4,24 +4,33 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EventParams } from '@backstage/plugin-events-node'; +import { Events } from '@backstage/plugin-bitbucket-cloud-common'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { Logger } from 'winston'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TaskRunner } from '@backstage/backend-tasks'; +import { TokenManager } from '@backstage/backend-common'; // @public -export class BitbucketCloudEntityProvider implements EntityProvider { +export class BitbucketCloudEntityProvider + implements EntityProvider, EventSubscriber +{ // (undocumented) connect(connection: EntityProviderConnection): Promise; // (undocumented) static fromConfig( config: Config, options: { + catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + tokenManager?: TokenManager; }, ): BitbucketCloudEntityProvider[]; // (undocumented) @@ -29,7 +38,13 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // (undocumented) getTaskId(): string; // (undocumented) + onEvent(params: EventParams): Promise; + // (undocumented) + onRepoPush(event: Events.RepoPushEvent): Promise; + // (undocumented) refresh(logger: Logger): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @alpha (undocumented) diff --git a/plugins/catalog-backend-module-bitbucket-cloud/package.json b/plugins/catalog-backend-module-bitbucket-cloud/package.json index 45876470d4..e7cd57b560 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/package.json +++ b/plugins/catalog-backend-module-bitbucket-cloud/package.json @@ -33,13 +33,19 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", + "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-bitbucket-cloud-common": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "p-limit": "^3.1.0", "uuid": "^8.0.0", "winston": "^3.2.1" }, diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts index 7d780cd064..72974a1c94 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts @@ -333,6 +333,8 @@ describe('BitbucketCloudEntityProvider', () => { annotations: { 'backstage.io/managed-by-location': `url:${url}`, 'backstage.io/managed-by-origin-location': `url:${url}`, + 'bitbucket.org/repo-url': + 'https://bitbucket.org/test-ws/test-repo2', }, name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b', }, diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts index f3334a6574..37ad60426d 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.ts @@ -14,7 +14,14 @@ * limitations under the License. */ +import { TokenManager } from '@backstage/backend-common'; import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import { CatalogApi } from '@backstage/catalog-client'; +import { + Entity, + LocationEntity, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { BitbucketCloudIntegration, @@ -22,22 +29,35 @@ import { } from '@backstage/integration'; import { BitbucketCloudClient, + Events, Models, } from '@backstage/plugin-bitbucket-cloud-common'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; import { BitbucketCloudEntityProviderConfig, readProviderConfigs, } from './BitbucketCloudEntityProviderConfig'; +import limiterFactory from 'p-limit'; import * as uuid from 'uuid'; import { Logger } from 'winston'; const DEFAULT_BRANCH = 'master'; +const TOPIC_REPO_PUSH = 'bitbucketCloud/repo:push'; + +/** @public */ +export const ANNOTATION_BITBUCKET_CLOUD_REPO_URL = 'bitbucket.org/repo-url'; + +interface IngestionTarget { + fileUrl: string; + repoUrl: string; +} /** * Discovers catalog files located in [Bitbucket Cloud](https://bitbucket.org). @@ -47,19 +67,27 @@ const DEFAULT_BRANCH = 'master'; * * @public */ -export class BitbucketCloudEntityProvider implements EntityProvider { +export class BitbucketCloudEntityProvider + implements EntityProvider, EventSubscriber +{ private readonly client: BitbucketCloudClient; private readonly config: BitbucketCloudEntityProviderConfig; private readonly logger: Logger; private readonly scheduleFn: () => Promise; + private readonly catalogApi?: CatalogApi; + private readonly tokenManager?: TokenManager; private connection?: EntityProviderConnection; + private eventConfigErrorThrown = false; + static fromConfig( config: Config, options: { + catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + tokenManager?: TokenManager; }, ): BitbucketCloudEntityProvider[] { const integrations = ScmIntegrations.fromConfig(config); @@ -90,6 +118,8 @@ export class BitbucketCloudEntityProvider implements EntityProvider { integration, options.logger, taskRunner, + options.catalogApi, + options.tokenManager, ); }); } @@ -99,6 +129,8 @@ export class BitbucketCloudEntityProvider implements EntityProvider { integration: BitbucketCloudIntegration, logger: Logger, taskRunner: TaskRunner, + catalogApi?: CatalogApi, + tokenManager?: TokenManager, ) { this.client = BitbucketCloudClient.fromConfig(integration.config); this.config = config; @@ -106,6 +138,8 @@ export class BitbucketCloudEntityProvider implements EntityProvider { target: this.getProviderName(), }); this.scheduleFn = this.createScheduleFn(taskRunner); + this.catalogApi = catalogApi; + this.tokenManager = tokenManager; } private createScheduleFn(schedule: TaskRunner): () => Promise { @@ -154,15 +188,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider { logger.info('Discovering catalog files in Bitbucket Cloud repositories'); const targets = await this.findCatalogFiles(); - const entities = targets - .map(BitbucketCloudEntityProvider.toLocationSpec) - .map(location => locationSpecToLocationEntity({ location })) - .map(entity => { - return { - locationKey: this.getProviderName(), - entity: entity, - }; - }); + const entities = this.toDeferredEntities(targets); await this.connection.applyMutation({ type: 'full', @@ -174,7 +200,135 @@ export class BitbucketCloudEntityProvider implements EntityProvider { ); } - private async findCatalogFiles(): Promise { + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return [TOPIC_REPO_PUSH]; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + if (params.metadata?.['x-event-key'] === 'repo:push') { + await this.onRepoPush(params.eventPayload as Events.RepoPushEvent); + } + } + + private canHandleEvents(): boolean { + if (this.catalogApi && this.tokenManager) { + return true; + } + + // throw only once + if (!this.eventConfigErrorThrown) { + this.eventConfigErrorThrown = true; + throw new Error( + `${this.getProviderName()} not well configured to handle repo:push. Missing CatalogApi and/or TokenManager.`, + ); + } + + return false; + } + + async onRepoPush(event: Events.RepoPushEvent): Promise { + if (!this.canHandleEvents()) { + return; + } + + if (!this.connection) { + throw new Error('Not initialized'); + } + + if (event.repository.workspace.slug !== this.config.workspace) { + return; + } + + if (!this.matchesFilters(event.repository)) { + return; + } + + const repoName = event.repository.slug; + const repoUrl = event.repository.links!.html!.href!; + this.logger.info(`handle repo:push event for ${repoUrl}`); + + // The commit information at the webhook only contains some high level metadata. + // In order to understand whether relevant files have changed we would need to + // look up all commits which would cost additional API calls. + // The overall goal is to optimize the necessary amount of API calls. + // Hence, we will just trigger a refresh for catalog file(s) within the repository + // if we get notified about changes there. + + const targets = await this.findCatalogFiles(repoName); + + const { token } = await this.tokenManager!.getToken(); + const existing = await this.findExistingLocations(repoUrl, token); + + const added: DeferredEntity[] = this.toDeferredEntities( + targets.filter( + // All Locations are managed by this provider and only have `target`, never `targets`. + // All URLs (fileUrl, target) are created using `BitbucketCloudEntityProvider.toUrl`. + // Hence, we can keep the comparison simple and don't need to handle different + // casing or encoding, etc. + target => !existing.find(item => item.spec.target === target.fileUrl), + ), + ); + + const limiter = limiterFactory(10); + + const stillExisting: Entity[] = []; + const removed: DeferredEntity[] = []; + existing.forEach(item => { + if (targets.find(value => value.fileUrl === item.spec.target)) { + stillExisting.push(item); + } else { + removed.push({ + locationKey: this.getProviderName(), + entity: item, + }); + } + }); + + const promises: Promise[] = stillExisting.map(entity => + limiter(async () => + this.catalogApi!.refreshEntity(stringifyEntityRef(entity), { token }), + ), + ); + + if (added.length > 0 || removed.length > 0) { + const connection = this.connection; + promises.push( + limiter(async () => + connection.applyMutation({ + type: 'delta', + added: added, + removed: removed, + }), + ), + ); + } + + await Promise.all(promises); + } + + private async findExistingLocations( + repoUrl: string, + token: string, + ): Promise { + const filter: Record = {}; + filter.kind = 'Location'; + filter[`metadata.annotations.${ANNOTATION_BITBUCKET_CLOUD_REPO_URL}`] = + repoUrl; + + return this.catalogApi!.getEntities({ filter }, { token }).then( + result => result.items, + ) as Promise; + } + + private async findCatalogFiles( + repoName?: string, + ): Promise { const workspace = this.config.workspace; const catalogPath = this.config.catalogPath; @@ -197,12 +351,13 @@ export class BitbucketCloudEntityProvider implements EntityProvider { // ...except the one we need '+values.file.commit.repository.links.html.href', ].join(','); - const query = `"${catalogFilename}" path:${catalogPath}`; + const optRepoFilter = repoName ? ` repo:${repoName}` : ''; + const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`; const searchResults = this.client .searchCode(workspace, query, { fields }) .iterateResults(); - const result: string[] = []; + const result: IngestionTarget[] = []; for await (const searchResult of searchResults) { // not a file match, but a code match @@ -212,12 +367,13 @@ export class BitbucketCloudEntityProvider implements EntityProvider { const repository = searchResult.file!.commit!.repository!; if (this.matchesFilters(repository)) { - result.push( - BitbucketCloudEntityProvider.toUrl( + result.push({ + fileUrl: BitbucketCloudEntityProvider.toUrl( repository, searchResult.file!.path!, ), - ); + repoUrl: repository.links!.html!.href!, + }); } } @@ -234,11 +390,32 @@ export class BitbucketCloudEntityProvider implements EntityProvider { ); } + private toDeferredEntities(targets: IngestionTarget[]): DeferredEntity[] { + return targets + .map(target => { + const location = BitbucketCloudEntityProvider.toLocationSpec( + target.fileUrl, + ); + const entity = locationSpecToLocationEntity({ location }); + entity.metadata.annotations = { + ...entity.metadata.annotations, + [ANNOTATION_BITBUCKET_CLOUD_REPO_URL]: target.repoUrl, + }; + return entity; + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } + private static toUrl( repository: Models.Repository, filePath: string, ): string { - const repoUrl = repository.links!.html!.href; + const repoUrl = repository.links!.html!.href!; const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH; return `${repoUrl}/src/${branch}/${filePath}`; diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts index 7606f7c36a..3fcc39fdd0 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts @@ -15,11 +15,17 @@ */ import { ConfigReader } from '@backstage/config'; -import { getVoidLogger } from '@backstage/backend-common'; +import { + getVoidLogger, + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; import { configServiceRef, + discoveryServiceRef, loggerServiceRef, schedulerServiceRef, + tokenManagerServiceRef, } from '@backstage/backend-plugin-api'; import { PluginTaskScheduler, @@ -27,6 +33,7 @@ import { } from '@backstage/backend-tasks'; import { startTestBackend } from '@backstage/backend-test-utils'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { eventsExtensionPoint } from '@backstage/plugin-events-node'; import { Duration } from 'luxon'; import { bitbucketCloudEntityProviderCatalogModule } from './BitbucketCloudEntityProviderCatalogModule'; import { BitbucketCloudEntityProvider } from '../BitbucketCloudEntityProvider'; @@ -34,13 +41,19 @@ import { BitbucketCloudEntityProvider } from '../BitbucketCloudEntityProvider'; describe('bitbucketCloudEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { let addedProviders: Array | undefined; + let addedSubscribers: Array | undefined; let usedSchedule: TaskScheduleDefinition | undefined; - const extensionPoint = { + const catalogExtensionPointImpl = { addEntityProvider: (providers: any) => { addedProviders = providers; }, }; + const eventsExtensionPointImpl = { + addSubscribers: (subscribers: any) => { + addedSubscribers = subscribers; + }, + }; const runner = jest.fn(); const scheduler = { createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => { @@ -48,6 +61,8 @@ describe('bitbucketCloudEntityProviderCatalogModule', () => { return runner; }, } as unknown as PluginTaskScheduler; + const discovery = jest.fn() as any as PluginEndpointDiscovery; + const tokenManager = jest.fn() as any as TokenManager; const config = new ConfigReader({ catalog: { @@ -64,11 +79,16 @@ describe('bitbucketCloudEntityProviderCatalogModule', () => { }); await startTestBackend({ - extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + extensionPoints: [ + [catalogProcessingExtensionPoint, catalogExtensionPointImpl], + [eventsExtensionPoint, eventsExtensionPointImpl], + ], services: [ [configServiceRef, config], + [discoveryServiceRef, discovery], [loggerServiceRef, getVoidLogger()], [schedulerServiceRef, scheduler], + [tokenManagerServiceRef, tokenManager], ], features: [bitbucketCloudEntityProviderCatalogModule()], }); @@ -79,6 +99,7 @@ describe('bitbucketCloudEntityProviderCatalogModule', () => { expect(addedProviders?.pop()?.getProviderName()).toEqual( 'bitbucketCloud-provider:default', ); + expect(addedSubscribers).toEqual(addedProviders); expect(runner).not.toHaveBeenCalled(); }); }); diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts index d137bd7166..1c005991a6 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.ts @@ -20,8 +20,13 @@ import { loggerServiceRef, loggerToWinstonLogger, schedulerServiceRef, + tokenManagerServiceRef, } from '@backstage/backend-plugin-api'; -import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { + catalogProcessingExtensionPoint, + catalogServiceRef, +} from '@backstage/plugin-catalog-node'; +import { eventsExtensionPoint } from '@backstage/plugin-events-node'; import { BitbucketCloudEntityProvider } from '../BitbucketCloudEntityProvider'; /** @@ -34,18 +39,34 @@ export const bitbucketCloudEntityProviderCatalogModule = createBackendModule({ env.registerInit({ deps: { catalog: catalogProcessingExtensionPoint, + catalogApi: catalogServiceRef, config: configServiceRef, + // TODO(pjungermann): How to make this optional for those which only want the provider without event support? + // Do we even want to support this? + events: eventsExtensionPoint, logger: loggerServiceRef, scheduler: schedulerServiceRef, + tokenManager: tokenManagerServiceRef, }, - async init({ catalog, config, logger, scheduler }) { + async init({ + catalog, + catalogApi, + config, + events, + logger, + scheduler, + tokenManager, + }) { const winstonLogger = loggerToWinstonLogger(logger); const providers = BitbucketCloudEntityProvider.fromConfig(config, { + catalogApi, logger: winstonLogger, scheduler, + tokenManager, }); catalog.addEntityProvider(providers); + events.addSubscribers(providers); }, }); }, diff --git a/yarn.lock b/yarn.lock index 6d392e7561..7bed901253 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5449,14 +5449,19 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-client": "workspace:^" + "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-bitbucket-cloud-common": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" luxon: ^3.0.0 msw: ^0.48.0 + p-limit: ^3.1.0 uuid: ^8.0.0 winston: ^3.2.1 languageName: unknown