diff --git a/.changeset/flat-crabs-love.md b/.changeset/flat-crabs-love.md new file mode 100644 index 0000000000..96f566d712 --- /dev/null +++ b/.changeset/flat-crabs-love.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-node': minor +--- + +Added refresh function to the `EntityProviderConnection` to be able to schedule refreshes from entity providers. diff --git a/.changeset/rude-weeks-retire.md b/.changeset/rude-weeks-retire.md new file mode 100644 index 0000000000..c2073c5826 --- /dev/null +++ b/.changeset/rude-weeks-retire.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added the `refresh` function to the Connection of the entity providers. diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts index 9d3bba0e8a..3d59d738cf 100644 --- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts @@ -100,6 +100,7 @@ describe('AwsS3EntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = AwsS3EntityProvider.fromConfig(config, { diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts index 1c2c3d0082..c57455d8d6 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts @@ -71,6 +71,7 @@ describe('AzureDevOpsEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = AzureDevOpsEntityProvider.fromConfig(config, { diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts index 08b127f69e..641150acbf 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts @@ -127,6 +127,7 @@ describe('BitbucketCloudEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = BitbucketCloudEntityProvider.fromConfig(config, { logger, diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts index fe2c81b9b6..39689b6c1d 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts @@ -221,6 +221,7 @@ describe('BitbucketServerEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = BitbucketServerEntityProvider.fromConfig(config, { logger, @@ -296,6 +297,7 @@ describe('BitbucketServerEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = BitbucketServerEntityProvider.fromConfig(config, { logger, diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts index d5be045e6c..d6ad0ff668 100644 --- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts @@ -83,6 +83,7 @@ describe('GerritEntityProvider', () => { const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; it('discovers projects from the api.', async () => { diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 871718cc05..63fb77288b 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -146,6 +146,7 @@ describe('GitHubEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitHubEntityProvider.fromConfig(config, { @@ -233,6 +234,7 @@ describe('GitHubEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitHubEntityProvider.fromConfig(config, { @@ -306,6 +308,7 @@ describe('GitHubEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitHubEntityProvider.fromConfig(config, { @@ -404,6 +407,7 @@ it('apply full update on scheduled execution with topic exclusion taking priorit const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitHubEntityProvider.fromConfig(config, { diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts index 77d585eae1..2d05346f01 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts @@ -83,6 +83,7 @@ describe('GitHubOrgEntityProvider', () => { const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const logger = getVoidLogger(); diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts index 7fb4bf8d09..3f3c4e7f91 100644 --- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts +++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts @@ -155,6 +155,7 @@ describe('GitlabDiscoveryEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, @@ -253,6 +254,7 @@ describe('GitlabDiscoveryEntityProvider', () => { const schedule = new PersistingTaskRunner(); const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = GitlabDiscoveryEntityProvider.fromConfig(config, { logger, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts index e34eee0499..ff68235c4c 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts @@ -95,6 +95,7 @@ describe('MicrosoftGraphOrgEntityProvider', () => { }; const entityProviderConnection: EntityProviderConnection = { applyMutation: jest.fn(), + refresh: jest.fn(), }; const provider = MicrosoftGraphOrgEntityProvider.fromConfig( new ConfigReader(config), diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 07241f0682..2d30f8bd86 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -157,6 +157,14 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + /** + * Schedules a refresh for every entity that has a matching set of refresh key stored for it. + */ + refreshByRefreshKeys( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ): Promise; + /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index dfbc37f6c9..2a9b1f8d65 100644 --- a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -26,7 +26,7 @@ describe('DefaultLocationStore', () => { async function createLocationStore(databaseId: TestDatabaseId) { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); - const connection = { applyMutation: jest.fn() }; + const connection = { applyMutation: jest.fn(), refresh: jest.fn() }; const store = new DefaultLocationStore(knex); await store.connect(connection); return { store, connection }; diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts index 70a43de589..a4795532f0 100644 --- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts @@ -22,6 +22,7 @@ import { ProcessingDatabase } from '../database/types'; import { EntityProvider, EntityProviderConnection, + EntityProviderRefreshOptions, EntityProviderMutation, } from '@backstage/plugin-catalog-node'; @@ -61,6 +62,16 @@ class Connection implements EntityProviderConnection { } } + async refresh(options: EntityProviderRefreshOptions): Promise { + const db = this.config.processingDatabase; + + await db.transaction(async (tx: any) => { + return db.refreshByRefreshKeys(tx, { + keys: options.keys, + }); + }); + } + private check(entities: Entity[]) { for (const entity of entities) { try { diff --git a/plugins/catalog-node/api-report.md b/plugins/catalog-node/api-report.md index 7d12c2e065..3873e3376d 100644 --- a/plugins/catalog-node/api-report.md +++ b/plugins/catalog-node/api-report.md @@ -126,6 +126,7 @@ export interface EntityProvider { // @public export interface EntityProviderConnection { applyMutation(mutation: EntityProviderMutation): Promise; + refresh(options: EntityProviderRefreshOptions): Promise; } // @public @@ -140,6 +141,11 @@ export type EntityProviderMutation = removed: DeferredEntity[]; }; +// @public (undocumented) +export type EntityProviderRefreshOptions = { + keys: string[]; +}; + // @public export type EntityRelationSpec = { source: CompoundEntityRef; diff --git a/plugins/catalog-node/src/api/index.ts b/plugins/catalog-node/src/api/index.ts index 136c23d20d..66e8d6bd21 100644 --- a/plugins/catalog-node/src/api/index.ts +++ b/plugins/catalog-node/src/api/index.ts @@ -32,4 +32,5 @@ export type { EntityProvider, EntityProviderConnection, EntityProviderMutation, + EntityProviderRefreshOptions, } from './provider'; diff --git a/plugins/catalog-node/src/api/provider.ts b/plugins/catalog-node/src/api/provider.ts index fa7659ee32..fc9a172e0d 100644 --- a/plugins/catalog-node/src/api/provider.ts +++ b/plugins/catalog-node/src/api/provider.ts @@ -26,6 +26,13 @@ export type EntityProviderMutation = | { type: 'full'; entities: DeferredEntity[] } | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] }; +/** + * @public + */ +export type EntityProviderRefreshOptions = { + keys: string[]; +}; + /** * The EntityProviderConnection is the connection between the catalog and the entity provider. * The EntityProvider use this connection to add and remove entities from the catalog. @@ -36,6 +43,11 @@ export interface EntityProviderConnection { * Applies either a full or delta update to the catalog engine. */ applyMutation(mutation: EntityProviderMutation): Promise; + + /** + * Schedules a refresh on all of the entities that has a matching refresh key associated with the provided keys. + */ + refresh(options: EntityProviderRefreshOptions): Promise; } /**