Merge pull request #13575 from RoadieHQ/expose-refresh-on-entity-providers

feat: add refresh to EntityProviderConnection

.changeset/flat-crabs-love.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-node': minor
+---
+
+Added refresh function to the `EntityProviderConnection` to be able to schedule refreshes from entity providers.

.changeset/rude-weeks-retire.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added the `refresh` function to the Connection of the entity providers.

plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts

@@ -100,6 +100,7 @@ describe('AwsS3EntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = AwsS3EntityProvider.fromConfig(config, {

plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts

@@ -71,6 +71,7 @@ describe('AzureDevOpsEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = AzureDevOpsEntityProvider.fromConfig(config, {

plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts

@@ -127,6 +127,7 @@ describe('BitbucketCloudEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketCloudEntityProvider.fromConfig(config, {
       logger,

plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts

@@ -221,6 +221,7 @@ describe('BitbucketServerEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketServerEntityProvider.fromConfig(config, {
       logger,
@@ -296,6 +297,7 @@ describe('BitbucketServerEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketServerEntityProvider.fromConfig(config, {
       logger,

plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts

@@ -83,6 +83,7 @@ describe('GerritEntityProvider', () => {
 
   const entityProviderConnection: EntityProviderConnection = {
     applyMutation: jest.fn(),
+    refresh: jest.fn(),
   };
 
   it('discovers projects from the api.', async () => {

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts

@@ -146,6 +146,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -233,6 +234,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -306,6 +308,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -404,6 +407,7 @@ it('apply full update on scheduled execution with topic exclusion taking priorit
   const schedule = new PersistingTaskRunner();
   const entityProviderConnection: EntityProviderConnection = {
     applyMutation: jest.fn(),
+    refresh: jest.fn(),
   };
 
   const provider = GitHubEntityProvider.fromConfig(config, {

plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts

@@ -83,6 +83,7 @@ describe('GitHubOrgEntityProvider', () => {
 
       const entityProviderConnection: EntityProviderConnection = {
         applyMutation: jest.fn(),
+        refresh: jest.fn(),
       };
 
       const logger = getVoidLogger();

plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts

@@ -155,6 +155,7 @@ describe('GitlabDiscoveryEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
@@ -253,6 +254,7 @@ describe('GitlabDiscoveryEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts

@@ -95,6 +95,7 @@ describe('MicrosoftGraphOrgEntityProvider', () => {
     };
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = MicrosoftGraphOrgEntityProvider.fromConfig(
       new ConfigReader(config),

plugins/catalog-backend/src/database/types.ts

@@ -157,6 +157,14 @@ export interface ProcessingDatabase {
    */
   refresh(txOpaque: Transaction, options: RefreshOptions): Promise<void>;
 
+  /**
+   * Schedules a refresh for every entity that has a matching set of refresh key stored for it.
+   */
+  refreshByRefreshKeys(
+    txOpaque: Transaction,
+    options: RefreshByKeyOptions,
+  ): Promise<void>;
+
   /**
    * Lists all ancestors of a given entityRef.
    *

plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts

@@ -26,7 +26,7 @@ describe('DefaultLocationStore', () => {
   async function createLocationStore(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);
     await applyDatabaseMigrations(knex);
-    const connection = { applyMutation: jest.fn() };
+    const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
     const store = new DefaultLocationStore(knex);
     await store.connect(connection);
     return { store, connection };

plugins/catalog-backend/src/processing/connectEntityProviders.ts

@@ -22,6 +22,7 @@ import { ProcessingDatabase } from '../database/types';
 import {
   EntityProvider,
   EntityProviderConnection,
+  EntityProviderRefreshOptions,
   EntityProviderMutation,
 } from '@backstage/plugin-catalog-node';
 
@@ -61,6 +62,16 @@ class Connection implements EntityProviderConnection {
     }
   }
 
+  async refresh(options: EntityProviderRefreshOptions): Promise<void> {
+    const db = this.config.processingDatabase;
+
+    await db.transaction(async (tx: any) => {
+      return db.refreshByRefreshKeys(tx, {
+        keys: options.keys,
+      });
+    });
+  }
+
   private check(entities: Entity[]) {
     for (const entity of entities) {
       try {

plugins/catalog-node/api-report.md

@@ -126,6 +126,7 @@ export interface EntityProvider {
 // @public
 export interface EntityProviderConnection {
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
+  refresh(options: EntityProviderRefreshOptions): Promise<void>;
 }
 
 // @public
@@ -140,6 +141,11 @@ export type EntityProviderMutation =
       removed: DeferredEntity[];
     };
 
+// @public (undocumented)
+export type EntityProviderRefreshOptions = {
+  keys: string[];
+};
+
 // @public
 export type EntityRelationSpec = {
   source: CompoundEntityRef;

plugins/catalog-node/src/api/index.ts

@@ -32,4 +32,5 @@ export type {
   EntityProvider,
   EntityProviderConnection,
   EntityProviderMutation,
+  EntityProviderRefreshOptions,
 } from './provider';

plugins/catalog-node/src/api/provider.ts

@@ -26,6 +26,13 @@ export type EntityProviderMutation =
   | { type: 'full'; entities: DeferredEntity[] }
   | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
 
+/**
+ * @public
+ */
+export type EntityProviderRefreshOptions = {
+  keys: string[];
+};
+
 /**
  * The EntityProviderConnection is the connection between the catalog and the entity provider.
  * The EntityProvider use this connection to add and remove entities from the catalog.
@@ -36,6 +43,11 @@ export interface EntityProviderConnection {
    * Applies either a full or delta update to the catalog engine.
    */
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
+
+  /**
+   * Schedules a refresh on all of the entities that has a matching refresh key associated with the provided keys.
+   */
+  refresh(options: EntityProviderRefreshOptions): Promise<void>;
 }
 
 /**