From 858833a4d5d5bbf4b2d1d5418f6ae5a92ec0d5d3 Mon Sep 17 00:00:00 2001
From: Kiss Miklos <miklos@roadie.io>
Date: Wed, 7 Sep 2022 13:00:48 +0200
Subject: [PATCH 1/5] add refresh to connection

Signed-off-by: Kiss Miklos <miklos@roadie.io>
---
 plugins/catalog-backend/src/database/types.ts          |  8 ++++++++
 .../src/processing/connectEntityProviders.ts           | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts
index 07241f0682..2d30f8bd86 100644
--- a/plugins/catalog-backend/src/database/types.ts
+++ b/plugins/catalog-backend/src/database/types.ts
@@ -157,6 +157,14 @@ export interface ProcessingDatabase {
    */
   refresh(txOpaque: Transaction, options: RefreshOptions): Promise<void>;
 
+  /**
+   * Schedules a refresh for every entity that has a matching set of refresh key stored for it.
+   */
+  refreshByRefreshKeys(
+    txOpaque: Transaction,
+    options: RefreshByKeyOptions,
+  ): Promise<void>;
+
   /**
    * Lists all ancestors of a given entityRef.
    *
diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts
index 70a43de589..8325e2bcac 100644
--- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts
+++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts
@@ -61,6 +61,16 @@ class Connection implements EntityProviderConnection {
     }
   }
 
+  async refresh(keys: string[]): Promise<void> {
+    const db = this.config.processingDatabase;
+
+    await db.transaction(async (tx: any) => {
+      return db.refreshByRefreshKeys(tx, {
+        keys,
+      });
+    });
+  }
+
   private check(entities: Entity[]) {
     for (const entity of entities) {
       try {

From 6e63bc43f27d0b72ee17f2273fe7e1267223138b Mon Sep 17 00:00:00 2001
From: Kiss Miklos <miklos@roadie.io>
Date: Wed, 7 Sep 2022 13:58:39 +0200
Subject: [PATCH 2/5] add changeset

Signed-off-by: Kiss Miklos <miklos@roadie.io>
---
 .changeset/rude-weeks-retire.md          | 5 +++++
 plugins/catalog-node/src/api/provider.ts | 1 +
 2 files changed, 6 insertions(+)
 create mode 100644 .changeset/rude-weeks-retire.md

diff --git a/.changeset/rude-weeks-retire.md b/.changeset/rude-weeks-retire.md
new file mode 100644
index 0000000000..fddddd7a9c
--- /dev/null
+++ b/.changeset/rude-weeks-retire.md
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added refresh function to the EntityProviderConnection to be able to schedule refreshes from entity providers.
diff --git a/plugins/catalog-node/src/api/provider.ts b/plugins/catalog-node/src/api/provider.ts
index fa7659ee32..437da4e7d6 100644
--- a/plugins/catalog-node/src/api/provider.ts
+++ b/plugins/catalog-node/src/api/provider.ts
@@ -36,6 +36,7 @@ export interface EntityProviderConnection {
    * Applies either a full or delta update to the catalog engine.
    */
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
+  refresh(keys: string[]): Promise<void>;
 }
 
 /**

From 652dc0fa0c6b554e0989e38a46a3353d2965b2b5 Mon Sep 17 00:00:00 2001
From: Kiss Miklos <miklos@roadie.io>
Date: Wed, 7 Sep 2022 14:03:47 +0200
Subject: [PATCH 3/5] fix tests

Signed-off-by: Kiss Miklos <miklos@roadie.io>
---
 .../src/providers/AwsS3EntityProvider.test.ts                 | 1 +
 .../src/providers/AzureDevOpsEntityProvider.test.ts           | 1 +
 .../src/BitbucketCloudEntityProvider.test.ts                  | 1 +
 .../src/providers/BitbucketServerEntityProvider.test.ts       | 2 ++
 .../src/providers/GerritEntityProvider.test.ts                | 1 +
 .../src/providers/GitHubEntityProvider.test.ts                | 4 ++++
 .../src/providers/GitHubOrgEntityProvider.test.ts             | 1 +
 .../src/providers/GitlabDiscoveryEntityProvider.test.ts       | 2 ++
 .../src/processors/MicrosoftGraphOrgEntityProvider.test.ts    | 1 +
 .../src/modules/core/DefaultLocationStore.test.ts             | 2 +-
 plugins/catalog-node/src/api/provider.ts                      | 4 ++++
 11 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts
index 9d3bba0e8a..3d59d738cf 100644
--- a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts
+++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts
@@ -100,6 +100,7 @@ describe('AwsS3EntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = AwsS3EntityProvider.fromConfig(config, {
diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts
index 1c2c3d0082..c57455d8d6 100644
--- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.test.ts
@@ -71,6 +71,7 @@ describe('AzureDevOpsEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = AzureDevOpsEntityProvider.fromConfig(config, {
diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts
index 08b127f69e..641150acbf 100644
--- a/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-bitbucket-cloud/src/BitbucketCloudEntityProvider.test.ts
@@ -127,6 +127,7 @@ describe('BitbucketCloudEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketCloudEntityProvider.fromConfig(config, {
       logger,
diff --git a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts
index fe2c81b9b6..39689b6c1d 100644
--- a/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts
@@ -221,6 +221,7 @@ describe('BitbucketServerEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketServerEntityProvider.fromConfig(config, {
       logger,
@@ -296,6 +297,7 @@ describe('BitbucketServerEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = BitbucketServerEntityProvider.fromConfig(config, {
       logger,
diff --git a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts
index d5be045e6c..d6ad0ff668 100644
--- a/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-gerrit/src/providers/GerritEntityProvider.test.ts
@@ -83,6 +83,7 @@ describe('GerritEntityProvider', () => {
 
   const entityProviderConnection: EntityProviderConnection = {
     applyMutation: jest.fn(),
+    refresh: jest.fn(),
   };
 
   it('discovers projects from the api.', async () => {
diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
index 871718cc05..63fb77288b 100644
--- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
@@ -146,6 +146,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -233,6 +234,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -306,6 +308,7 @@ describe('GitHubEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
 
     const provider = GitHubEntityProvider.fromConfig(config, {
@@ -404,6 +407,7 @@ it('apply full update on scheduled execution with topic exclusion taking priorit
   const schedule = new PersistingTaskRunner();
   const entityProviderConnection: EntityProviderConnection = {
     applyMutation: jest.fn(),
+    refresh: jest.fn(),
   };
 
   const provider = GitHubEntityProvider.fromConfig(config, {
diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts
index 77d585eae1..2d05346f01 100644
--- a/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts
@@ -83,6 +83,7 @@ describe('GitHubOrgEntityProvider', () => {
 
       const entityProviderConnection: EntityProviderConnection = {
         applyMutation: jest.fn(),
+        refresh: jest.fn(),
       };
 
       const logger = getVoidLogger();
diff --git a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
index 7fb4bf8d09..3f3c4e7f91 100644
--- a/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
@@ -155,6 +155,7 @@ describe('GitlabDiscoveryEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
@@ -253,6 +254,7 @@ describe('GitlabDiscoveryEntityProvider', () => {
     const schedule = new PersistingTaskRunner();
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = GitlabDiscoveryEntityProvider.fromConfig(config, {
       logger,
diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts
index e34eee0499..ff68235c4c 100644
--- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts
+++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts
@@ -95,6 +95,7 @@ describe('MicrosoftGraphOrgEntityProvider', () => {
     };
     const entityProviderConnection: EntityProviderConnection = {
       applyMutation: jest.fn(),
+      refresh: jest.fn(),
     };
     const provider = MicrosoftGraphOrgEntityProvider.fromConfig(
       new ConfigReader(config),
diff --git a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts
index dfbc37f6c9..2a9b1f8d65 100644
--- a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts
+++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts
@@ -26,7 +26,7 @@ describe('DefaultLocationStore', () => {
   async function createLocationStore(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);
     await applyDatabaseMigrations(knex);
-    const connection = { applyMutation: jest.fn() };
+    const connection = { applyMutation: jest.fn(), refresh: jest.fn() };
     const store = new DefaultLocationStore(knex);
     await store.connect(connection);
     return { store, connection };
diff --git a/plugins/catalog-node/src/api/provider.ts b/plugins/catalog-node/src/api/provider.ts
index 437da4e7d6..a29df7bcba 100644
--- a/plugins/catalog-node/src/api/provider.ts
+++ b/plugins/catalog-node/src/api/provider.ts
@@ -36,6 +36,10 @@ export interface EntityProviderConnection {
    * Applies either a full or delta update to the catalog engine.
    */
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
+
+  /**
+   * Schedules a refresh on all of the entities that has a matching refresh key associated with the provided keys.
+   */
   refresh(keys: string[]): Promise<void>;
 }
 

From fabd576219bea5086293b4a058941ff7ce27f722 Mon Sep 17 00:00:00 2001
From: Kiss Miklos <miklos@roadie.io>
Date: Wed, 7 Sep 2022 14:21:35 +0200
Subject: [PATCH 4/5] add new api-report.md

Signed-off-by: Kiss Miklos <miklos@roadie.io>
---
 plugins/catalog-node/api-report.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/catalog-node/api-report.md b/plugins/catalog-node/api-report.md
index 7d12c2e065..7ffae771ef 100644
--- a/plugins/catalog-node/api-report.md
+++ b/plugins/catalog-node/api-report.md
@@ -126,6 +126,7 @@ export interface EntityProvider {
 // @public
 export interface EntityProviderConnection {
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
+  refresh(keys: string[]): Promise<void>;
 }
 
 // @public

From 9743bc788cd71d2cca37b5b8f5f6f86ab36be79b Mon Sep 17 00:00:00 2001
From: Kiss Miklos <miklos@roadie.io>
Date: Wed, 7 Sep 2022 17:16:54 +0200
Subject: [PATCH 5/5] use object as param in refresh function

Signed-off-by: Kiss Miklos <miklos@roadie.io>
---
 .changeset/flat-crabs-love.md                            | 5 +++++
 .changeset/rude-weeks-retire.md                          | 2 +-
 .../src/processing/connectEntityProviders.ts             | 5 +++--
 plugins/catalog-node/api-report.md                       | 7 ++++++-
 plugins/catalog-node/src/api/index.ts                    | 1 +
 plugins/catalog-node/src/api/provider.ts                 | 9 ++++++++-
 6 files changed, 24 insertions(+), 5 deletions(-)
 create mode 100644 .changeset/flat-crabs-love.md

diff --git a/.changeset/flat-crabs-love.md b/.changeset/flat-crabs-love.md
new file mode 100644
index 0000000000..96f566d712
--- /dev/null
+++ b/.changeset/flat-crabs-love.md
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-node': minor
+---
+
+Added refresh function to the `EntityProviderConnection` to be able to schedule refreshes from entity providers.
diff --git a/.changeset/rude-weeks-retire.md b/.changeset/rude-weeks-retire.md
index fddddd7a9c..c2073c5826 100644
--- a/.changeset/rude-weeks-retire.md
+++ b/.changeset/rude-weeks-retire.md
@@ -2,4 +2,4 @@
 '@backstage/plugin-catalog-backend': minor
 ---
 
-Added refresh function to the EntityProviderConnection to be able to schedule refreshes from entity providers.
+Added the `refresh` function to the Connection of the entity providers.
diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts
index 8325e2bcac..a4795532f0 100644
--- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts
+++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts
@@ -22,6 +22,7 @@ import { ProcessingDatabase } from '../database/types';
 import {
   EntityProvider,
   EntityProviderConnection,
+  EntityProviderRefreshOptions,
   EntityProviderMutation,
 } from '@backstage/plugin-catalog-node';
 
@@ -61,12 +62,12 @@ class Connection implements EntityProviderConnection {
     }
   }
 
-  async refresh(keys: string[]): Promise<void> {
+  async refresh(options: EntityProviderRefreshOptions): Promise<void> {
     const db = this.config.processingDatabase;
 
     await db.transaction(async (tx: any) => {
       return db.refreshByRefreshKeys(tx, {
-        keys,
+        keys: options.keys,
       });
     });
   }
diff --git a/plugins/catalog-node/api-report.md b/plugins/catalog-node/api-report.md
index 7ffae771ef..3873e3376d 100644
--- a/plugins/catalog-node/api-report.md
+++ b/plugins/catalog-node/api-report.md
@@ -126,7 +126,7 @@ export interface EntityProvider {
 // @public
 export interface EntityProviderConnection {
   applyMutation(mutation: EntityProviderMutation): Promise<void>;
-  refresh(keys: string[]): Promise<void>;
+  refresh(options: EntityProviderRefreshOptions): Promise<void>;
 }
 
 // @public
@@ -141,6 +141,11 @@ export type EntityProviderMutation =
       removed: DeferredEntity[];
     };
 
+// @public (undocumented)
+export type EntityProviderRefreshOptions = {
+  keys: string[];
+};
+
 // @public
 export type EntityRelationSpec = {
   source: CompoundEntityRef;
diff --git a/plugins/catalog-node/src/api/index.ts b/plugins/catalog-node/src/api/index.ts
index 136c23d20d..66e8d6bd21 100644
--- a/plugins/catalog-node/src/api/index.ts
+++ b/plugins/catalog-node/src/api/index.ts
@@ -32,4 +32,5 @@ export type {
   EntityProvider,
   EntityProviderConnection,
   EntityProviderMutation,
+  EntityProviderRefreshOptions,
 } from './provider';
diff --git a/plugins/catalog-node/src/api/provider.ts b/plugins/catalog-node/src/api/provider.ts
index a29df7bcba..fc9a172e0d 100644
--- a/plugins/catalog-node/src/api/provider.ts
+++ b/plugins/catalog-node/src/api/provider.ts
@@ -26,6 +26,13 @@ export type EntityProviderMutation =
   | { type: 'full'; entities: DeferredEntity[] }
   | { type: 'delta'; added: DeferredEntity[]; removed: DeferredEntity[] };
 
+/**
+ * @public
+ */
+export type EntityProviderRefreshOptions = {
+  keys: string[];
+};
+
 /**
  * The EntityProviderConnection is the connection between the catalog and the entity provider.
  * The EntityProvider use this connection to add and remove entities from the catalog.
@@ -40,7 +47,7 @@ export interface EntityProviderConnection {
   /**
    * Schedules a refresh on all of the entities that has a matching refresh key associated with the provided keys.
    */
-  refresh(keys: string[]): Promise<void>;
+  refresh(options: EntityProviderRefreshOptions): Promise<void>;
 }
 
 /**