feat(catalog/github): Add option to configure schedule via app-config.yaml

Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>

plugins/catalog-backend-module-github/api-report.md

@@ -16,6 +16,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
 import { TaskRunner } from '@backstage/backend-tasks';
@@ -55,7 +56,8 @@ export class GitHubEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): GitHubEntityProvider[];
   // (undocumented)

plugins/catalog-backend-module-github/config.d.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
+
 export interface Config {
   catalog?: {
     processors?: {
@@ -103,6 +105,10 @@ export interface Config {
                 exclude?: string[];
               };
             };
+            /**
+             * (Optional) TaskScheduleDefinition for the refresh.
+             */
+            schedule?: TaskScheduleDefinitionConfig;
           }
         | Record<
             string,
@@ -156,6 +162,10 @@ export interface Config {
                   exclude?: string[];
                 };
               };
+              /**
+               * (Optional) TaskScheduleDefinition for the refresh.
+               */
+              schedule?: TaskScheduleDefinitionConfig;
             }
           >;
     };

plugins/catalog-backend-module-github/package.json

@@ -56,7 +56,8 @@
   "devDependencies": {
     "@backstage/backend-test-utils": "workspace:^",
     "@backstage/cli": "workspace:^",
-    "@types/lodash": "^4.14.151"
+    "@types/lodash": "^4.14.151",
+    "luxon": "^3.0.0"
   },
   "files": [
     "dist",

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts

@@ -15,7 +15,11 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
-import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
+import {
+  PluginTaskScheduler,
+  TaskInvocationDefinition,
+  TaskRunner,
+} from '@backstage/backend-tasks';
 import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GitHubEntityProvider } from './GitHubEntityProvider';
@@ -381,132 +385,201 @@ describe('GitHubEntityProvider', () => {
       entities: expectedEntities,
     });
   });
-});
 
-it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
-  const config = new ConfigReader({
-    catalog: {
-      providers: {
-        github: {
-          myProvider: {
-            organization: 'test-org',
-            catalogPath: 'custom/path/catalog-custom.yaml',
-            filters: {
-              branch: 'main',
-              repository: 'test-.*',
-              topic: {
-                exclude: ['backstage-exclude'],
-                include: ['backstage-include'],
+  it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            myProvider: {
+              organization: 'test-org',
+              catalogPath: 'custom/path/catalog-custom.yaml',
+              filters: {
+                branch: 'main',
+                repository: 'test-.*',
+                topic: {
+                  exclude: ['backstage-exclude'],
+                  include: ['backstage-include'],
+                },
               },
             },
           },
         },
       },
-    },
+    });
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+
+    const provider = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const mockGetOrganizationRepositories = jest.spyOn(
+      helpers,
+      'getOrganizationRepositories',
+    );
+
+    mockGetOrganizationRepositories.mockReturnValue(
+      Promise.resolve({
+        repositories: [
+          {
+            name: 'test-repo',
+            url: 'https://github.com/test-org/test-repo',
+            repositoryTopics: {
+              nodes: [
+                {
+                  topic: { name: 'backstage-include' },
+                },
+              ],
+            },
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+          },
+          {
+            name: 'test-repo-2',
+            url: 'https://github.com/test-org/test-repo-2',
+            repositoryTopics: {
+              nodes: [
+                {
+                  topic: { name: 'backstage-include' },
+                },
+                {
+                  topic: { name: 'backstage-exclude' },
+                },
+              ],
+            },
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+          },
+          {
+            name: 'test-repo-3',
+            url: 'https://github.com/test-org/test-repo-3',
+            repositoryTopics: {
+              nodes: [
+                {
+                  topic: { name: 'backstage-exclude' },
+                },
+              ],
+            },
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+          },
+        ],
+      }),
+    );
+
+    await provider.connect(entityProviderConnection);
+
+    const taskDef = schedule.getTasks()[0];
+    expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
+    await (taskDef.fn as () => Promise<void>)();
+
+    const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:myProvider',
+      },
+    ];
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: expectedEntities,
+    });
   });
-  const schedule = new PersistingTaskRunner();
-  const entityProviderConnection: EntityProviderConnection = {
-    applyMutation: jest.fn(),
-    refresh: jest.fn(),
-  };
 
-  const provider = GitHubEntityProvider.fromConfig(config, {
-    logger,
-    schedule,
-  })[0];
-
-  const mockGetOrganizationRepositories = jest.spyOn(
-    helpers,
-    'getOrganizationRepositories',
-  );
-
-  mockGetOrganizationRepositories.mockReturnValue(
-    Promise.resolve({
-      repositories: [
-        {
-          name: 'test-repo',
-          url: 'https://github.com/test-org/test-repo',
-          repositoryTopics: {
-            nodes: [
-              {
-                topic: { name: 'backstage-include' },
-              },
-            ],
+  it('fail without schedule and scheduler', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
           },
-          isArchived: false,
-          defaultBranchRef: {
-            name: 'main',
-          },
-        },
-        {
-          name: 'test-repo-2',
-          url: 'https://github.com/test-org/test-repo-2',
-          repositoryTopics: {
-            nodes: [
-              {
-                topic: { name: 'backstage-include' },
-              },
-              {
-                topic: { name: 'backstage-exclude' },
-              },
-            ],
-          },
-          isArchived: false,
-          defaultBranchRef: {
-            name: 'main',
-          },
-        },
-        {
-          name: 'test-repo-3',
-          url: 'https://github.com/test-org/test-repo-3',
-          repositoryTopics: {
-            nodes: [
-              {
-                topic: { name: 'backstage-exclude' },
-              },
-            ],
-          },
-          isArchived: false,
-          defaultBranchRef: {
-            name: 'main',
-          },
-        },
-      ],
-    }),
-  );
-
-  await provider.connect(entityProviderConnection);
-
-  const taskDef = schedule.getTasks()[0];
-  expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
-  await (taskDef.fn as () => Promise<void>)();
-
-  const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
-  const expectedEntities = [
-    {
-      entity: {
-        apiVersion: 'backstage.io/v1alpha1',
-        kind: 'Location',
-        metadata: {
-          annotations: {
-            'backstage.io/managed-by-location': `url:${url}`,
-            'backstage.io/managed-by-origin-location': `url:${url}`,
-          },
-          name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
-        },
-        spec: {
-          presence: 'optional',
-          target: `${url}`,
-          type: 'url',
         },
       },
-      locationKey: 'github-provider:myProvider',
-    },
-  ];
+    });
 
-  expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
-  expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
-    type: 'full',
-    entities: expectedEntities,
+    expect(() =>
+      GitHubEntityProvider.fromConfig(config, {
+        logger,
+      }),
+    ).toThrow('Either schedule or scheduler must be provided');
+  });
+
+  it('fail with scheduler but no schedule config', () => {
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => jest.fn(),
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    expect(() =>
+      GitHubEntityProvider.fromConfig(config, {
+        logger,
+        scheduler,
+      }),
+    ).toThrow(
+      'No schedule provided neither via code nor config for github-provider:default',
+    );
+  });
+
+  it('single simple provider config with schedule in config', async () => {
+    const schedule = new PersistingTaskRunner();
+    const scheduler = {
+      createScheduledTaskRunner: (_: any) => schedule,
+    } as unknown as PluginTaskScheduler;
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+            schedule: {
+              frequency: 'P1M',
+              timeout: 'PT3M',
+            },
+          },
+        },
+      },
+    });
+    const providers = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      scheduler,
+    });
+
+    expect(providers).toHaveLength(1);
+    expect(providers[0].getProviderName()).toEqual('github-provider:default');
   });
 });

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { TaskRunner } from '@backstage/backend-tasks';
+import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 import {
   GithubCredentialsProvider,
@@ -60,9 +60,14 @@ export class GitHubEntityProvider implements EntityProvider {
     config: Config,
     options: {
       logger: Logger;
-      schedule: TaskRunner;
+      schedule?: TaskRunner;
+      scheduler?: PluginTaskScheduler;
     },
   ): GitHubEntityProvider[] {
+    if (!options.schedule && !options.scheduler) {
+      throw new Error('Either schedule or scheduler must be provided.');
+    }
+
     const integrations = ScmIntegrations.fromConfig(config);
 
     return readProviderConfigs(config).map(providerConfig => {
@@ -75,11 +80,21 @@ export class GitHubEntityProvider implements EntityProvider {
         );
       }
 
+      if (!options.schedule && !providerConfig.schedule) {
+        throw new Error(
+          `No schedule provided neither via code nor config for github-provider:${providerConfig.id}.`,
+        );
+      }
+
+      const taskRunner =
+        options.schedule ??
+        options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
+
       return new GitHubEntityProvider(
         providerConfig,
         integration,
         options.logger,
-        options.schedule,
+        taskRunner,
       );
     });
   }
@@ -88,14 +103,14 @@ export class GitHubEntityProvider implements EntityProvider {
     config: GitHubEntityProviderConfig,
     integration: GitHubIntegration,
     logger: Logger,
-    schedule: TaskRunner,
+    taskRunner: TaskRunner,
   ) {
     this.config = config;
     this.integration = integration.config;
     this.logger = logger.child({
       target: this.getProviderName(),
     });
-    this.scheduleFn = this.createScheduleFn(schedule);
+    this.scheduleFn = this.createScheduleFn(taskRunner);
     this.githubCredentialsProvider =
       SingleInstanceGithubCredentialsProvider.create(integration.config);
   }
@@ -111,10 +126,10 @@ export class GitHubEntityProvider implements EntityProvider {
     return await this.scheduleFn();
   }
 
-  private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
+  private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
     return async () => {
       const taskId = `${this.getProviderName()}:refresh`;
-      return schedule.run({
+      return taskRunner.run({
         id: taskId,
         fn: async () => {
           const logger = this.logger.child({

plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import { Duration } from 'luxon';
 import { readProviderConfigs } from './GitHubEntityProviderConfig';
 
 describe('readProviderConfigs', () => {
@@ -81,13 +82,22 @@ describe('readProviderConfigs', () => {
               organization: 'test-org1',
               host: 'ghe.internal.com',
             },
+            providerWithSchedule: {
+              organization: 'test-org1',
+              schedule: {
+                frequency: 'PT30M',
+                timeout: {
+                  minutes: 3,
+                },
+              },
+            },
           },
         },
       },
     });
     const providerConfigs = readProviderConfigs(config);
 
-    expect(providerConfigs).toHaveLength(6);
+    expect(providerConfigs).toHaveLength(7);
     expect(providerConfigs[0]).toEqual({
       id: 'providerOrganizationOnly',
       organization: 'test-org1',
@@ -101,6 +111,7 @@ describe('readProviderConfigs', () => {
           exclude: undefined,
         },
       },
+      schedule: undefined,
     });
     expect(providerConfigs[1]).toEqual({
       id: 'providerCustomCatalogPath',
@@ -115,6 +126,7 @@ describe('readProviderConfigs', () => {
           exclude: undefined,
         },
       },
+      schedule: undefined,
     });
     expect(providerConfigs[2]).toEqual({
       id: 'providerWithRepositoryFilter',
@@ -129,6 +141,7 @@ describe('readProviderConfigs', () => {
           exclude: undefined,
         },
       },
+      schedule: undefined,
     });
     expect(providerConfigs[3]).toEqual({
       id: 'providerWithBranchFilter',
@@ -143,6 +156,7 @@ describe('readProviderConfigs', () => {
           exclude: undefined,
         },
       },
+      schedule: undefined,
     });
     expect(providerConfigs[4]).toEqual({
       id: 'providerWithTopicFilter',
@@ -157,6 +171,7 @@ describe('readProviderConfigs', () => {
           exclude: ['backstage-exclude'],
         },
       },
+      schedule: undefined,
     });
     expect(providerConfigs[5]).toEqual({
       id: 'providerWithHost',
@@ -171,6 +186,27 @@ describe('readProviderConfigs', () => {
           exclude: undefined,
         },
       },
+      schedule: undefined,
+    });
+    expect(providerConfigs[6]).toEqual({
+      id: 'providerWithSchedule',
+      organization: 'test-org1',
+      catalogPath: '/catalog-info.yaml',
+      host: 'github.com',
+      filters: {
+        repository: undefined,
+        branch: undefined,
+        topic: {
+          include: undefined,
+          exclude: undefined,
+        },
+      },
+      schedule: {
+        frequency: Duration.fromISO('PT30M'),
+        timeout: {
+          minutes: 3,
+        },
+      },
     });
   });
 });

plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+import {
+  readTaskScheduleDefinitionFromConfig,
+  TaskScheduleDefinition,
+} from '@backstage/backend-tasks';
 import { Config } from '@backstage/config';
 
 const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
@@ -29,6 +33,7 @@ export type GitHubEntityProviderConfig = {
     branch?: string;
     topic?: GithubTopicFilters;
   };
+  schedule?: TaskScheduleDefinition;
 };
 
 export type GithubTopicFilters = {
@@ -73,6 +78,10 @@ function readProviderConfig(
     'filters.topic.exclude',
   );
 
+  const schedule = config.has('schedule')
+    ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
+    : undefined;
+
   return {
     id,
     catalogPath,
@@ -88,8 +97,10 @@ function readProviderConfig(
         exclude: topicFilterExclude,
       },
     },
+    schedule,
   };
 }
+
 /**
  * Compiles a RegExp while enforcing the pattern to contain
  * the start-of-line and end-of-line anchors.