diff --git a/.changeset/famous-ligers-repair.md b/.changeset/famous-ligers-repair.md new file mode 100644 index 0000000000..bc3780cba7 --- /dev/null +++ b/.changeset/famous-ligers-repair.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +`GitHubEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code. + +Please find how to configure the schedule at the config at +https://backstage.io/docs/integrations/github/discovery diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 9b60b41682..26cd660cde 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -39,10 +39,13 @@ And then add the entity provider to your catalog builder: + builder.addEntityProvider( + GitHubEntityProvider.fromConfig(env.config, { + logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, + }), + ); @@ -68,6 +71,11 @@ catalog: filters: branch: 'main' # string repository: '.*' # Regex + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } customProviderId: organization: 'new-org' # string catalogPath: '/custom/path/catalog-info.yaml' # string @@ -111,26 +119,35 @@ This provider supports multiple organizations via unique provider IDs. Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. You can use wildcards - `*` or `**` - to search the path and/or the filename -- **filters** _(optional)_: - - **branch** _(optional)_: +- **`filters`** _(optional)_: + - **`branch`** _(optional)_: String used to filter results based on the branch name. - - **repository** _(optional)_: + - **`repository`** _(optional)_: Regular expression used to filter results based on the repository name. - - **topic** _(optional)_: + - **`topic`** _(optional)_: Both of the filters below may be used at the same time but the exclusion filter has the highest priority. In the example above, a repository with the `backstage-include` topic would still be excluded if it were also carrying the `experiments` topic. - - **include** _(optional)_: + - **`include`** _(optional)_: An array of strings used to filter in results based on their associated GitHub topics. If configured, only repositories with one (or more) topic(s) present in the inclusion filter will be ingested - - **exclude** _(optional)_: + - **`exclude`** _(optional)_: An array of strings used to filter out results based on their associated GitHub topics. If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested. -- **organization**: +- **`host`** _(optional)_: + The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). +- **`organization`**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. -- **host** _(optional)_: - The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). +- **`schedule`** _(optional)_: + - **`frequency`**: + How often you want the task to run. The system does its best to avoid overlapping invocations. + - **`timeout`**: + The maximum amount of time that a single task invocation can take. + - **`initialDelay`** _(optional)_: + The amount of time that should pass before the first invocation happens. + - **`scope`** _(optional)_: + `'global'` or `'local'`. Sets the scope of concurrency control. ## GitHub API Rate Limits diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index c89a34c259..86fc94a532 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -16,6 +16,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -55,7 +56,8 @@ export class GitHubEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GitHubEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index 1dcd49ea4e..8648490001 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks'; + export interface Config { catalog?: { processors?: { @@ -103,6 +105,10 @@ export interface Config { exclude?: string[]; }; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } | Record< string, @@ -156,6 +162,10 @@ export interface Config { exclude?: string[]; }; }; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: TaskScheduleDefinitionConfig; } >; }; diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index 1fb82a648a..86433d5c73 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -56,7 +56,8 @@ "devDependencies": { "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@types/lodash": "^4.14.151" + "@types/lodash": "^4.14.151", + "luxon": "^3.0.0" }, "files": [ "dist", diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 63fb77288b..b2078172e9 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -15,7 +15,11 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskRunner, +} from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GitHubEntityProvider } from './GitHubEntityProvider'; @@ -381,132 +385,201 @@ describe('GitHubEntityProvider', () => { entities: expectedEntities, }); }); -}); -it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { - const config = new ConfigReader({ - catalog: { - providers: { - github: { - myProvider: { - organization: 'test-org', - catalogPath: 'custom/path/catalog-custom.yaml', - filters: { - branch: 'main', - repository: 'test-.*', - topic: { - exclude: ['backstage-exclude'], - include: ['backstage-include'], + it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + topic: { + exclude: ['backstage-exclude'], + include: ['backstage-include'], + }, }, }, }, }, }, - }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + { + name: 'test-repo-2', + url: 'https://github.com/test-org/test-repo-2', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + { + name: 'test-repo-3', + url: 'https://github.com/test-org/test-repo-3', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); }); - const schedule = new PersistingTaskRunner(); - const entityProviderConnection: EntityProviderConnection = { - applyMutation: jest.fn(), - refresh: jest.fn(), - }; - const provider = GitHubEntityProvider.fromConfig(config, { - logger, - schedule, - })[0]; - - const mockGetOrganizationRepositories = jest.spyOn( - helpers, - 'getOrganizationRepositories', - ); - - mockGetOrganizationRepositories.mockReturnValue( - Promise.resolve({ - repositories: [ - { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-include' }, - }, - ], + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - }, - { - name: 'test-repo-2', - url: 'https://github.com/test-org/test-repo-2', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-include' }, - }, - { - topic: { name: 'backstage-exclude' }, - }, - ], - }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - }, - { - name: 'test-repo-3', - url: 'https://github.com/test-org/test-repo-3', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-exclude' }, - }, - ], - }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - }, - ], - }), - ); - - await provider.connect(entityProviderConnection); - - const taskDef = schedule.getTasks()[0]; - expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); - await (taskDef.fn as () => Promise)(); - - const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`; - const expectedEntities = [ - { - entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - annotations: { - 'backstage.io/managed-by-location': `url:${url}`, - 'backstage.io/managed-by-origin-location': `url:${url}`, - }, - name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f', - }, - spec: { - presence: 'optional', - target: `${url}`, - type: 'url', }, }, - locationKey: 'github-provider:myProvider', - }, - ]; + }); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'full', - entities: expectedEntities, + expect(() => + GitHubEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + expect(() => + GitHubEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for github-provider:default', + ); + }); + + it('single simple provider config with schedule in config', async () => { + const schedule = new PersistingTaskRunner(); + const scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as PluginTaskScheduler; + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + scheduler, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual('github-provider:default'); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3909b5a3c2..7d04b749dd 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskRunner } from '@backstage/backend-tasks'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import { GithubCredentialsProvider, @@ -60,9 +60,14 @@ export class GitHubEntityProvider implements EntityProvider { config: Config, options: { logger: Logger; - schedule: TaskRunner; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; }, ): GitHubEntityProvider[] { + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + const integrations = ScmIntegrations.fromConfig(config); return readProviderConfigs(config).map(providerConfig => { @@ -75,11 +80,21 @@ export class GitHubEntityProvider implements EntityProvider { ); } + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for github-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + return new GitHubEntityProvider( providerConfig, integration, options.logger, - options.schedule, + taskRunner, ); }); } @@ -88,14 +103,14 @@ export class GitHubEntityProvider implements EntityProvider { config: GitHubEntityProviderConfig, integration: GitHubIntegration, logger: Logger, - schedule: TaskRunner, + taskRunner: TaskRunner, ) { this.config = config; this.integration = integration.config; this.logger = logger.child({ target: this.getProviderName(), }); - this.scheduleFn = this.createScheduleFn(schedule); + this.scheduleFn = this.createScheduleFn(taskRunner); this.githubCredentialsProvider = SingleInstanceGithubCredentialsProvider.create(integration.config); } @@ -111,10 +126,10 @@ export class GitHubEntityProvider implements EntityProvider { return await this.scheduleFn(); } - private createScheduleFn(schedule: TaskRunner): () => Promise { + private createScheduleFn(taskRunner: TaskRunner): () => Promise { return async () => { const taskId = `${this.getProviderName()}:refresh`; - return schedule.run({ + return taskRunner.run({ id: taskId, fn: async () => { const logger = this.logger.child({ diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index 66a5463bf5..e1b3a6cfbf 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; import { readProviderConfigs } from './GitHubEntityProviderConfig'; describe('readProviderConfigs', () => { @@ -81,13 +82,22 @@ describe('readProviderConfigs', () => { organization: 'test-org1', host: 'ghe.internal.com', }, + providerWithSchedule: { + organization: 'test-org1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(6); + expect(providerConfigs).toHaveLength(7); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -101,6 +111,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + schedule: undefined, }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -115,6 +126,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + schedule: undefined, }); expect(providerConfigs[2]).toEqual({ id: 'providerWithRepositoryFilter', @@ -129,6 +141,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + schedule: undefined, }); expect(providerConfigs[3]).toEqual({ id: 'providerWithBranchFilter', @@ -143,6 +156,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + schedule: undefined, }); expect(providerConfigs[4]).toEqual({ id: 'providerWithTopicFilter', @@ -157,6 +171,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, }, + schedule: undefined, }); expect(providerConfigs[5]).toEqual({ id: 'providerWithHost', @@ -171,6 +186,27 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + schedule: undefined, + }); + expect(providerConfigs[6]).toEqual({ + id: 'providerWithSchedule', + organization: 'test-org1', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + topic: { + include: undefined, + exclude: undefined, + }, + }, + schedule: { + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 3, + }, + }, }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index d40b206751..6785d18984 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; @@ -29,6 +33,7 @@ export type GitHubEntityProviderConfig = { branch?: string; topic?: GithubTopicFilters; }; + schedule?: TaskScheduleDefinition; }; export type GithubTopicFilters = { @@ -73,6 +78,10 @@ function readProviderConfig( 'filters.topic.exclude', ); + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + return { id, catalogPath, @@ -88,8 +97,10 @@ function readProviderConfig( exclude: topicFilterExclude, }, }, + schedule, }; } + /** * Compiles a RegExp while enforcing the pattern to contain * the start-of-line and end-of-line anchors. diff --git a/yarn.lock b/yarn.lock index 8be2e5c8a6..06bac843bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4590,6 +4590,7 @@ __metadata: "@types/lodash": ^4.14.151 git-url-parse: ^13.0.0 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.47.0 node-fetch: ^2.6.7 uuid: ^8.0.0