paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(catalog/github): Add option to configure schedule via app-config.yaml

Browse Source 
Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-10-07 13:56:34 +02:00
parent 09dc3083a6
commit 8749df3d02
10 changed files with 309 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/famous-ligers-repair.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
`GitHubEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
Please find how to configure the schedule at the config at
https://backstage.io/docs/integrations/github/discovery
docs/integrations/github/discovery.md
+26 -9
View File
@@ -39,10 +39,13 @@ And then add the entity provider to your catalog builder:
+ builder.addEntityProvider(
+ GitHubEntityProvider.fromConfig(env.config, {
+ logger: env.logger,
+ // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+ schedule: env.scheduler.createScheduledTaskRunner({
+ frequency: { minutes: 30 },
+ timeout: { minutes: 3 },
+ }),
+ // optional: alternatively, use schedule
+ scheduler: env.scheduler,
+ }),
+ );
@@ -68,6 +71,11 @@ catalog:
filters:
branch: 'main' # string
repository: '.*' # Regex
schedule: # optional; same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
customProviderId:
organization: 'new-org' # string
catalogPath: '/custom/path/catalog-info.yaml' # string
@@ -111,26 +119,35 @@ This provider supports multiple organizations via unique provider IDs.
Default: `/catalog-info.yaml`.
Path where to look for `catalog-info.yaml` files.
You can use wildcards - `*` or `**` - to search the path and/or the filename
- **filters** _(optional)_:
- **branch** _(optional)_:
- **`filters`** _(optional)_:
- **`branch`** _(optional)_:
String used to filter results based on the branch name.
- **repository** _(optional)_:
- **`repository`** _(optional)_:
Regular expression used to filter results based on the repository name.
- **topic** _(optional)_:
- **`topic`** _(optional)_:
Both of the filters below may be used at the same time but the exclusion filter has the highest priority.
In the example above, a repository with the `backstage-include` topic would still be excluded
if it were also carrying the `experiments` topic.
- **include** _(optional)_:
- **`include`** _(optional)_:
An array of strings used to filter in results based on their associated GitHub topics.
If configured, only repositories with one (or more) topic(s) present in the inclusion filter will be ingested
- **exclude** _(optional)_:
- **`exclude`** _(optional)_:
An array of strings used to filter out results based on their associated GitHub topics.
If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested.
- **organization**:
- **`host`** _(optional)_:
The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md).
- **`organization`**:
Name of your organization account/workspace.
If you want to add multiple organizations, you need to add one provider config each.
- **host** _(optional)_:
The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md).
- **`schedule`** _(optional)_:
- **`frequency`**:
How often you want the task to run. The system does its best to avoid overlapping invocations.
- **`timeout`**:
The maximum amount of time that a single task invocation can take.
- **`initialDelay`** _(optional)_:
The amount of time that should pass before the first invocation happens.
- **`scope`** _(optional)_:
`'global'` or `'local'`. Sets the scope of concurrency control.
## GitHub API Rate Limits
plugins/catalog-backend-module-github/api-report.md
+3 -1
View File
@@ -16,6 +16,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { ScmIntegrationRegistry } from '@backstage/integration';
import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
import { TaskRunner } from '@backstage/backend-tasks';
@@ -55,7 +56,8 @@ export class GitHubEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitHubEntityProvider[];
// (undocumented)
plugins/catalog-backend-module-github/config.d.ts
Vendored
+10
View File
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
catalog?: {
processors?: {
@@ -103,6 +105,10 @@ export interface Config {
exclude?: string[];
};
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
| Record<
string,
@@ -156,6 +162,10 @@ export interface Config {
exclude?: string[];
};
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
>;
};
plugins/catalog-backend-module-github/package.json
+2 -1
View File
@@ -56,7 +56,8 @@
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151"
"@types/lodash": "^4.14.151",
"luxon": "^3.0.0"
},
"files": [
"dist",
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
+189 -116
View File
@@ -15,7 +15,11 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
TaskRunner,
} from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GitHubEntityProvider } from './GitHubEntityProvider';
@@ -381,132 +385,201 @@ describe('GitHubEntityProvider', () => {
entities: expectedEntities,
});
});
});
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
expect(() =>
GitHubEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
expect(() =>
GitHubEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for github-provider:default',
);
});
it('single simple provider config with schedule in config', async () => {
const schedule = new PersistingTaskRunner();
const scheduler = {
createScheduledTaskRunner: (_: any) => schedule,
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
},
},
},
});
const providers = GitHubEntityProvider.fromConfig(config, {
logger,
scheduler,
});
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual('github-provider:default');
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts
+22 -7
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import {
GithubCredentialsProvider,
@@ -60,9 +60,14 @@ export class GitHubEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitHubEntityProvider[] {
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
const integrations = ScmIntegrations.fromConfig(config);
return readProviderConfigs(config).map(providerConfig => {
@@ -75,11 +80,21 @@ export class GitHubEntityProvider implements EntityProvider {
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for github-provider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
return new GitHubEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
taskRunner,
);
});
}
@@ -88,14 +103,14 @@ export class GitHubEntityProvider implements EntityProvider {
config: GitHubEntityProviderConfig,
integration: GitHubIntegration,
logger: Logger,
schedule: TaskRunner,
taskRunner: TaskRunner,
) {
this.config = config;
this.integration = integration.config;
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
this.scheduleFn = this.createScheduleFn(taskRunner);
this.githubCredentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
}
@@ -111,10 +126,10 @@ export class GitHubEntityProvider implements EntityProvider {
return await this.scheduleFn();
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
return taskRunner.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts
+37 -1
View File
@@ -15,6 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readProviderConfigs } from './GitHubEntityProviderConfig';
describe('readProviderConfigs', () => {
@@ -81,13 +82,22 @@ describe('readProviderConfigs', () => {
organization: 'test-org1',
host: 'ghe.internal.com',
},
providerWithSchedule: {
organization: 'test-org1',
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(6);
expect(providerConfigs).toHaveLength(7);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -101,6 +111,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[1]).toEqual({
id: 'providerCustomCatalogPath',
@@ -115,6 +126,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[2]).toEqual({
id: 'providerWithRepositoryFilter',
@@ -129,6 +141,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[3]).toEqual({
id: 'providerWithBranchFilter',
@@ -143,6 +156,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[4]).toEqual({
id: 'providerWithTopicFilter',
@@ -157,6 +171,7 @@ describe('readProviderConfigs', () => {
exclude: ['backstage-exclude'],
},
},
schedule: undefined,
});
expect(providerConfigs[5]).toEqual({
id: 'providerWithHost',
@@ -171,6 +186,27 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[6]).toEqual({
id: 'providerWithSchedule',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
schedule: {
frequency: Duration.fromISO('PT30M'),
timeout: {
minutes: 3,
},
},
});
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts
+11
View File
@@ -14,6 +14,10 @@
* limitations under the License.
*/
import {
readTaskScheduleDefinitionFromConfig,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
@@ -29,6 +33,7 @@ export type GitHubEntityProviderConfig = {
branch?: string;
topic?: GithubTopicFilters;
};
schedule?: TaskScheduleDefinition;
};
export type GithubTopicFilters = {
@@ -73,6 +78,10 @@ function readProviderConfig(
'filters.topic.exclude',
);
const schedule = config.has('schedule')
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
: undefined;
return {
id,
catalogPath,
@@ -88,8 +97,10 @@ function readProviderConfig(
exclude: topicFilterExclude,
},
},
schedule,
};
}
/**
* Compiles a RegExp while enforcing the pattern to contain
* the start-of-line and end-of-line anchors.
yarn.lock
+1
View File
@@ -4590,6 +4590,7 @@ __metadata:
"@types/lodash": ^4.14.151
git-url-parse: ^13.0.0
lodash: ^4.17.21
luxon: ^3.0.0
msw: ^0.47.0
node-fetch: ^2.6.7
uuid: ^8.0.0