paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

Merge pull request #14039 from Bonial-International-GmbH/pjungermann/github/schedule-in-config

Browse Source 
feat(catalog/github): schedule via config + use at backend module
This commit is contained in:
Fredrik Adelöw
2022-10-12 10:01:19 +01:00
committed by GitHub
parent 716e2490d8 51046b58b0
commit a593009d42
14 changed files with 412 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend-module-github/api-report.md
+4 -8
View File
@@ -16,10 +16,10 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { ScmIntegrationRegistry } from '@backstage/integration';
import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
import { TaskRunner } from '@backstage/backend-tasks';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
// @public
export class GithubDiscoveryProcessor implements CatalogProcessor {
@@ -55,7 +55,8 @@ export class GitHubEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitHubEntityProvider[];
// (undocumented)
@@ -66,14 +67,9 @@ export class GitHubEntityProvider implements EntityProvider {
// @alpha
export const githubEntityProviderCatalogModule: (
options?: GithubEntityProviderCatalogModuleOptions | undefined,
options?: undefined,
) => BackendFeature;
// @alpha
export type GithubEntityProviderCatalogModuleOptions = {
schedule?: TaskScheduleDefinition;
};
// @public (undocumented)
export class GitHubLocationAnalyzer implements ScmLocationAnalyzer {
constructor(options: GitHubLocationAnalyzerOptions);
plugins/catalog-backend-module-github/config.d.ts
Vendored
+10
View File
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
catalog?: {
processors?: {
@@ -103,6 +105,10 @@ export interface Config {
exclude?: string[];
};
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
| Record<
string,
@@ -156,6 +162,10 @@ export interface Config {
exclude?: string[];
};
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
>;
};
plugins/catalog-backend-module-github/package.json
+2 -1
View File
@@ -56,7 +56,8 @@
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151"
"@types/lodash": "^4.14.151",
"luxon": "^3.0.0"
},
"files": [
"dist",
plugins/catalog-backend-module-github/src/index.ts
+4 -5
View File
@@ -20,14 +20,13 @@
* @packageDocumentation
*/
export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer';
export type { GitHubLocationAnalyzerOptions } from './analyzers/GitHubLocationAnalyzer';
export type { GithubMultiOrgConfig } from './lib';
export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
export { GitHubEntityProvider } from './providers/GitHubEntityProvider';
export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider';
export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider';
export type { GithubMultiOrgConfig } from './lib';
export { githubEntityProviderCatalogModule } from './module';
export type { GithubEntityProviderCatalogModuleOptions } from './module';
export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer';
export type { GitHubLocationAnalyzerOptions } from './analyzers/GitHubLocationAnalyzer';
export { githubEntityProviderCatalogModule } from './service/GithubEntityProviderCatalogModule';
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
+189 -116
View File
@@ -15,7 +15,11 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
TaskRunner,
} from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GitHubEntityProvider } from './GitHubEntityProvider';
@@ -381,132 +385,201 @@ describe('GitHubEntityProvider', () => {
entities: expectedEntities,
});
});
});
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
expect(() =>
GitHubEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
expect(() =>
GitHubEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for github-provider:default',
);
});
it('single simple provider config with schedule in config', async () => {
const schedule = new PersistingTaskRunner();
const scheduler = {
createScheduledTaskRunner: (_: any) => schedule,
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
},
},
},
});
const providers = GitHubEntityProvider.fromConfig(config, {
logger,
scheduler,
});
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual('github-provider:default');
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts
+22 -7
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import {
GithubCredentialsProvider,
@@ -60,9 +60,14 @@ export class GitHubEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitHubEntityProvider[] {
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
const integrations = ScmIntegrations.fromConfig(config);
return readProviderConfigs(config).map(providerConfig => {
@@ -75,11 +80,21 @@ export class GitHubEntityProvider implements EntityProvider {
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for github-provider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
return new GitHubEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
taskRunner,
);
});
}
@@ -88,14 +103,14 @@ export class GitHubEntityProvider implements EntityProvider {
config: GitHubEntityProviderConfig,
integration: GitHubIntegration,
logger: Logger,
schedule: TaskRunner,
taskRunner: TaskRunner,
) {
this.config = config;
this.integration = integration.config;
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
this.scheduleFn = this.createScheduleFn(taskRunner);
this.githubCredentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
}
@@ -111,10 +126,10 @@ export class GitHubEntityProvider implements EntityProvider {
return await this.scheduleFn();
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
return taskRunner.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts
+37 -1
View File
@@ -15,6 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readProviderConfigs } from './GitHubEntityProviderConfig';
describe('readProviderConfigs', () => {
@@ -81,13 +82,22 @@ describe('readProviderConfigs', () => {
organization: 'test-org1',
host: 'ghe.internal.com',
},
providerWithSchedule: {
organization: 'test-org1',
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(6);
expect(providerConfigs).toHaveLength(7);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -101,6 +111,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[1]).toEqual({
id: 'providerCustomCatalogPath',
@@ -115,6 +126,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[2]).toEqual({
id: 'providerWithRepositoryFilter',
@@ -129,6 +141,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[3]).toEqual({
id: 'providerWithBranchFilter',
@@ -143,6 +156,7 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[4]).toEqual({
id: 'providerWithTopicFilter',
@@ -157,6 +171,7 @@ describe('readProviderConfigs', () => {
exclude: ['backstage-exclude'],
},
},
schedule: undefined,
});
expect(providerConfigs[5]).toEqual({
id: 'providerWithHost',
@@ -171,6 +186,27 @@ describe('readProviderConfigs', () => {
exclude: undefined,
},
},
schedule: undefined,
});
expect(providerConfigs[6]).toEqual({
id: 'providerWithSchedule',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
schedule: {
frequency: Duration.fromISO('PT30M'),
timeout: {
minutes: 3,
},
},
});
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts
+11
View File
@@ -14,6 +14,10 @@
* limitations under the License.
*/
import {
readTaskScheduleDefinitionFromConfig,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
@@ -29,6 +33,7 @@ export type GitHubEntityProviderConfig = {
branch?: string;
topic?: GithubTopicFilters;
};
schedule?: TaskScheduleDefinition;
};
export type GithubTopicFilters = {
@@ -73,6 +78,10 @@ function readProviderConfig(
'filters.topic.exclude',
);
const schedule = config.has('schedule')
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
: undefined;
return {
id,
catalogPath,
@@ -88,8 +97,10 @@ function readProviderConfig(
exclude: topicFilterExclude,
},
},
schedule,
};
}
/**
* Compiles a RegExp while enforcing the pattern to contain
* the start-of-line and end-of-line anchors.
plugins/catalog-backend-module-github/src/service/GithubEntityProviderCatalogModule.test.ts
+84
View File
@@ -0,0 +1,84 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
configServiceRef,
loggerServiceRef,
schedulerServiceRef,
} from '@backstage/backend-plugin-api';
import {
PluginTaskScheduler,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { startTestBackend } from '@backstage/backend-test-utils';
import { ConfigReader } from '@backstage/config';
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';
import { Duration } from 'luxon';
import { githubEntityProviderCatalogModule } from './GithubEntityProviderCatalogModule';
import { GitHubEntityProvider } from '../providers/GitHubEntityProvider';
describe('githubEntityProviderCatalogModule', () => {
it('should register provider at the catalog extension point', async () => {
let addedProviders: Array<GitHubEntityProvider> | undefined;
let usedSchedule: TaskScheduleDefinition | undefined;
const extensionPoint = {
addEntityProvider: (providers: any) => {
addedProviders = providers;
},
};
const runner = jest.fn();
const scheduler = {
createScheduledTaskRunner: (schedule: TaskScheduleDefinition) => {
usedSchedule = schedule;
return runner;
},
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'module-test',
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
},
},
},
});
await startTestBackend({
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
services: [
[configServiceRef, config],
[loggerServiceRef, getVoidLogger()],
[schedulerServiceRef, scheduler],
],
features: [githubEntityProviderCatalogModule()],
});
expect(usedSchedule?.frequency).toEqual(Duration.fromISO('P1M'));
expect(usedSchedule?.timeout).toEqual(Duration.fromISO('PT3M'));
expect(addedProviders?.length).toEqual(1);
expect(addedProviders?.pop()?.getProviderName()).toEqual(
'github-provider:default',
);
expect(runner).not.toHaveBeenCalled();
});
});
plugins/catalog-backend-module-github/src/module.ts → plugins/catalog-backend-module-github/src/service/GithubEntityProviderCatalogModule.ts
+5 -21
View File
@@ -21,18 +21,8 @@ import {
loggerServiceRef,
schedulerServiceRef,
} from '@backstage/backend-plugin-api';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';
import { GitHubEntityProvider } from './providers/GitHubEntityProvider';
/**
* Options for {@link githubEntityProviderCatalogModule}.
*
* @alpha
*/
export type GithubEntityProviderCatalogModuleOptions = {
schedule?: TaskScheduleDefinition;
};
import { GitHubEntityProvider } from '../providers/GitHubEntityProvider';
/**
* Registers the GitHubEntityProvider with the catalog processing extension point.
@@ -42,25 +32,19 @@ export type GithubEntityProviderCatalogModuleOptions = {
export const githubEntityProviderCatalogModule = createBackendModule({
pluginId: 'catalog',
moduleId: 'githubEntityProvider',
register(env, options?: GithubEntityProviderCatalogModuleOptions) {
register(env) {
env.registerInit({
deps: {
config: configServiceRef,
catalog: catalogProcessingExtensionPoint,
config: configServiceRef,
logger: loggerServiceRef,
scheduler: schedulerServiceRef,
},
async init({ config, catalog, logger, scheduler }) {
const scheduleDef = options?.schedule ?? {
frequency: { seconds: 600 },
timeout: { seconds: 900 },
initialDelay: { seconds: 3 },
};
async init({ catalog, config, logger, scheduler }) {
catalog.addEntityProvider(
GitHubEntityProvider.fromConfig(config, {
logger: loggerToWinstonLogger(logger),
schedule: scheduler.createScheduledTaskRunner(scheduleDef),
scheduler,
}),
);
},