paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(catalog/gitlab): Add option to configure schedule via app-config.yaml

Browse Source 
Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-10-07 12:13:18 +02:00
parent 9b698abfd9
commit 81cedb5033
11 changed files with 225 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/flat-kangaroos-kiss.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-gitlab': patch
---
`GitlabDiscoveryEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
Please find how to configure the schedule at the config at
https://backstage.io/docs/integrations/gitlab/discovery
docs/integrations/gitlab/discovery.md
+8
View File
@@ -25,6 +25,11 @@ catalog:
group: example-group # Optional. Group and subgroup (if needed) to look for repositories. If not present the whole project will be scanned
entityFilename: catalog-info.yaml # Optional. Defaults to `catalog-info.yaml`
projectPattern: /[\s\S]*/ # Optional. Filters found projects based on provided patter. Defaults to `/[\s\S]*/`, what means to not filter anything
schedule: # optional; same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
```
As this provider is not one of the default providers, you will first need to install
@@ -47,10 +52,13 @@ const builder = await CatalogBuilder.create(env);
builder.addEntityProvider(
...GitlabDiscoveryEntityProvider.fromConfig(env.config, {
logger: env.logger,
// optional: alternatively, use scheduler with schedule defined in app-config.yaml
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 3 },
}),
// optional: alternatively, use schedule
scheduler: env.scheduler,
}),
);
```
plugins/catalog-backend-module-gitlab/api-report.md
+3 -1
View File
@@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { TaskRunner } from '@backstage/backend-tasks';
// @public
@@ -21,7 +22,8 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitlabDiscoveryEntityProvider[];
// (undocumented)
plugins/catalog-backend-module-gitlab/config.d.ts
Vendored
+6 -7
View File
@@ -14,11 +14,10 @@
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
catalog?: {
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* GitlabDiscoveryEntityProvider configuration
@@ -28,27 +27,27 @@ export interface Config {
{
/**
* (Required) Gitlab's host name.
* @visibility backend
*/
host: string;
/**
* (Optional) Gitlab's group[/subgroup] where the discovery is done.
* If not defined the whole project will be scanned.
* @visibility backend
*/
group?: string;
/**
* (Optional) Default branch to read the catalog-info.yaml file.
* If not set, 'master' will be used.
* @visibility backend
*/
branch?: string;
/**
* (Optional) The name used for the catalog file.
* If not set, 'catalog-info.yaml' will be used.
* @visibility backend
*/
entityFilename?: string;
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
>;
};
plugins/catalog-backend-module-gitlab/package.json
+2 -1
View File
@@ -50,7 +50,8 @@
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151",
"@types/uuid": "^8.0.0"
"@types/uuid": "^8.0.0",
"luxon": "^3.0.0"
},
"files": [
"config.d.ts",
plugins/catalog-backend-module-gitlab/src/lib/types.ts
+3
View File
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
export type GitlabGroupDescription = {
id: number;
web_url: string;
@@ -36,4 +38,5 @@ export type GitlabProviderConfig = {
branch: string;
catalogFile: string;
projectPattern: RegExp;
schedule?: TaskScheduleDefinition;
};
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.test.ts
+114 -3
View File
@@ -15,13 +15,17 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
TaskRunner,
} from '@backstage/backend-tasks';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { GitlabDiscoveryEntityProvider } from './GitlabDiscoveryEntityProvider';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
@@ -338,4 +342,111 @@ describe('GitlabDiscoveryEntityProvider', () => {
],
});
});
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'test-gitlab',
apiBaseUrl: 'https://api.gitlab.example/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'test-gitlab',
group: 'test-group',
},
},
},
},
});
expect(() =>
GitlabDiscoveryEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'test-gitlab',
apiBaseUrl: 'https://api.gitlab.example/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'test-gitlab',
group: 'test-group',
},
},
},
},
});
expect(() =>
GitlabDiscoveryEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:test-id',
);
});
it('single simple provider config with schedule in config', async () => {
const schedule = new PersistingTaskRunner();
const scheduler = {
createScheduledTaskRunner: (_: any) => schedule,
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
integrations: {
gitlab: [
{
host: 'test-gitlab',
apiBaseUrl: 'https://api.gitlab.example/api/v4',
token: '1234',
},
],
},
catalog: {
providers: {
gitlab: {
'test-id': {
host: 'test-gitlab',
group: 'test-group',
schedule: {
frequency: 'PT30M',
timeout: 'PT3M',
},
},
},
},
},
});
const providers = GitlabDiscoveryEntityProvider.fromConfig(config, {
logger,
scheduler,
});
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual(
'GitlabDiscoveryEntityProvider:test-id',
);
});
});
plugins/catalog-backend-module-gitlab/src/providers/GitlabDiscoveryEntityProvider.ts
+29 -9
View File
@@ -14,14 +14,16 @@
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { GitLabIntegration, ScmIntegrations } from '@backstage/integration';
import {
EntityProvider,
EntityProviderConnection,
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import {
GitLabClient,
@@ -30,8 +32,6 @@ import {
paginated,
readGitlabConfigs,
} from '../lib';
import * as uuid from 'uuid';
import { locationSpecToLocationEntity } from '@backstage/plugin-catalog-backend';
type Result = {
scanned: number;
@@ -51,8 +51,16 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
static fromConfig(
config: Config,
options: { logger: Logger; schedule: TaskRunner },
options: {
logger: Logger;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): GitlabDiscoveryEntityProvider[] {
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
const providerConfigs = readGitlabConfigs(config);
const integrations = ScmIntegrations.fromConfig(config).gitlab;
const providers: GitlabDiscoveryEntityProvider[] = [];
@@ -64,11 +72,23 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
`No gitlab integration found that matches host ${providerConfig.host}`,
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for GitlabDiscoveryEntityProvider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
providers.push(
new GitlabDiscoveryEntityProvider({
...options,
config: providerConfig,
integration,
taskRunner,
}),
);
});
@@ -79,14 +99,14 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
config: GitlabProviderConfig;
integration: GitLabIntegration;
logger: Logger;
schedule: TaskRunner;
taskRunner: TaskRunner;
}) {
this.config = options.config;
this.integration = options.integration;
this.logger = options.logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(options.schedule);
this.scheduleFn = this.createScheduleFn(options.taskRunner);
}
getProviderName(): string {
@@ -98,10 +118,10 @@ export class GitlabDiscoveryEntityProvider implements EntityProvider {
await this.scheduleFn();
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
return taskRunner.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
plugins/catalog-backend-module-gitlab/src/providers/config.test.ts
+45
View File
@@ -15,6 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readGitlabConfigs } from './config';
describe('config', () => {
@@ -53,6 +54,7 @@ describe('config', () => {
host: 'host',
catalogFile: 'catalog-info.yaml',
projectPattern: /[\s\S]*/,
schedule: undefined,
}),
);
});
@@ -83,6 +85,49 @@ describe('config', () => {
host: 'host',
catalogFile: 'custom-file.yaml',
projectPattern: /[\s\S]*/,
schedule: undefined,
}),
);
});
it('valid config with schedule', () => {
const config = new ConfigReader({
catalog: {
providers: {
gitlab: {
test: {
group: 'group',
host: 'host',
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
},
});
const result = readGitlabConfigs(config);
expect(result).toHaveLength(1);
result.forEach(r =>
expect(r).toStrictEqual({
id: 'test',
group: 'group',
branch: 'master',
host: 'host',
catalogFile: 'catalog-info.yaml',
projectPattern: /[\s\S]*/,
schedule: {
frequency: Duration.fromISO('PT30M'),
timeout: {
minutes: 3,
},
initialDelay: undefined,
scope: undefined,
},
}),
);
});
plugins/catalog-backend-module-gitlab/src/providers/config.ts
+6
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { GitlabProviderConfig } from '../lib';
@@ -35,6 +36,10 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
config.getOptionalString('projectPattern') ?? /[\s\S]*/,
);
const schedule = config.has('schedule')
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
: undefined;
return {
id,
group,
@@ -42,6 +47,7 @@ function readGitlabConfig(id: string, config: Config): GitlabProviderConfig {
host,
catalogFile,
projectPattern,
schedule,
};
}
yarn.lock
+1
View File
@@ -4622,6 +4622,7 @@ __metadata:
"@types/lodash": ^4.14.151
"@types/uuid": ^8.0.0
lodash: ^4.17.21
luxon: ^3.0.0
msw: ^0.47.0
node-fetch: ^2.6.7
uuid: ^8.0.0