paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(catalog/bitbucketServer): Add option to configure schedule via app-config.yaml

Browse Source 
Relates-to: PR #13859
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-10-07 10:11:29 +02:00
parent 0ca399b31b
commit 68f7f5a857
10 changed files with 285 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/tasty-scissors-tickle.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-bitbucket-server': patch
---
`BitbucketServerEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
Please find how to configure the schedule at the config at
https://backstage.io/docs/integrations/bitbucketServer/discovery
docs/integrations/bitbucketServer/discovery.md
+25 -11
View File
@@ -37,10 +37,13 @@ And then add the entity provider to your catalog builder:
+ builder.addEntityProvider(
+ BitbucketServerEntityProvider.fromConfig(env.config, {
+ logger: env.logger,
+ // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+ schedule: env.scheduler.createScheduledTaskRunner({
+ frequency: { minutes: 30 },
+ timeout: { minutes: 3 },
+ }),
+ // optional: alternatively, use schedule
+ scheduler: env.scheduler,
+ }),
+ );
@@ -66,19 +69,33 @@ catalog:
filters: # optional
projectKey: '^apis-.*$' # optional; RegExp
repoSlug: '^service-.*$' # optional; RegExp
schedule: # optional; same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
```
- **host**:
- **`host`**:
The host of the Bitbucket Server instance, **note**: the host needs to registered as an integration as well, see [location](locations.md).
- **`catalogPath`** _(optional)_:
Default: `/catalog-info.yaml`.
Path where to look for `catalog-info.yaml` files.
When started with `/`, it is an absolute path from the repo root.
- **filters** _(optional)_:
- **`filters`** _(optional)_:
- **`projectKey`** _(optional)_:
Regular expression used to filter results based on the project key.
- **repoSlug** _(optional)_:
- **`repoSlug`** _(optional)_:
Regular expression used to filter results based on the repo slug.
- **`schedule`** _(optional)_:
- **`frequency`**:
How often you want the task to run. The system does its best to avoid overlapping invocations.
- **`timeout`**:
The maximum amount of time that a single task invocation can take.
- **`initialDelay`** _(optional)_:
The amount of time that should pass before the first invocation happens.
- **`scope`** _(optional)_:
`'global'` or `'local'`. Sets the scope of concurrency control.
## Custom location processing
@@ -93,18 +110,15 @@ repository.
```typescript
const provider = BitbucketServerEntityProvider.fromConfig(env.config, {
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 3 },
}),
schedule: env.scheduler,
parser: async function* customLocationParser(options: {
location: LocationSpec,
client: BitbucketServerClient,
location: LocationSpec;
client: BitbucketServerClient;
}) {
// Custom logic for interpreting the matching repository
// See defaultBitbucketServerLocationParser for an example
}
);
},
});
```
## Alternative
plugins/catalog-backend-module-bitbucket-server/api-report.md
+3 -1
View File
@@ -10,6 +10,7 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { Response as Response_2 } from 'node-fetch';
import { TaskRunner } from '@backstage/backend-tasks';
@@ -55,8 +56,9 @@ export class BitbucketServerEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
parser?: BitbucketServerLocationParser;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): BitbucketServerEntityProvider[];
// (undocumented)
plugins/catalog-backend-module-bitbucket-server/config.d.ts
Vendored
+10 -3
View File
@@ -14,11 +14,10 @@
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
catalog?: {
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* BitbucketServerEntityProvider configuration
@@ -48,6 +47,10 @@ export interface Config {
*/
projectKey?: RegExp;
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
| Record<
string,
@@ -73,6 +76,10 @@ export interface Config {
*/
projectKey?: RegExp;
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
>;
};
plugins/catalog-backend-module-bitbucket-server/package.json
+1
View File
@@ -46,6 +46,7 @@
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"luxon": "^3.0.0",
"msw": "^0.47.0"
},
"files": [
plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.test.ts
+167 -4
View File
@@ -15,14 +15,18 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
TaskRunner,
} from '@backstage/backend-tasks';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { BitbucketServerPagedResponse } from '../lib/BitbucketServerClient';
import { BitbucketServerEntityProvider } from './BitbucketServerEntityProvider';
import { BitbucketServerPagedResponse } from '../lib';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
@@ -367,4 +371,163 @@ describe('BitbucketServerEntityProvider', () => {
entities: expectedEntities,
});
});
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
host: 'bitbucket.mycompany.com',
},
},
},
integrations: {
bitbucketServer: [
{
host: 'bitbucket.mycompany.com',
},
],
},
});
expect(() =>
BitbucketServerEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const scheduler = {
createScheduledTaskRunner: (_: any) => jest.fn(),
} as unknown as PluginTaskScheduler;
const config = new ConfigReader({
catalog: {
providers: {
bitbucketServer: {
host: 'bitbucket.mycompany.com',
},
},
},
integrations: {
bitbucketServer: [
{
host: 'bitbucket.mycompany.com',
},
],
},
});
expect(() =>
BitbucketServerEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for bitbucketServer-provider:default',
);
});
it('apply full update with schedule in config', async () => {
const host = 'bitbucket.mycompany.com';
const config = new ConfigReader({
integrations: {
bitbucketServer: [
{
host: host,
},
],
},
catalog: {
providers: {
bitbucketServer: {
mainProvider: {
host: host,
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const scheduler = {
createScheduledTaskRunner: (_: any) => schedule,
} as unknown as PluginTaskScheduler;
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = BitbucketServerEntityProvider.fromConfig(config, {
logger,
scheduler,
})[0];
expect(provider.getProviderName()).toEqual(
'bitbucketServer-provider:mainProvider',
);
setupStubs(
[
{ key: 'project-test', repos: ['repo-test'] },
{ key: 'other-project', repos: ['other-repo'] },
],
`https://${host}`,
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('bitbucketServer-provider:mainProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
'backstage.io/managed-by-origin-location': `url:https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
},
name: 'generated-77f4323822420990f8c3e3c981d38c2dec4ae3a6',
},
spec: {
presence: 'optional',
target: `https://${host}/projects/project-test/repos/repo-test/browse/catalog-info.yaml`,
type: 'url',
},
},
locationKey: 'bitbucketServer-provider:mainProvider',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
'backstage.io/managed-by-origin-location': `url:https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
},
name: 'generated-d8d4944c30c2906dfee172ddda9537f9893b2c0f',
},
spec: {
presence: 'optional',
target: `https://${host}/projects/other-project/repos/other-repo/browse/catalog-info.yaml`,
type: 'url',
},
},
locationKey: 'bitbucketServer-provider:mainProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
});
plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProvider.ts
+34 -18
View File
@@ -14,29 +14,29 @@
* limitations under the License.
*/
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
import {
BitbucketServerIntegration,
ScmIntegrations,
} from '@backstage/integration';
import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { TaskRunner } from '@backstage/backend-tasks';
import * as uuid from 'uuid';
import {
BitbucketServerLocationParser,
defaultBitbucketServerLocationParser,
} from './BitbucketServerLocationParser';
import {
BitbucketServerIntegration,
ScmIntegrations,
} from '@backstage/integration';
import { BitbucketServerClient, paginated } from '../lib';
import { InputError } from '@backstage/errors';
import {
BitbucketServerEntityProviderConfig,
readProviderConfigs,
} from './BitbucketServerEntityProviderConfig';
import { Entity } from '@backstage/catalog-model';
import {
BitbucketServerLocationParser,
defaultBitbucketServerLocationParser,
} from './BitbucketServerLocationParser';
/**
* Discovers catalog files located in Bitbucket Server.
@@ -58,12 +58,17 @@ export class BitbucketServerEntityProvider implements EntityProvider {
config: Config,
options: {
logger: Logger;
schedule: TaskRunner;
parser?: BitbucketServerLocationParser;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
},
): BitbucketServerEntityProvider[] {
const integrations = ScmIntegrations.fromConfig(config);
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
return readProviderConfigs(config).map(providerConfig => {
const integration = integrations.bitbucketServer.byHost(
providerConfig.host,
@@ -73,11 +78,22 @@ export class BitbucketServerEntityProvider implements EntityProvider {
`No BitbucketServer integration found that matches host ${providerConfig.host}`,
);
}
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for bitbucketServer-provider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
return new BitbucketServerEntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
taskRunner,
options.parser,
);
});
@@ -87,7 +103,7 @@ export class BitbucketServerEntityProvider implements EntityProvider {
config: BitbucketServerEntityProviderConfig,
integration: BitbucketServerIntegration,
logger: Logger,
schedule: TaskRunner,
taskRunner: TaskRunner,
parser?: BitbucketServerLocationParser,
) {
this.integration = integration;
@@ -96,13 +112,13 @@ export class BitbucketServerEntityProvider implements EntityProvider {
this.logger = logger.child({
target: this.getProviderName(),
});
this.scheduleFn = this.createScheduleFn(schedule);
this.scheduleFn = this.createScheduleFn(taskRunner);
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
return taskRunner.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.test.ts
+26 -1
View File
@@ -15,6 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readProviderConfigs } from './BitbucketServerEntityProviderConfig';
describe('readProviderConfigs', () => {
@@ -63,13 +64,22 @@ describe('readProviderConfigs', () => {
host: 'bitbucket2.mycompany.com',
catalogPath: 'custom/path/catalog-info.yaml',
},
thirdProvider: {
host: 'bitbucket3.mycompany.com',
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(2);
expect(providerConfigs).toHaveLength(3);
expect(providerConfigs[0]).toEqual({
id: 'mainProvider',
catalogPath: '/catalog-info.yaml',
@@ -88,6 +98,21 @@ describe('readProviderConfigs', () => {
repoSlug: undefined,
},
});
expect(providerConfigs[2]).toEqual({
id: 'thirdProvider',
catalogPath: '/catalog-info.yaml',
host: 'bitbucket3.mycompany.com',
filters: {
projectKey: undefined,
repoSlug: undefined,
},
schedule: {
frequency: Duration.fromISO('PT30M'),
timeout: {
minutes: 3,
},
},
});
});
it('single provider config with filters', () => {
plugins/catalog-backend-module-bitbucket-server/src/providers/BitbucketServerEntityProviderConfig.ts
+10
View File
@@ -14,6 +14,10 @@
* limitations under the License.
*/
import {
readTaskScheduleDefinitionFromConfig,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
@@ -27,6 +31,7 @@ export type BitbucketServerEntityProviderConfig = {
projectKey?: RegExp;
repoSlug?: RegExp;
};
schedule?: TaskScheduleDefinition;
};
export function readProviderConfigs(
@@ -60,6 +65,10 @@ function readProviderConfig(
const projectKeyPattern = config.getOptionalString('filters.projectKey');
const repoSlugPattern = config.getOptionalString('filters.repoSlug');
const schedule = config.has('schedule')
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
: undefined;
return {
id,
host,
@@ -68,5 +77,6 @@ function readProviderConfig(
projectKey: projectKeyPattern ? new RegExp(projectKeyPattern) : undefined,
repoSlug: repoSlugPattern ? new RegExp(repoSlugPattern) : undefined,
},
schedule,
};
}
yarn.lock
+1
View File
@@ -4522,6 +4522,7 @@ __metadata:
"@backstage/integration": "workspace:^"
"@backstage/plugin-catalog-backend": "workspace:^"
"@types/node-fetch": ^2.5.12
luxon: ^3.0.0
msw: ^0.47.0
node-fetch: ^2.6.7
uuid: ^8.0.0