paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 20 Packages Projects Releases Wiki Activity

Merge pull request #13859 from Bonial-International-GmbH/pjungermann/catalog-modules/new-backend-api

Browse Source 
feat(catalog/bitbucketCloud): schedule via config + backend-plugin-api support
This commit is contained in:
Johan Haals
2022-10-05 12:52:18 +02:00
committed by GitHub
parent b2485bd9b4 a9b91d39bb
commit b2bec2e5a2
20 changed files with 685 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-tasks/api-report.md
+18
View File
@@ -30,6 +30,11 @@ export interface PluginTaskScheduler {
triggerTask(id: string): Promise<void>;
}
// @public
export function readTaskScheduleDefinitionFromConfig(
config: Config,
): TaskScheduleDefinition;
// @public
export type TaskFunction =
| ((abortSignal: AbortSignal_2) => void | Promise<void>)
@@ -60,6 +65,19 @@ export interface TaskScheduleDefinition {
timeout: Duration | HumanDuration;
}
// @public
export interface TaskScheduleDefinitionConfig {
frequency:
| {
cron: string;
}
| string
| HumanDuration;
initialDelay?: string | HumanDuration;
scope?: 'global' | 'local';
timeout: string | HumanDuration;
}
// @public
export class TaskScheduler {
constructor(databaseManager: DatabaseManager, logger: Logger);
packages/backend-tasks/src/tasks/index.ts
+2
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig';
export { TaskScheduler } from './TaskScheduler';
export type {
PluginTaskScheduler,
@@ -21,5 +22,6 @@ export type {
TaskInvocationDefinition,
TaskRunner,
TaskScheduleDefinition,
TaskScheduleDefinitionConfig,
HumanDuration,
} from './types';
packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts
+119
View File
@@ -0,0 +1,119 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig';
import { HumanDuration } from './types';
describe('readTaskScheduleDefinitionFromConfig', () => {
it('all valid values', () => {
const config = new ConfigReader({
frequency: {
cron: '0 30 * * * *',
},
timeout: 'PT3M',
initialDelay: {
minutes: 20,
},
scope: 'global',
});
const result = readTaskScheduleDefinitionFromConfig(config);
expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *');
expect(result.timeout).toEqual(Duration.fromISO('PT3M'));
expect((result.initialDelay as HumanDuration).minutes).toEqual(20);
expect(result.scope).toBe('global');
});
it('all valid required values', () => {
const config = new ConfigReader({
frequency: {
cron: '0 30 * * * *',
},
timeout: 'PT3M',
});
const result = readTaskScheduleDefinitionFromConfig(config);
expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *');
expect(result.timeout).toEqual(Duration.fromISO('PT3M'));
expect(result.initialDelay).toBeUndefined();
expect(result.scope).toBeUndefined();
});
it('fail without required frequency', () => {
const config = new ConfigReader({
timeout: 'PT3M',
});
expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow(
"Missing required config value at 'frequency'",
);
});
it('fail without required timeout', () => {
const config = new ConfigReader({
frequency: 'PT30M',
});
expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow(
"Missing required config value at 'timeout'",
);
});
it('invalid frequency value', () => {
const config = new ConfigReader({
frequency: {
invalid: 'value',
},
timeout: 'PT3M',
});
expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow(
'HumanDuration needs at least one of',
);
});
it('frequency value with additional invalid prop', () => {
const config = new ConfigReader({
frequency: {
minutes: 20,
invalid: 'value',
},
timeout: 'PT3M',
});
expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow(
'HumanDuration does not contain properties: invalid',
);
});
it('invalid scope value', () => {
const config = new ConfigReader({
frequency: {
years: 2,
},
timeout: 'PT3M',
scope: 'invalid',
});
expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow(
'Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: invalid',
);
});
});
packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts
+102
View File
@@ -0,0 +1,102 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { HumanDuration, TaskScheduleDefinition } from './types';
import { Duration } from 'luxon';
const propsOfHumanDuration = [
'years',
'months',
'weeks',
'days',
'hours',
'minutes',
'seconds',
'milliseconds',
];
function convertToHumanDuration(config: Config, key: string): HumanDuration {
const props = config.getConfig(key).keys();
if (!props.find(prop => propsOfHumanDuration.includes(prop))) {
throw new Error(
`HumanDuration needs at least one of: ${propsOfHumanDuration}`,
);
}
const invalidProps = props.filter(
prop => !propsOfHumanDuration.includes(prop),
);
if (invalidProps.length > 0) {
throw new Error(
`HumanDuration does not contain properties: ${invalidProps}`,
);
}
return config.get<JsonObject>(key) as HumanDuration;
}
function readDuration(config: Config, key: string): Duration | HumanDuration {
return typeof config.get(key) === 'string'
? Duration.fromISO(config.getString(key))
: convertToHumanDuration(config, key);
}
function readCronOrDuration(
config: Config,
key: string,
): { cron: string } | Duration | HumanDuration {
const value = config.get(key);
if (typeof value === 'object' && (value as { cron?: string }).cron) {
return value as { cron: string };
}
return readDuration(config, key);
}
/**
* Reads a TaskScheduleDefinition from a Config.
* Expects the config not to be the root config,
* but the config for the definition.
*
* @param config - config for a TaskScheduleDefinition.
* @public
*/
export function readTaskScheduleDefinitionFromConfig(
config: Config,
): TaskScheduleDefinition {
const frequency = readCronOrDuration(config, 'frequency');
const timeout = readDuration(config, 'timeout');
const initialDelay = config.has('initialDelay')
? readDuration(config, 'initialDelay')
: undefined;
const scope = config.getOptionalString('scope');
if (scope && !['global', 'local'].includes(scope)) {
throw new Error(
`Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: ${scope}`,
);
}
return {
frequency,
timeout,
initialDelay,
scope: scope as 'global' | 'local' | undefined,
};
}
packages/backend-tasks/src/tasks/types.ts
+100 -3
View File
@@ -58,7 +58,7 @@ export interface TaskScheduleDefinition {
*
* @remarks
*
* This is a best effort value; under some circumstances there can be
* This is the best effort value; under some circumstances there can be
* deviations. For example, if the task runtime is longer than the frequency
* and the timeout has not been given or not been exceeded yet, the next
* invocation of this task will be delayed until after the previous one
@@ -112,7 +112,7 @@ export interface TaskScheduleDefinition {
* collaborating on a task that has its `scope` field set to `'global'`, then
* you may still see the task being processed by other long-lived workers,
* while any given single worker is in its initial sleep delay time e.g. after
* a deployment. Therefore this parameter is not useful for "globally" pausing
* a deployment. Therefore, this parameter is not useful for "globally" pausing
* work; its main intended use is for individual machines to get a chance to
* reach some equilibrium at startup before triggering heavy batch workloads.
*/
@@ -128,7 +128,104 @@ export interface TaskScheduleDefinition {
* attempt to ensure that only one worker machine runs the task at a time,
* according to the given cadence. This means that as the number of worker
* hosts increases, the invocation frequency of this task will not go up.
* Instead the load is spread randomly across hosts. This setting is useful
* Instead, the load is spread randomly across hosts. This setting is useful
* for tasks that access shared resources, for example catalog ingestion tasks
* where you do not want many machines to repeatedly import the same data and
* trample over each other.
*
* When the scope is set to `'local'`, there is no concurrency control across
* hosts. Each host runs the task according to the given cadence similarly to
* `setInterval`, but the runtime ensures that there are no overlapping runs.
*
* @defaultValue 'global'
*/
scope?: 'global' | 'local';
}
/**
* Config options for {@link TaskScheduleDefinition}
* that control the scheduling of a task.
*
* @public
*/
export interface TaskScheduleDefinitionConfig {
/**
* How often you want the task to run. The system does its best to avoid
* overlapping invocations.
*
* @remarks
*
* This is the best effort value; under some circumstances there can be
* deviations. For example, if the task runtime is longer than the frequency
* and the timeout has not been given or not been exceeded yet, the next
* invocation of this task will be delayed until after the previous one
* finishes.
*
* This is a required field.
*/
frequency:
| {
/**
* A crontab style string.
*
* @remarks
*
* Overview:
*
* ```
* ┌────────────── second (optional)
* │ ┌──────────── minute
* │ │ ┌────────── hour
* │ │ │ ┌──────── day of month
* │ │ │ │ ┌────── month
* │ │ │ │ │ ┌──── day of week
* │ │ │ │ │ │
* │ │ │ │ │ │
* * * * * * *
* ```
*/
cron: string;
}
| string
| HumanDuration;
/**
* The maximum amount of time that a single task invocation can take, before
* it's considered timed out and gets "released" such that a new invocation
* is permitted to take place (possibly, then, on a different worker).
*/
timeout: string | HumanDuration;
/**
* The amount of time that should pass before the first invocation happens.
*
* @remarks
*
* This can be useful in cold start scenarios to stagger or delay some heavy
* compute jobs. If no value is given for this field then the first invocation
* will happen as soon as possible according to the cadence.
*
* NOTE: This is a per-worker delay. If you have a cluster of workers all
* collaborating on a task that has its `scope` field set to `'global'`, then
* you may still see the task being processed by other long-lived workers,
* while any given single worker is in its initial sleep delay time e.g. after
* a deployment. Therefore, this parameter is not useful for "globally" pausing
* work; its main intended use is for individual machines to get a chance to
* reach some equilibrium at startup before triggering heavy batch workloads.
*/
initialDelay?: string | HumanDuration;
/**
* Sets the scope of concurrency control / locking to apply for invocations of
* this task.
*
* @remarks
*
* When the scope is set to the default value `'global'`, the scheduler will
* attempt to ensure that only one worker machine runs the task at a time,
* according to the given cadence. This means that as the number of worker
* hosts increases, the invocation frequency of this task will not go up.
* Instead, the load is spread randomly across hosts. This setting is useful
* for tasks that access shared resources, for example catalog ingestion tasks
* where you do not want many machines to repeatedly import the same data and
* trample over each other.