diff --git a/.changeset/fuzzy-dolls-give.md b/.changeset/fuzzy-dolls-give.md new file mode 100644 index 0000000000..4de4d2e128 --- /dev/null +++ b/.changeset/fuzzy-dolls-give.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-tasks': patch +--- + +Added new function `readTaskScheduleDefinitionFromConfig` to read `TaskScheduleDefinition` (aka. schedule) from the `Config`. diff --git a/packages/backend-tasks/api-report.md b/packages/backend-tasks/api-report.md index c0802fa46c..a60f6dba37 100644 --- a/packages/backend-tasks/api-report.md +++ b/packages/backend-tasks/api-report.md @@ -30,6 +30,11 @@ export interface PluginTaskScheduler { triggerTask(id: string): Promise; } +// @public +export function readTaskScheduleDefinitionFromConfig( + config: Config, +): TaskScheduleDefinition; + // @public export type TaskFunction = | ((abortSignal: AbortSignal_2) => void | Promise) @@ -60,6 +65,19 @@ export interface TaskScheduleDefinition { timeout: Duration | HumanDuration; } +// @public +export interface TaskScheduleDefinitionConfig { + frequency: + | { + cron: string; + } + | string + | HumanDuration; + initialDelay?: string | HumanDuration; + scope?: 'global' | 'local'; + timeout: string | HumanDuration; +} + // @public export class TaskScheduler { constructor(databaseManager: DatabaseManager, logger: Logger); diff --git a/packages/backend-tasks/src/tasks/index.ts b/packages/backend-tasks/src/tasks/index.ts index 990d6332fc..8f30e5c8dd 100644 --- a/packages/backend-tasks/src/tasks/index.ts +++ b/packages/backend-tasks/src/tasks/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig'; export { TaskScheduler } from './TaskScheduler'; export type { PluginTaskScheduler, @@ -21,5 +22,6 @@ export type { TaskInvocationDefinition, TaskRunner, TaskScheduleDefinition, + TaskScheduleDefinitionConfig, HumanDuration, } from './types'; diff --git a/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts new file mode 100644 index 0000000000..115ef89168 --- /dev/null +++ b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { Duration } from 'luxon'; +import { readTaskScheduleDefinitionFromConfig } from './readTaskScheduleDefinitionFromConfig'; +import { HumanDuration } from './types'; + +describe('readTaskScheduleDefinitionFromConfig', () => { + it('all valid values', () => { + const config = new ConfigReader({ + frequency: { + cron: '0 30 * * * *', + }, + timeout: 'PT3M', + initialDelay: { + minutes: 20, + }, + scope: 'global', + }); + + const result = readTaskScheduleDefinitionFromConfig(config); + + expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *'); + expect(result.timeout).toEqual(Duration.fromISO('PT3M')); + expect((result.initialDelay as HumanDuration).minutes).toEqual(20); + expect(result.scope).toBe('global'); + }); + + it('all valid required values', () => { + const config = new ConfigReader({ + frequency: { + cron: '0 30 * * * *', + }, + timeout: 'PT3M', + }); + + const result = readTaskScheduleDefinitionFromConfig(config); + + expect((result.frequency as { cron: string }).cron).toBe('0 30 * * * *'); + expect(result.timeout).toEqual(Duration.fromISO('PT3M')); + expect(result.initialDelay).toBeUndefined(); + expect(result.scope).toBeUndefined(); + }); + + it('fail without required frequency', () => { + const config = new ConfigReader({ + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + "Missing required config value at 'frequency'", + ); + }); + + it('fail without required timeout', () => { + const config = new ConfigReader({ + frequency: 'PT30M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + "Missing required config value at 'timeout'", + ); + }); + + it('invalid frequency value', () => { + const config = new ConfigReader({ + frequency: { + invalid: 'value', + }, + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'HumanDuration needs at least one of', + ); + }); + + it('frequency value with additional invalid prop', () => { + const config = new ConfigReader({ + frequency: { + minutes: 20, + invalid: 'value', + }, + timeout: 'PT3M', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'HumanDuration does not contain properties: invalid', + ); + }); + + it('invalid scope value', () => { + const config = new ConfigReader({ + frequency: { + years: 2, + }, + timeout: 'PT3M', + scope: 'invalid', + }); + + expect(() => readTaskScheduleDefinitionFromConfig(config)).toThrow( + 'Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: invalid', + ); + }); +}); diff --git a/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts new file mode 100644 index 0000000000..3267d8ef14 --- /dev/null +++ b/packages/backend-tasks/src/tasks/readTaskScheduleDefinitionFromConfig.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { JsonObject } from '@backstage/types'; +import { HumanDuration, TaskScheduleDefinition } from './types'; +import { Duration } from 'luxon'; + +const propsOfHumanDuration = [ + 'years', + 'months', + 'weeks', + 'days', + 'hours', + 'minutes', + 'seconds', + 'milliseconds', +]; + +function convertToHumanDuration(config: Config, key: string): HumanDuration { + const props = config.getConfig(key).keys(); + if (!props.find(prop => propsOfHumanDuration.includes(prop))) { + throw new Error( + `HumanDuration needs at least one of: ${propsOfHumanDuration}`, + ); + } + + const invalidProps = props.filter( + prop => !propsOfHumanDuration.includes(prop), + ); + if (invalidProps.length > 0) { + throw new Error( + `HumanDuration does not contain properties: ${invalidProps}`, + ); + } + + return config.get(key) as HumanDuration; +} + +function readDuration(config: Config, key: string): Duration | HumanDuration { + return typeof config.get(key) === 'string' + ? Duration.fromISO(config.getString(key)) + : convertToHumanDuration(config, key); +} + +function readCronOrDuration( + config: Config, + key: string, +): { cron: string } | Duration | HumanDuration { + const value = config.get(key); + if (typeof value === 'object' && (value as { cron?: string }).cron) { + return value as { cron: string }; + } + + return readDuration(config, key); +} + +/** + * Reads a TaskScheduleDefinition from a Config. + * Expects the config not to be the root config, + * but the config for the definition. + * + * @param config - config for a TaskScheduleDefinition. + * @public + */ +export function readTaskScheduleDefinitionFromConfig( + config: Config, +): TaskScheduleDefinition { + const frequency = readCronOrDuration(config, 'frequency'); + const timeout = readDuration(config, 'timeout'); + + const initialDelay = config.has('initialDelay') + ? readDuration(config, 'initialDelay') + : undefined; + + const scope = config.getOptionalString('scope'); + if (scope && !['global', 'local'].includes(scope)) { + throw new Error( + `Only "global" or "local" are allowed for TaskScheduleDefinition.scope, but got: ${scope}`, + ); + } + + return { + frequency, + timeout, + initialDelay, + scope: scope as 'global' | 'local' | undefined, + }; +} diff --git a/packages/backend-tasks/src/tasks/types.ts b/packages/backend-tasks/src/tasks/types.ts index 86c0c5c462..92ec045605 100644 --- a/packages/backend-tasks/src/tasks/types.ts +++ b/packages/backend-tasks/src/tasks/types.ts @@ -58,7 +58,7 @@ export interface TaskScheduleDefinition { * * @remarks * - * This is a best effort value; under some circumstances there can be + * This is the best effort value; under some circumstances there can be * deviations. For example, if the task runtime is longer than the frequency * and the timeout has not been given or not been exceeded yet, the next * invocation of this task will be delayed until after the previous one @@ -112,7 +112,7 @@ export interface TaskScheduleDefinition { * collaborating on a task that has its `scope` field set to `'global'`, then * you may still see the task being processed by other long-lived workers, * while any given single worker is in its initial sleep delay time e.g. after - * a deployment. Therefore this parameter is not useful for "globally" pausing + * a deployment. Therefore, this parameter is not useful for "globally" pausing * work; its main intended use is for individual machines to get a chance to * reach some equilibrium at startup before triggering heavy batch workloads. */ @@ -128,7 +128,104 @@ export interface TaskScheduleDefinition { * attempt to ensure that only one worker machine runs the task at a time, * according to the given cadence. This means that as the number of worker * hosts increases, the invocation frequency of this task will not go up. - * Instead the load is spread randomly across hosts. This setting is useful + * Instead, the load is spread randomly across hosts. This setting is useful + * for tasks that access shared resources, for example catalog ingestion tasks + * where you do not want many machines to repeatedly import the same data and + * trample over each other. + * + * When the scope is set to `'local'`, there is no concurrency control across + * hosts. Each host runs the task according to the given cadence similarly to + * `setInterval`, but the runtime ensures that there are no overlapping runs. + * + * @defaultValue 'global' + */ + scope?: 'global' | 'local'; +} + +/** + * Config options for {@link TaskScheduleDefinition} + * that control the scheduling of a task. + * + * @public + */ +export interface TaskScheduleDefinitionConfig { + /** + * How often you want the task to run. The system does its best to avoid + * overlapping invocations. + * + * @remarks + * + * This is the best effort value; under some circumstances there can be + * deviations. For example, if the task runtime is longer than the frequency + * and the timeout has not been given or not been exceeded yet, the next + * invocation of this task will be delayed until after the previous one + * finishes. + * + * This is a required field. + */ + frequency: + | { + /** + * A crontab style string. + * + * @remarks + * + * Overview: + * + * ``` + * ┌────────────── second (optional) + * │ ┌──────────── minute + * │ │ ┌────────── hour + * │ │ │ ┌──────── day of month + * │ │ │ │ ┌────── month + * │ │ │ │ │ ┌──── day of week + * │ │ │ │ │ │ + * │ │ │ │ │ │ + * * * * * * * + * ``` + */ + cron: string; + } + | string + | HumanDuration; + + /** + * The maximum amount of time that a single task invocation can take, before + * it's considered timed out and gets "released" such that a new invocation + * is permitted to take place (possibly, then, on a different worker). + */ + timeout: string | HumanDuration; + + /** + * The amount of time that should pass before the first invocation happens. + * + * @remarks + * + * This can be useful in cold start scenarios to stagger or delay some heavy + * compute jobs. If no value is given for this field then the first invocation + * will happen as soon as possible according to the cadence. + * + * NOTE: This is a per-worker delay. If you have a cluster of workers all + * collaborating on a task that has its `scope` field set to `'global'`, then + * you may still see the task being processed by other long-lived workers, + * while any given single worker is in its initial sleep delay time e.g. after + * a deployment. Therefore, this parameter is not useful for "globally" pausing + * work; its main intended use is for individual machines to get a chance to + * reach some equilibrium at startup before triggering heavy batch workloads. + */ + initialDelay?: string | HumanDuration; + + /** + * Sets the scope of concurrency control / locking to apply for invocations of + * this task. + * + * @remarks + * + * When the scope is set to the default value `'global'`, the scheduler will + * attempt to ensure that only one worker machine runs the task at a time, + * according to the given cadence. This means that as the number of worker + * hosts increases, the invocation frequency of this task will not go up. + * Instead, the load is spread randomly across hosts. This setting is useful * for tasks that access shared resources, for example catalog ingestion tasks * where you do not want many machines to repeatedly import the same data and * trample over each other.