move to a separate package instead and address comments

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
2021-10-24 19:08:50 +02:00
parent 222793c849
commit e09bf604cd
25 changed files with 229 additions and 101 deletions
@@ -2,4 +2,4 @@
 '@backstage/backend-common': patch
 ---

-Add support for distributed mutexes and scheduled tasks, through the `TaskManager` class. This class can be particularly useful for coordinating things across many deployed instances of a given backend plugin. An example of this is catalog entity providers - with this facility you can register tasks similar to a cron job, and make sure that only one host at a time tries to execute the job, and that the timing (call frequency, timeouts etc) are retained as a global concern, letting you scale your workload safely without affecting the task behavior.
+Added the `isDatabaseConflictError` function.
@@ -12,7 +12,6 @@ import { BitbucketIntegration } from '@backstage/integration';
 import { Config } from '@backstage/config';
 import cors from 'cors';
 import Docker from 'dockerode';
-import { Duration } from 'luxon';
 import { ErrorRequestHandler } from 'express';
 import express from 'express';
 import { GithubCredentialsProvider } from '@backstage/integration';
@@ -384,11 +383,6 @@ export function loadBackendConfig(options: {
  argv: string[];
 }): Promise<Config>;

-// @public
-export interface LockOptions {
-  timeout: Duration;
-}
-
 // @public
 export function notFoundHandler(): RequestHandler;

@@ -408,29 +402,6 @@ export type PluginEndpointDiscovery = {
  getExternalBaseUrl(pluginId: string): Promise<string>;
 };

-// @public
-export interface PluginTaskManager {
-  acquireLock(
-    id: string,
-    options: LockOptions,
-  ): Promise<
-    | {
-        acquired: false;
-      }
-    | {
-        acquired: true;
-        release(): Promise<void>;
-      }
-  >;
-  scheduleTask(
-    id: string,
-    options: TaskOptions,
-    fn: () => void | Promise<void>,
-  ): Promise<{
-    unschedule: () => Promise<void>;
-  }>;
-}
-
 // @public
 export type ReaderFactory = (options: {
  config: Config;
@@ -608,27 +579,6 @@ export interface StatusCheckHandlerOptions {
  statusCheck?: StatusCheck;
 }

-// @public
-export class TaskManager {
-  constructor(databaseManager: DatabaseManager, logger: Logger_2);
-  forPlugin(pluginId: string): PluginTaskManager;
-  // (undocumented)
-  static fromConfig(
-    config: Config,
-    options?: {
-      databaseManager?: DatabaseManager;
-      logger?: Logger_2;
-    },
-  ): TaskManager;
-}
-
-// @public
-export interface TaskOptions {
-  frequency?: Duration;
-  initialDelay?: Duration;
-  timeout?: Duration;
-}
-
 // @public
 export type UrlReader = {
  read(url: string): Promise<Buffer>;
@@ -41,7 +41,6 @@
    "@types/cors": "^2.8.6",
    "@types/dockerode": "^3.2.1",
    "@types/express": "^4.17.6",
-    "@types/luxon": "^2.0.4",
    "archiver": "^5.0.2",
    "aws-sdk": "^2.840.0",
    "compression": "^1.7.4",
@@ -60,7 +59,6 @@
    "knex": "^0.95.1",
    "lodash": "^4.17.21",
    "logform": "^2.1.1",
-    "luxon": "^2.0.2",
    "minimatch": "^3.0.4",
    "minimist": "^1.2.5",
    "morgan": "^1.10.0",
@@ -69,10 +67,8 @@
    "stoppable": "^1.1.0",
    "tar": "^6.1.2",
    "unzipper": "^0.10.11",
-    "uuid": "^8.0.0",
    "winston": "^3.2.1",
-    "yn": "^4.0.0",
-    "zod": "^3.9.5"
+    "yn": "^4.0.0"
  },
  "peerDependencies": {
    "pg-connection-string": "^2.3.0"
@@ -85,7 +81,6 @@
  "devDependencies": {
    "@backstage/backend-test-utils": "^0.1.8",
    "@backstage/cli": "^0.8.2",
-    "@backstage/test-utils": "^0.1.21",
    "@types/archiver": "^5.1.0",
    "@types/compression": "^1.7.0",
    "@types/concat-stream": "^1.6.0",
@@ -108,12 +103,10 @@
    "msw": "^0.35.0",
    "mysql2": "^2.2.5",
    "recursive-readdir": "^2.2.2",
-    "supertest": "^6.1.3",
-    "wait-for-expect": "^3.0.2"
+    "supertest": "^6.1.3"
  },
  "files": [
    "dist",
-    "migrations/**/*.{js,d.ts}",
    "config.d.ts"
  ],
  "configSchema": "config.d.ts"
@@ -31,5 +31,4 @@ export * from './paths';
 export * from './reading';
 export * from './scm';
 export * from './service';
-export * from './tasks';
 export * from './util';
@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};
@@ -0,0 +1,36 @@
+# @backstage/backend-tasks
+
+Common distributed task management / locking library for Backstage backends.
+
+## Usage
+
+Add the library to your backend package:
+
+```sh
+# From your Backstage root directory
+cd packages/backend
+yarn add @backstage/backend-tasks
+```
+
+then make use of its facilities as necessary:
+
+```typescript
+import { TaskManager } from '@backstage/backend-tasks';
+
+const manager = TaskManager.fromConfig(rootConfig).forPlugin('my-plugin');
+
+const { unschedule } = await manager.scheduleTask(
+  'refresh-things',
+  {
+    frequency: Duration.fromObject({ minutes: 10 }),
+  },
+  async () => {
+    await entityProvider.run();
+  },
+);
+```
+
+## Documentation
+
+- [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md)
+- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md)
@@ -0,0 +1,59 @@
+## API Report File for "@backstage/backend-tasks"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { Config } from '@backstage/config';
+import { DatabaseManager } from '@backstage/backend-common';
+import { Duration } from 'luxon';
+import { Logger as Logger_2 } from 'winston';
+
+// @public
+export interface LockOptions {
+  timeout: Duration;
+}
+
+// @public
+export interface PluginTaskManager {
+  acquireLock(
+    id: string,
+    options: LockOptions,
+  ): Promise<
+    | {
+        acquired: false;
+      }
+    | {
+        acquired: true;
+        release(): Promise<void>;
+      }
+  >;
+  scheduleTask(
+    id: string,
+    options: TaskOptions,
+    fn: () => void | Promise<void>,
+  ): Promise<{
+    unschedule: () => Promise<void>;
+  }>;
+}
+
+// @public
+export class TaskManager {
+  constructor(databaseManager: DatabaseManager, logger: Logger_2);
+  forPlugin(pluginId: string): PluginTaskManager;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options?: {
+      databaseManager?: DatabaseManager;
+      logger?: Logger_2;
+    },
+  ): TaskManager;
+}
+
+// @public
+export interface TaskOptions {
+  frequency?: Duration;
+  initialDelay?: Duration;
+  timeout?: Duration;
+}
+```
@@ -23,7 +23,7 @@ exports.up = async function up(knex) {
  //
  // mutexes
  //
-  await knex.schema.createTable('backstage_backend_common__mutexes', table => {
+  await knex.schema.createTable('backstage_backend_tasks__mutexes', table => {
    table.comment('Locks used for mutual exclusion among multiple workers');
    table
      .text('id')
@@ -32,7 +32,7 @@ exports.up = async function up(knex) {
      .comment('The unique ID of this particular mutex');
    table
      .text('current_lock_ticket')
-      .nullable()
+      .notNullable()
      .comment('A unique ticket for the current mutex lock');
    table
      .dateTime('current_lock_acquired_at')
@@ -42,12 +42,12 @@ exports.up = async function up(knex) {
      .dateTime('current_lock_expires_at')
      .nullable()
      .comment('The time when a locked mutex will time out and auto-release');
-    table.index(['id'], 'backstage_backend_common__mutexes__id_idx');
+    table.index(['id'], 'backstage_backend_tasks__mutexes__id_idx');
  });
  //
  // tasks
  //
-  await knex.schema.createTable('backstage_backend_common__tasks', table => {
+  await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
    table.comment('Tasks used for scheduling work on multiple workers');
    table
      .text('id')
@@ -74,7 +74,7 @@ exports.up = async function up(knex) {
      .dateTime('current_run_expires_at')
      .nullable()
      .comment('The time that the current task run will time out');
-    table.index(['id'], 'backstage_backend_common__tasks__id_idx');
+    table.index(['id'], 'backstage_backend_tasks__tasks__id_idx');
  });
 };

@@ -85,18 +85,15 @@ exports.down = async function down(knex) {
  //
  // tasks
  //
-  await knex.schema.alterTable('backstage_backend_common__tasks', table => {
-    table.dropIndex([], 'backstage_backend_common__tasks__id_idx');
+  await knex.schema.alterTable('backstage_backend_tasks__tasks', table => {
+    table.dropIndex([], 'backstage_backend_tasks__tasks__id_idx');
  });
-  await knex.schema.dropTable('backstage_backend_common__tasks');
+  await knex.schema.dropTable('backstage_backend_tasks__tasks');
  //
  // locks
  //
-  await knex.schema.alterTable(
-    'backstage_backend_common__task_locks',
-    table => {
-      table.dropIndex([], 'backstage_backend_common__task_locks__id_idx');
-    },
-  );
-  await knex.schema.dropTable('backstage_backend_common__task_locks');
+  await knex.schema.alterTable('backstage_backend_tasks__task_locks', table => {
+    table.dropIndex([], 'backstage_backend_tasks__task_locks__id_idx');
+  });
+  await knex.schema.dropTable('backstage_backend_tasks__task_locks');
 };
@@ -0,0 +1,54 @@
+{
+  "name": "@backstage/backend-tasks",
+  "description": "Common distributed task management / locking library for Backstage backends",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "packages/backend-tasks"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "license": "Apache-2.0",
+  "scripts": {
+    "build": "backstage-cli build --outputs cjs,types",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.9.7",
+    "@backstage/config": "^0.1.10",
+    "@backstage/errors": "^0.1.3",
+    "@backstage/types": "^0.1.1",
+    "@types/luxon": "^2.0.4",
+    "knex": "^0.95.1",
+    "lodash": "^4.17.21",
+    "luxon": "^2.0.2",
+    "uuid": "^8.0.0",
+    "winston": "^3.2.1",
+    "zod": "^3.9.5"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.7",
+    "@backstage/cli": "^0.8.0",
+    "jest": "^26.0.1",
+    "wait-for-expect": "^3.0.2"
+  },
+  "files": [
+    "dist",
+    "migrations/**/*.{js,d.ts}"
+  ]
+}
@@ -14,16 +14,16 @@
 * limitations under the License.
 */

+import { resolvePackagePath } from '@backstage/backend-common';
 import { Knex } from 'knex';
-import { resolvePackagePath } from '../paths';
 import { DB_MIGRATIONS_TABLE } from './tables';

 const migrationsDir = resolvePackagePath(
-  '@backstage/backend-common',
+  '@backstage/backend-tasks',
  'migrations',
 );

-export async function migrateBackendCommon(knex: Knex): Promise<void> {
+export async function migrateBackendTasks(knex: Knex): Promise<void> {
  await knex.migrate.latest({
    directory: migrationsDir,
    tableName: DB_MIGRATIONS_TABLE,
@@ -14,13 +14,13 @@
 * limitations under the License.
 */

-export const DB_MIGRATIONS_TABLE = 'backstage_backend_common__knex_migrations';
-export const DB_MUTEXES_TABLE = 'backstage_backend_common__mutexes';
-export const DB_TASKS_TABLE = 'backstage_backend_common__tasks';
+export const DB_MIGRATIONS_TABLE = 'backstage_backend_tasks__knex_migrations';
+export const DB_MUTEXES_TABLE = 'backstage_backend_tasks__mutexes';
+export const DB_TASKS_TABLE = 'backstage_backend_tasks__tasks';

 export type DbMutexesRow = {
  id: string;
-  current_lock_ticket?: string;
+  current_lock_ticket: string;
  current_lock_acquired_at?: Date | string;
  current_lock_expires_at?: Date | string;
 };
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common distributed task management / locking library for Backstage backends
+ *
+ * @packageDocumentation
+ */
+
+export * from './tasks';
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};
@@ -14,11 +14,11 @@
 * limitations under the License.
 */

+import { getVoidLogger } from '@backstage/backend-common';
 import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { Duration } from 'luxon';
 import waitForExpect from 'wait-for-expect';
-import { migrateBackendCommon } from '../database/migrateBackendCommon';
-import { getVoidLogger } from '../logging';
+import { migrateBackendTasks } from '../database/migrateBackendTasks';
 import { PluginTaskManagerImpl } from './PluginTaskManagerImpl';

 describe('PluginTaskManagerImpl', () => {
@@ -28,7 +28,7 @@ describe('PluginTaskManagerImpl', () => {

  async function init(databaseId: TestDatabaseId) {
    const knex = await databases.init(databaseId);
-    await migrateBackendCommon(knex);
+    await migrateBackendTasks(knex);
    const manager = new PluginTaskManagerImpl(
      async () => knex,
      getVoidLogger(),
@@ -14,10 +14,10 @@
 * limitations under the License.
 */

+import { isDatabaseConflictError } from '@backstage/backend-common';
 import { Knex } from 'knex';
 import { v4 as uuid } from 'uuid';
 import { Logger } from 'winston';
-import { isDatabaseConflictError } from '../database';
 import { DbMutexesRow, DB_MUTEXES_TABLE } from '../database/tables';
 import { TaskWorker } from './TaskWorker';
 import { LockOptions, PluginTaskManager, TaskOptions } from './types';
@@ -65,7 +65,6 @@ export class PluginTaskManagerImpl implements PluginTaskManager {
    // First try to overwrite an existing lock, that has timed out
    const stolen = await knex<DbMutexesRow>(DB_MUTEXES_TABLE)
      .where('id', '=', id)
-      .whereNotNull('current_lock_ticket')
      .where('current_lock_expires_at', '<', knex.fn.now())
      .update(record);

@@ -14,10 +14,9 @@
 * limitations under the License.
 */

+import { DatabaseManager, getVoidLogger } from '@backstage/backend-common';
 import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { Duration } from 'luxon';
-import { DatabaseManager } from '../database';
-import { getVoidLogger } from '../logging';
 import { TaskManager } from './TaskManager';

 describe('TaskManager', () => {
@@ -14,13 +14,12 @@
 * limitations under the License.
 */

+import { DatabaseManager, getRootLogger } from '@backstage/backend-common';
 import { Config } from '@backstage/config';
 import { memoize } from 'lodash';
 import { Duration } from 'luxon';
 import { Logger } from 'winston';
-import { DatabaseManager } from '../database';
-import { migrateBackendCommon } from '../database/migrateBackendCommon';
-import { getRootLogger } from '../logging';
+import { migrateBackendTasks } from '../database/migrateBackendTasks';
 import { PluginTaskManagerImpl } from './PluginTaskManagerImpl';
 import { PluginTaskManagerJanitor } from './PluginTaskManagerJanitor';
 import { PluginTaskManager } from './types';
@@ -61,7 +60,7 @@ export class TaskManager {
    const databaseFactory = memoize(async () => {
      const knex = await this.databaseManager.forPlugin(pluginId).getClient();

-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const janitor = new PluginTaskManagerJanitor({
        knex,
@@ -14,12 +14,12 @@
 * limitations under the License.
 */

+import { getVoidLogger } from '@backstage/backend-common';
 import { TestDatabases } from '@backstage/backend-test-utils';
 import { Duration } from 'luxon';
 import waitForExpect from 'wait-for-expect';
-import { migrateBackendCommon } from '../database/migrateBackendCommon';
+import { migrateBackendTasks } from '../database/migrateBackendTasks';
 import { DbTasksRow, DB_TASKS_TABLE } from '../database/tables';
-import { getVoidLogger } from '../logging';
 import { TaskWorker } from './TaskWorker';
 import { TaskSettingsV1 } from './types';

@@ -37,7 +37,7 @@ describe('TaskWorker', () => {
    'can run a single task to completion, %p',
    async databaseId => {
      const knex = await databases.init(databaseId);
-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const fn = jest.fn(
        async () => new Promise<void>(resolve => setTimeout(resolve, 50)),
@@ -60,7 +60,7 @@ describe('TaskWorker', () => {
    'goes through the expected states for a single run, %p',
    async databaseId => {
      const knex = await databases.init(databaseId);
-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const fn = jest.fn(
        async () => new Promise<void>(resolve => setTimeout(resolve, 50)),
@@ -135,7 +135,7 @@ describe('TaskWorker', () => {
    'runs tasks more than once even when the task throws, %p',
    async databaseId => {
      const knex = await databases.init(databaseId);
-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const fn = jest.fn().mockRejectedValue(new Error('failed'));
      const settings: TaskSettingsV1 = {
@@ -159,7 +159,7 @@ describe('TaskWorker', () => {
    'does not clobber ticket lock when stolen, %p',
    async databaseId => {
      const knex = await databases.init(databaseId);
-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const fn = jest.fn(
        async () => new Promise<void>(resolve => setTimeout(resolve, 50)),
@@ -212,7 +212,7 @@ describe('TaskWorker', () => {
    'gracefully handles a disappeared task row, %p',
    async databaseId => {
      const knex = await databases.init(databaseId);
-      await migrateBackendCommon(knex);
+      await migrateBackendTasks(knex);

      const fn = jest.fn(async () => {});
      const settings: TaskSettingsV1 = {