scaffolder: add the bare minimum of a db abstraction

plugins/scaffolder-backend/knexfile.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = {
+  development: {
+    client: 'sqlite3',
+    connection: ':memory:',
+    migrations: {
+      directory: 'migrations',
+    },
+  },
+};

plugins/scaffolder-backend/migrations/20210120143715_init.js

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.createTable('tasks', table => {
+    table.comment('The table of scaffolder tasks');
+    table.uuid('id').primary().notNullable().comment('The ID of the task');
+    table
+      .text('context')
+      .notNullable()
+      .comment('A JSON object with task specific context information');
+    table
+      .text('spec')
+      .notNullable()
+      .comment('A JSON encoded task specification');
+    table
+      .text('status')
+      .notNullable()
+      .comment('The current status of the task');
+    table
+      .integer('run_id')
+      .nullable()
+      .comment('The current run ID of the task');
+    table
+      .dateTime('created_at')
+      .notNullable()
+      .comment('The timestamp when this task was created');
+    table
+      .dateTime('last_heartbeat_at')
+      .nullable()
+      .comment('The last timestamp when a heartbeat was received');
+    table
+      .integer('retry_count')
+      .notNullable()
+      .defaultTo(0)
+      .comment('The number of times that this task has been attempted');
+  });
+  await knex.schema.createTable('task_events', table => {
+    table.comment('The event stream a given task');
+    table
+      .bigIncrements('id')
+      .primary()
+      .notNullable()
+      .comment('The ID of the event');
+    table
+      .uuid('task_id')
+      .references('id')
+      .inTable('tasks')
+      .notNullable()
+      .onDelete('CASCADE')
+      .comment('The task that generated the event');
+    table
+      .integer('run_id')
+      .nullable()
+      .comment('The run ID of the task that this event applies to');
+    table
+      .text('stage_name')
+      .nullable()
+      .comment('The stage of the task that this event applies to');
+    table.text('event_type').notNullable().comment('The type of event');
+    table
+      .dateTime('created_at')
+      .defaultTo(knex.fn.now())
+      .notNullable()
+      .comment('The timestamp when this event was generated');
+    table.text('text').notNullable().comment('The text of the event');
+    table.index(['task_id'], 'task_events_task_id_idx');
+  });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+  if (knex.client.config.client !== 'sqlite3') {
+    await knex.schema.alterTable('task_events', table => {
+      table.dropIndex([], 'task_events_task_id_idx');
+    });
+  }
+  await knex.schema.dropTable('task_events');
+  await knex.schema.dropTable('tasks');
+};

plugins/scaffolder-backend/package.json

@@ -53,7 +53,7 @@
     "helmet": "^4.0.0",
     "isomorphic-git": "^1.8.0",
     "jsonschema": "^1.2.6",
-    "luxon": "^1.25.0",
+    "knex": "^0.21.6",
     "morgan": "^1.10.0",
     "p-queue": "^6.3.0",
     "uuid": "^8.2.0",
@@ -73,6 +73,7 @@
   },
   "files": [
     "dist",
+    "migrations",
     "config.d.ts"
   ],
   "configSchema": "config.d.ts"

plugins/scaffolder-backend/src/tasks/Database.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConflictError, resolvePackagePath } from '@backstage/backend-common';
+import Knex from 'knex';
+import { Logger } from 'winston';
+import { Database, Transaction } from './types';
+
+const migrationsDir = resolvePackagePath(
+  '@backstage/plugin-scaffolder-backend',
+  'migrations',
+);
+
+export class CommonDatabase implements Database {
+  static async create(knex: Knex, logger: Logger): Promise<CommonDatabase> {
+    await knex.migrate.latest({
+      directory: migrationsDir,
+    });
+    return new CommonDatabase(knex, logger);
+  }
+
+  constructor(
+    private readonly database: Knex,
+    private readonly logger: Logger,
+  ) {}
+
+  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {
+    try {
+      let result: T | undefined = undefined;
+
+      await this.database.transaction(
+        async tx => {
+          // We can't return here, as knex swallows the return type in case the transaction is rolled back:
+          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
+          result = await fn(tx);
+        },
+        {
+          // If we explicitly trigger a rollback, don't fail.
+          doNotRejectOnRollback: true,
+        },
+      );
+
+      return result!;
+    } catch (e) {
+      this.logger.debug(`Error during transaction, ${e}`);
+
+      if (
+        /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) ||
+        /unique constraint/.test(e.message)
+      ) {
+        throw new ConflictError(`Rejected due to a conflicting entity`, e);
+      }
+
+      throw e;
+    }
+  }
+}

plugins/scaffolder-backend/src/tasks/types.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The core database implementation.
+ */
+export interface Database {
+  /**
+   * Runs a transaction.
+   *
+   * The callback is expected to make calls back into this class. When it
+   * completes, the transaction is closed.
+   *
+   * @param fn The callback that implements the transaction
+   */
+  transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
+}
+
+/**
+ * An abstraction for transactions of the underlying database technology.
+ */
+export type Transaction = {
+  rollback(): Promise<void>;
+};