diff --git a/plugins/scaffolder-backend/knexfile.js b/plugins/scaffolder-backend/knexfile.js new file mode 100644 index 0000000000..f469df4c08 --- /dev/null +++ b/plugins/scaffolder-backend/knexfile.js @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = { + development: { + client: 'sqlite3', + connection: ':memory:', + migrations: { + directory: 'migrations', + }, + }, +}; diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js new file mode 100644 index 0000000000..d2535bfa86 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -0,0 +1,100 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('tasks', table => { + table.comment('The table of scaffolder tasks'); + table.uuid('id').primary().notNullable().comment('The ID of the task'); + table + .text('context') + .notNullable() + .comment('A JSON object with task specific context information'); + table + .text('spec') + .notNullable() + .comment('A JSON encoded task specification'); + table + .text('status') + .notNullable() + .comment('The current status of the task'); + table + .integer('run_id') + .nullable() + .comment('The current run ID of the task'); + table + .dateTime('created_at') + .notNullable() + .comment('The timestamp when this task was created'); + table + .dateTime('last_heartbeat_at') + .nullable() + .comment('The last timestamp when a heartbeat was received'); + table + .integer('retry_count') + .notNullable() + .defaultTo(0) + .comment('The number of times that this task has been attempted'); + }); + await knex.schema.createTable('task_events', table => { + table.comment('The event stream a given task'); + table + .bigIncrements('id') + .primary() + .notNullable() + .comment('The ID of the event'); + table + .uuid('task_id') + .references('id') + .inTable('tasks') + .notNullable() + .onDelete('CASCADE') + .comment('The task that generated the event'); + table + .integer('run_id') + .nullable() + .comment('The run ID of the task that this event applies to'); + table + .text('stage_name') + .nullable() + .comment('The stage of the task that this event applies to'); + table.text('event_type').notNullable().comment('The type of event'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this event was generated'); + table.text('text').notNullable().comment('The text of the event'); + table.index(['task_id'], 'task_events_task_id_idx'); + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('task_events', table => { + table.dropIndex([], 'task_events_task_id_idx'); + }); + } + await knex.schema.dropTable('task_events'); + await knex.schema.dropTable('tasks'); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 97ab723e81..5d024c2798 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -53,7 +53,7 @@ "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", - "luxon": "^1.25.0", + "knex": "^0.21.6", "morgan": "^1.10.0", "p-queue": "^6.3.0", "uuid": "^8.2.0", @@ -73,6 +73,7 @@ }, "files": [ "dist", + "migrations", "config.d.ts" ], "configSchema": "config.d.ts" diff --git a/plugins/scaffolder-backend/src/tasks/Database.ts b/plugins/scaffolder-backend/src/tasks/Database.ts new file mode 100644 index 0000000000..c135380f87 --- /dev/null +++ b/plugins/scaffolder-backend/src/tasks/Database.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, resolvePackagePath } from '@backstage/backend-common'; +import Knex from 'knex'; +import { Logger } from 'winston'; +import { Database, Transaction } from './types'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-scaffolder-backend', + 'migrations', +); + +export class CommonDatabase implements Database { + static async create(knex: Knex, logger: Logger): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return new CommonDatabase(knex, logger); + } + + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} + + async transaction(fn: (tx: Transaction) => Promise): Promise { + try { + let result: T | undefined = undefined; + + await this.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.logger.debug(`Error during transaction, ${e}`); + + if ( + /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || + /unique constraint/.test(e.message) + ) { + throw new ConflictError(`Rejected due to a conflicting entity`, e); + } + + throw e; + } + } +} diff --git a/plugins/scaffolder-backend/src/tasks/types.ts b/plugins/scaffolder-backend/src/tasks/types.ts new file mode 100644 index 0000000000..27cbe7f7f1 --- /dev/null +++ b/plugins/scaffolder-backend/src/tasks/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The core database implementation. + */ +export interface Database { + /** + * Runs a transaction. + * + * The callback is expected to make calls back into this class. When it + * completes, the transaction is closed. + * + * @param fn The callback that implements the transaction + */ + transaction(fn: (tx: Transaction) => Promise): Promise; +} + +/** + * An abstraction for transactions of the underlying database technology. + */ +export type Transaction = { + rollback(): Promise; +};