From 877f46c1777f16acf84b3884a2a5c5ab59bf90d5 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 29 Jan 2021 16:15:34 -0500 Subject: [PATCH 01/78] Support custom status codes --- .../reader/components/TechDocsNotFound.test.tsx | 17 +++++++++++++++++ .../src/reader/components/TechDocsNotFound.tsx | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs/src/reader/components/TechDocsNotFound.test.tsx b/plugins/techdocs/src/reader/components/TechDocsNotFound.test.tsx index d3c98fec78..c6562ec8ad 100644 --- a/plugins/techdocs/src/reader/components/TechDocsNotFound.test.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsNotFound.test.tsx @@ -41,3 +41,20 @@ describe('', ( expect(rendered.getByTestId('go-back-link')).toBeDefined(); }); }); + +describe('', () => { + it('should render with a custom status code, custom error message and go back link', () => { + const rendered = render( + wrapInTestApp( + , + ), + ); + rendered.getByText(/This is a custom error message/i); + rendered.getByText(/500/i); + rendered.getByText(/Looks like someone dropped the mic!/i); + expect(rendered.getByTestId('go-back-link')).toBeDefined(); + }); +}); diff --git a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx index cdacc1cb7e..04f2106161 100644 --- a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx @@ -19,9 +19,10 @@ import { ErrorPage, useApi, configApiRef } from '@backstage/core'; type Props = { errorMessage?: string; + statusCode?: number; }; -export const TechDocsNotFound = ({ errorMessage }: Props) => { +export const TechDocsNotFound = ({ errorMessage, statusCode }: Props) => { const techdocsBuilder = useApi(configApiRef).getOptionalString( 'techdocs.builder', ); @@ -37,7 +38,7 @@ export const TechDocsNotFound = ({ errorMessage }: Props) => { return ( From cb61bc8cbb3c0d58dcfc4d74aaeeeb38bb6df263 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 29 Jan 2021 16:16:14 -0500 Subject: [PATCH 02/78] Hardcode file not found status --- plugins/techdocs/src/reader/components/Reader.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 1264e2b6eb..c9e76ddeeb 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -155,7 +155,7 @@ export const Reader = ({ entityId, onReady }: Props) => { ]); if (error) { - return ; + return ; } return ( From 45de779d5fa75721dd8bb1865c3e63a8ba8c4976 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 29 Jan 2021 16:20:07 -0500 Subject: [PATCH 03/78] Add changeset --- .changeset/green-rabbits-burn.md | 5 +++++ plugins/techdocs/src/api.ts | 24 ++++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 .changeset/green-rabbits-burn.md diff --git a/.changeset/green-rabbits-burn.md b/.changeset/green-rabbits-burn.md new file mode 100644 index 0000000000..5af68c2000 --- /dev/null +++ b/.changeset/green-rabbits-burn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Enhance API calls to support trapping 500 errors from techdocs-backend diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 3a950aeeb3..5f2cf924d7 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -119,14 +119,22 @@ export class TechDocsStorageApi implements TechDocsStorage { `${url.endsWith('/') ? url : `${url}/`}index.html`, ); - if (request.status === 404) { - let errorMessage = 'Page not found. '; - // path is empty for the home page of an entity's docs site - if (!path) { - errorMessage += - 'This could be because there is no index.md file in the root of the docs directory of this repository.'; - } - throw new Error(errorMessage); + let errorMessage = ''; + switch (request.status) { + case 404: + errorMessage = 'Page not found. '; + // path is empty for the home page of an entity's docs site + if (!path) { + errorMessage += + 'This could be because there is no index.md file in the root of the docs directory of this repository.'; + } + throw new Error(errorMessage); + case 500: + errorMessage = 'Could not generate documentation. '; + throw new Error(errorMessage); + default: + // Do nothing + break; } return request.text(); From 96dbdfbb41a65524411cce4544abd1f64fba4ded Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 31 Jan 2021 22:02:25 -0500 Subject: [PATCH 04/78] Update error message --- plugins/techdocs/src/api.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 5f2cf924d7..fc007503fa 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -130,7 +130,8 @@ export class TechDocsStorageApi implements TechDocsStorage { } throw new Error(errorMessage); case 500: - errorMessage = 'Could not generate documentation. '; + errorMessage = + 'Could not generate documentation or an error in the TechDocs backend. '; throw new Error(errorMessage); default: // Do nothing From 21e18aced4a1392053407dfab0fca997bdb50bf3 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 20 Jan 2021 17:18:07 +0100 Subject: [PATCH 05/78] Create inital class structure --- .../src/scaffolder/tasks/Database.ts | 35 +++++++++++++++++++ .../src/scaffolder/tasks/TaskObserver.ts | 15 ++++++++ .../src/scaffolder/tasks/taskWorker.ts | 34 ++++++++++++++++++ .../src/scaffolder/tasks/tasksDispatcher.ts | 29 +++++++++++++++ .../src/scaffolder/tasks/types.ts | 30 ++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/types.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts new file mode 100644 index 0000000000..f446f1e90a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Task, Status } from './types'; + +export class Database { + private readonly store = new Map(); + + get(taskId: string) { + return this.store.get(taskId); + } + + write(task: Task) { + return task; + } + + writeStatus(taskId: string, status: Status) { + const task = this.store.get(taskId); + if (task) { + this.store.set(taskId, { ...task, status }); + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts new file mode 100644 index 0000000000..863d6e76e1 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts new file mode 100644 index 0000000000..4b1754523c --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Status, ClaimResponse } from './types'; +import { Database } from './Database'; + +export class TaskWorker { + static async fromConfig() {} + + constructor(private readonly database: Database) {} + + claim(): Promise { + return Promise.resolve(undefined); + } + + setStatus(taskId: string, status: Status) { + this.database.writeStatus(taskId, status); + } + + heartbeat(runId: number) {} +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts new file mode 100644 index 0000000000..c3dfaa3c88 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Task } from './types'; +import { Database } from './Database'; + +export class taskDispatcher { + static async fromConfig(config: { database: Database }) {} + + constructor(private readonly db: Database) {} + + dispatch(task: Task): Promise { + this.db.write(task); + return Promise.resolve('uuid'); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts new file mode 100644 index 0000000000..000e292e16 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type Status = 'OPEN' | 'PROCESSING' | 'FAILED' | 'CANCELLED'; +export type Task = { + taskId: string; + metadata: string; + status: Status; + lastHeartbeat: string; + retryCount: number; + createdAt: string; +}; + +export type ClaimResponse = { + runId: number; + task: Task; +}; From 5e06a9f45b65f26b971ae7101dc2c2f709a4bba5 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 21 Jan 2021 15:53:01 +0100 Subject: [PATCH 06/78] Refactor into broker --- plugins/scaffolder-backend/package.json | 2 + .../src/scaffolder/tasks/Database.ts | 74 ++++++++++-- .../src/scaffolder/tasks/TaskBroker.test.ts | 63 ++++++++++ .../src/scaffolder/tasks/TaskBroker.ts | 109 ++++++++++++++++++ .../src/scaffolder/tasks/TaskObserver.ts | 15 --- .../src/scaffolder/tasks/taskWorker.ts | 34 ------ .../src/scaffolder/tasks/tasksDispatcher.ts | 29 ----- .../src/scaffolder/tasks/types.ts | 40 +++++-- 8 files changed, 272 insertions(+), 94 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 7af9f567e3..97ab723e81 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -53,7 +53,9 @@ "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", + "luxon": "^1.25.0", "morgan": "^1.10.0", + "p-queue": "^6.3.0", "uuid": "^8.2.0", "winston": "^3.2.1", "yaml": "^1.10.0" diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts index f446f1e90a..7c3f343b0f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts @@ -13,23 +13,79 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Task, Status } from './types'; +import { DbTaskRow, TaskSpec } from './types'; +import { v4 as uuid } from 'uuid'; +import { DateTime } from 'luxon'; -export class Database { - private readonly store = new Map(); +export interface Database { + get(taskId: string): Promise; + // updateTask(task: Task): Promise; + createTask(task: TaskSpec): Promise; + claimTask(): Promise; + heartBeat(runId: string): Promise; +} - get(taskId: string) { - return this.store.get(taskId); +export class InMemoryDatabase implements Database { + private readonly store = new Map(); + + async heartBeat(runId: string): Promise { + let task: DbTaskRow | undefined; + + for (const t of this.store.values()) { + if (t.runId === runId) { + task = t; + } + } + + if (!task) { + throw new Error('No task with matching runId found'); + } + + this.store.set(task.taskId, { + ...task, + lastHeartbeat: DateTime.local().toString(), + }); } - write(task: Task) { - return task; + async claimTask(): Promise { + for (const t of this.store.values()) { + if (t.status === 'OPEN') { + const task: DbTaskRow = { + ...t, + status: 'PROCESSING', + runId: uuid(), + }; + this.store.set(t.taskId, task); + return task; + } + } + throw new Error('No task found'); } - writeStatus(taskId: string, status: Status) { + async createTask(spec: TaskSpec): Promise { + return { + taskId: uuid(), + spec, + status: 'OPEN', + retryCount: 0, + createdAt: new Date().toISOString(), + }; + } + + async get(taskId: string): Promise { const task = this.store.get(taskId); if (task) { - this.store.set(taskId, { ...task, status }); + return task; } + throw new Error(`could not found task ${taskId}`); } + + // async updateTask(task: Task): Promise { + // if (!task.taskId) { + // throw new Error('Task must contain id'); + // } + + // this.store.set(task.taskId, task); + // return task; + // } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts new file mode 100644 index 0000000000..4fb7439bf8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MemoryTaskBroker, TaskAgent } from './TaskBroker'; + +describe('MemoryTaskBroker', () => { + it('should claim a dispatched work item', async () => { + const broker = new MemoryTaskBroker(); + + await broker.dispatch({}); + await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); + }); + + it('should wait for a dispatched work item', async () => { + const broker = new MemoryTaskBroker(); + + const promise = broker.claim(); + + await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); + + await broker.dispatch({}); + await expect(promise).resolves.toEqual(expect.any(TaskAgent)); + }); + + it('should dispatch multiple items and claim them in order', async () => { + const broker = new MemoryTaskBroker(); + + await broker.dispatch({ name: 'a' }); + await broker.dispatch({ name: 'b' }); + await broker.dispatch({ name: 'c' }); + + const taskA = await broker.claim(); + const taskB = await broker.claim(); + const taskC = await broker.claim(); + await expect(taskA).toEqual(expect.any(TaskAgent)); + await expect(taskB).toEqual(expect.any(TaskAgent)); + await expect(taskC).toEqual(expect.any(TaskAgent)); + await expect(taskA.spec.name).toBe('a'); + await expect(taskB.spec.name).toBe('b'); + await expect(taskC.spec.name).toBe('c'); + }); + + it('should complete a task', async () => { + const broker = new MemoryTaskBroker(); + + await broker.dispatch({}); + const task = await broker.claim(); + await task.complete('COMPLETED'); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts new file mode 100644 index 0000000000..322e669373 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CompletedTaskState, + Task, + TaskSpec, + TaskBroker, + Status, +} from './types'; +import { v4 as uuid } from 'uuid'; +import { InMemoryDatabase } from './database'; + +export class TaskAgent implements Task { + private heartbeartInterval?: ReturnType; + + static create(state: TaskState, db: InMemoryDatabase) { + const agent = new TaskAgent(state, db); + agent.start(); + return agent; + } + + // Runs heartbeat internally + private constructor( + private readonly state: TaskState, + private readonly db: InMemoryDatabase, + ) {} + + get spec() { + return this.state.spec; + } + + async emitLog(message: string): Promise { + throw new Error('Method not implemented.'); + } + + async complete(result: CompletedTaskState): Promise { + this.state.status = result === 'FAILED' ? 'COMPLETED' : 'FAILED'; + } + + private start() { + this.heartbeartInterval = setInterval(() => { + const runId = 'iiiid'; + this.db.heartBeat(runId); + }, 4269); + } +} + +interface TaskState { + spec: TaskSpec; + status: Status; + runId: string | undefined; +} + +function defer() { + let resolve = () => {}; + const promise = new Promise(_resolve => { + resolve = _resolve; + }); + return { promise, resolve }; +} + +export class MemoryTaskBroker implements TaskBroker { + private readonly db = new InMemoryDatabase(); + private readonly tasks = new Array(); + private deferredDispatch = defer(); + + async claim(): Promise { + for (;;) { + const pendingTask = await this.db.claimTask(); + if (pendingTask) { + return TaskAgent.create(pendingTask, this.db); + } + + await this.waitForDispatch(); + } + } + + async dispatch(spec: TaskSpec): Promise { + this.tasks.push({ + spec, + status: 'OPEN', + runId: undefined, + }); + this.signalDispatch(); + } + + private waitForDispatch() { + return this.deferredDispatch.promise; + } + + private signalDispatch() { + this.deferredDispatch.resolve(); + this.deferredDispatch = defer(); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts deleted file mode 100644 index 863d6e76e1..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskObserver.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts deleted file mode 100644 index 4b1754523c..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/taskWorker.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Status, ClaimResponse } from './types'; -import { Database } from './Database'; - -export class TaskWorker { - static async fromConfig() {} - - constructor(private readonly database: Database) {} - - claim(): Promise { - return Promise.resolve(undefined); - } - - setStatus(taskId: string, status: Status) { - this.database.writeStatus(taskId, status); - } - - heartbeat(runId: number) {} -} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts deleted file mode 100644 index c3dfaa3c88..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/tasksDispatcher.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Task } from './types'; -import { Database } from './Database'; - -export class taskDispatcher { - static async fromConfig(config: { database: Database }) {} - - constructor(private readonly db: Database) {} - - dispatch(task: Task): Promise { - this.db.write(task); - return Promise.resolve('uuid'); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 000e292e16..bd8bb0ade5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,17 +14,43 @@ * limitations under the License. */ -export type Status = 'OPEN' | 'PROCESSING' | 'FAILED' | 'CANCELLED'; -export type Task = { +export type Status = + | 'OPEN' + | 'PROCESSING' + | 'FAILED' + | 'CANCELLED' + | 'COMPLETED'; + +export type CompletedTaskState = 'FAILED' | 'COMPLETED'; + +export type DbTaskRow = { taskId: string; - metadata: string; + spec: TaskSpec; status: Status; - lastHeartbeat: string; + lastHeartbeat?: string; retryCount: number; createdAt: string; + runId?: string; +}; + +export type DbTaskEventRow = { + id: number; + runId?: number; + stageName: string; + createdAt: string; }; -export type ClaimResponse = { - runId: number; - task: Task; +export type TaskSpec = { + metadata: string; }; + +export interface Task { + spec: TaskSpec; + emitLog(message: string): Promise; + complete(result: CompletedTaskState): Promise; +} + +export interface TaskBroker { + claim(): Promise; + dispatch(spec: TaskSpec): Promise; +} From 5958318b322cba9c95464eadd5d617cde838d62b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 09:33:30 +0100 Subject: [PATCH 07/78] Implement createTask and setStatus --- .../src/scaffolder/tasks/Database.ts | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts index 7c3f343b0f..a3ce732586 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DbTaskRow, TaskSpec } from './types'; +import { DbTaskRow, Status, TaskSpec } from './types'; import { v4 as uuid } from 'uuid'; import { DateTime } from 'luxon'; export interface Database { get(taskId: string): Promise; - // updateTask(task: Task): Promise; createTask(task: TaskSpec): Promise; claimTask(): Promise; heartBeat(runId: string): Promise; + setStatus(taskId: string, status: Status): Promise; } export class InMemoryDatabase implements Database { @@ -63,13 +63,15 @@ export class InMemoryDatabase implements Database { } async createTask(spec: TaskSpec): Promise { - return { + const taskRow = { taskId: uuid(), spec, - status: 'OPEN', + status: 'OPEN' as Status, retryCount: 0, createdAt: new Date().toISOString(), }; + this.store.set(taskRow.taskId, taskRow); + return taskRow; } async get(taskId: string): Promise { @@ -80,12 +82,11 @@ export class InMemoryDatabase implements Database { throw new Error(`could not found task ${taskId}`); } - // async updateTask(task: Task): Promise { - // if (!task.taskId) { - // throw new Error('Task must contain id'); - // } - - // this.store.set(task.taskId, task); - // return task; - // } + async setStatus(taskId: string, status: Status): Promise { + const task = this.store.get(taskId); + if (!task) { + throw new Error(`no task found`); + } + this.store.set(task.taskId, { ...task, status }); + } } From a003fffbbb52a898166a91054a95f4ed51146ec6 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 09:34:05 +0100 Subject: [PATCH 08/78] Use InMemoryDatabase --- .../src/scaffolder/tasks/TaskBroker.test.ts | 18 +++++---- .../src/scaffolder/tasks/TaskBroker.ts | 40 +++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts index 4fb7439bf8..ef4b60cfd6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts @@ -20,7 +20,9 @@ describe('MemoryTaskBroker', () => { it('should claim a dispatched work item', async () => { const broker = new MemoryTaskBroker(); - await broker.dispatch({}); + await broker.dispatch({ + metadata: '', + }); await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); }); @@ -31,16 +33,16 @@ describe('MemoryTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); - await broker.dispatch({}); + await broker.dispatch({ metadata: 'foo' }); await expect(promise).resolves.toEqual(expect.any(TaskAgent)); }); it('should dispatch multiple items and claim them in order', async () => { const broker = new MemoryTaskBroker(); - await broker.dispatch({ name: 'a' }); - await broker.dispatch({ name: 'b' }); - await broker.dispatch({ name: 'c' }); + await broker.dispatch({ metadata: 'a' }); + await broker.dispatch({ metadata: 'b' }); + await broker.dispatch({ metadata: 'c' }); const taskA = await broker.claim(); const taskB = await broker.claim(); @@ -48,9 +50,9 @@ describe('MemoryTaskBroker', () => { await expect(taskA).toEqual(expect.any(TaskAgent)); await expect(taskB).toEqual(expect.any(TaskAgent)); await expect(taskC).toEqual(expect.any(TaskAgent)); - await expect(taskA.spec.name).toBe('a'); - await expect(taskB.spec.name).toBe('b'); - await expect(taskC.spec.name).toBe('c'); + await expect(taskA.spec.metadata).toBe('a'); + await expect(taskB.spec.metadata).toBe('b'); + await expect(taskC.spec.metadata).toBe('c'); }); it('should complete a task', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts index 322e669373..c6fb56b42a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts @@ -14,14 +14,7 @@ * limitations under the License. */ -import { - CompletedTaskState, - Task, - TaskSpec, - TaskBroker, - Status, -} from './types'; -import { v4 as uuid } from 'uuid'; +import { CompletedTaskState, Task, TaskSpec, TaskBroker } from './types'; import { InMemoryDatabase } from './database'; export class TaskAgent implements Task { @@ -48,20 +41,25 @@ export class TaskAgent implements Task { } async complete(result: CompletedTaskState): Promise { - this.state.status = result === 'FAILED' ? 'COMPLETED' : 'FAILED'; + this.db.setStatus( + this.state.taskId, + result === 'FAILED' ? 'COMPLETED' : 'FAILED', + ); } private start() { this.heartbeartInterval = setInterval(() => { - const runId = 'iiiid'; - this.db.heartBeat(runId); - }, 4269); + if (!this.state.runId) { + throw new Error('no run id provided'); + } + this.db.heartBeat(this.state.runId); + }, 1000); } } interface TaskState { spec: TaskSpec; - status: Status; + taskId: string; runId: string | undefined; } @@ -75,14 +73,20 @@ function defer() { export class MemoryTaskBroker implements TaskBroker { private readonly db = new InMemoryDatabase(); - private readonly tasks = new Array(); private deferredDispatch = defer(); async claim(): Promise { for (;;) { const pendingTask = await this.db.claimTask(); if (pendingTask) { - return TaskAgent.create(pendingTask, this.db); + return TaskAgent.create( + { + runId: pendingTask.runId, + taskId: pendingTask.taskId, + spec: pendingTask.spec, + }, + this.db, + ); } await this.waitForDispatch(); @@ -90,11 +94,7 @@ export class MemoryTaskBroker implements TaskBroker { } async dispatch(spec: TaskSpec): Promise { - this.tasks.push({ - spec, - status: 'OPEN', - runId: undefined, - }); + await this.db.createTask(spec); this.signalDispatch(); } From 971742b6cbdcf5e020cc9f76ebc4f8c66e76ccab Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 11:21:51 +0100 Subject: [PATCH 09/78] return undefined if no task is found --- plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts index a3ce732586..6f49a1f577 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts @@ -20,7 +20,7 @@ import { DateTime } from 'luxon'; export interface Database { get(taskId: string): Promise; createTask(task: TaskSpec): Promise; - claimTask(): Promise; + claimTask(): Promise; heartBeat(runId: string): Promise; setStatus(taskId: string, status: Status): Promise; } @@ -47,7 +47,7 @@ export class InMemoryDatabase implements Database { }); } - async claimTask(): Promise { + async claimTask(): Promise { for (const t of this.store.values()) { if (t.status === 'OPEN') { const task: DbTaskRow = { @@ -59,7 +59,7 @@ export class InMemoryDatabase implements Database { return task; } } - throw new Error('No task found'); + return undefined; } async createTask(spec: TaskSpec): Promise { From fcbfc56be8fcc5417219960a5a64e854151093d4 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 11:50:09 +0100 Subject: [PATCH 10/78] Add dispatchResult MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../src/scaffolder/tasks/TaskBroker.test.ts | 24 ++++++++----- .../src/scaffolder/tasks/TaskBroker.ts | 35 ++++++++++++------- .../src/scaffolder/tasks/types.ts | 6 +++- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts index ef4b60cfd6..388e2fdec8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { InMemoryDatabase } from './Database'; import { MemoryTaskBroker, TaskAgent } from './TaskBroker'; describe('MemoryTaskBroker', () => { - it('should claim a dispatched work item', async () => { - const broker = new MemoryTaskBroker(); + const storage = new InMemoryDatabase(); + const broker = new MemoryTaskBroker(storage); + it('should claim a dispatched work item', async () => { await broker.dispatch({ metadata: '', }); @@ -27,8 +29,6 @@ describe('MemoryTaskBroker', () => { }); it('should wait for a dispatched work item', async () => { - const broker = new MemoryTaskBroker(); - const promise = broker.claim(); await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); @@ -38,8 +38,6 @@ describe('MemoryTaskBroker', () => { }); it('should dispatch multiple items and claim them in order', async () => { - const broker = new MemoryTaskBroker(); - await broker.dispatch({ metadata: 'a' }); await broker.dispatch({ metadata: 'b' }); await broker.dispatch({ metadata: 'c' }); @@ -56,10 +54,18 @@ describe('MemoryTaskBroker', () => { }); it('should complete a task', async () => { - const broker = new MemoryTaskBroker(); - - await broker.dispatch({}); + const dispatchResult = await broker.dispatch({ metadata: 'foo' }); const task = await broker.claim(); await task.complete('COMPLETED'); + const taskRow = await storage.get(dispatchResult.taskId); + expect(taskRow.status).toBe('COMPLETED'); + }); + + it('should fail a task', async () => { + const dispatchResult = await broker.dispatch({ metadata: 'foo' }); + const task = await broker.claim(); + await task.complete('FAILED'); + const taskRow = await storage.get(dispatchResult.taskId); + expect(taskRow.status).toBe('FAILED'); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts index c6fb56b42a..6d6fb0a72f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts @@ -14,14 +14,20 @@ * limitations under the License. */ -import { CompletedTaskState, Task, TaskSpec, TaskBroker } from './types'; -import { InMemoryDatabase } from './database'; +import { + CompletedTaskState, + Task, + TaskSpec, + TaskBroker, + DispatchResult, +} from './types'; +import { InMemoryDatabase } from './Database'; export class TaskAgent implements Task { private heartbeartInterval?: ReturnType; - static create(state: TaskState, db: InMemoryDatabase) { - const agent = new TaskAgent(state, db); + static create(state: TaskState, storage: InMemoryDatabase) { + const agent = new TaskAgent(state, storage); agent.start(); return agent; } @@ -29,7 +35,7 @@ export class TaskAgent implements Task { // Runs heartbeat internally private constructor( private readonly state: TaskState, - private readonly db: InMemoryDatabase, + private readonly storage: InMemoryDatabase, ) {} get spec() { @@ -41,9 +47,9 @@ export class TaskAgent implements Task { } async complete(result: CompletedTaskState): Promise { - this.db.setStatus( + this.storage.setStatus( this.state.taskId, - result === 'FAILED' ? 'COMPLETED' : 'FAILED', + result === 'FAILED' ? 'FAILED' : 'COMPLETED', ); } @@ -52,7 +58,7 @@ export class TaskAgent implements Task { if (!this.state.runId) { throw new Error('no run id provided'); } - this.db.heartBeat(this.state.runId); + this.storage.heartBeat(this.state.runId); }, 1000); } } @@ -72,12 +78,12 @@ function defer() { } export class MemoryTaskBroker implements TaskBroker { - private readonly db = new InMemoryDatabase(); + constructor(private readonly storage: InMemoryDatabase) {} private deferredDispatch = defer(); async claim(): Promise { for (;;) { - const pendingTask = await this.db.claimTask(); + const pendingTask = await this.storage.claimTask(); if (pendingTask) { return TaskAgent.create( { @@ -85,7 +91,7 @@ export class MemoryTaskBroker implements TaskBroker { taskId: pendingTask.taskId, spec: pendingTask.spec, }, - this.db, + this.storage, ); } @@ -93,9 +99,12 @@ export class MemoryTaskBroker implements TaskBroker { } } - async dispatch(spec: TaskSpec): Promise { - await this.db.createTask(spec); + async dispatch(spec: TaskSpec): Promise { + const taskRow = await this.storage.createTask(spec); this.signalDispatch(); + return { + taskId: taskRow.taskId, + }; } private waitForDispatch() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index bd8bb0ade5..e8cea06005 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -44,6 +44,10 @@ export type TaskSpec = { metadata: string; }; +export type DispatchResult = { + taskId: string; +}; + export interface Task { spec: TaskSpec; emitLog(message: string): Promise; @@ -52,5 +56,5 @@ export interface Task { export interface TaskBroker { claim(): Promise; - dispatch(spec: TaskSpec): Promise; + dispatch(spec: TaskSpec): Promise; } From 06da4425e0ecb46920ad95c700864c15f4eda4ba Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 11:58:59 +0100 Subject: [PATCH 11/78] Prepend broker files with Memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../src/scaffolder/tasks/{Database.ts => MemoryDatabase.ts} | 2 +- .../tasks/{TaskBroker.test.ts => MemoryTaskBroker.test.ts} | 6 +++--- .../scaffolder/tasks/{TaskBroker.ts => MemoryTaskBroker.ts} | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename plugins/scaffolder-backend/src/scaffolder/tasks/{Database.ts => MemoryDatabase.ts} (97%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{TaskBroker.test.ts => MemoryTaskBroker.test.ts} (93%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{TaskBroker.ts => MemoryTaskBroker.ts} (94%) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts similarity index 97% rename from plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts index 6f49a1f577..65e64bf2a8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/Database.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts @@ -25,7 +25,7 @@ export interface Database { setStatus(taskId: string, status: Status): Promise; } -export class InMemoryDatabase implements Database { +export class MemoryDatabase implements Database { private readonly store = new Map(); async heartBeat(runId: string): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts similarity index 93% rename from plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts index 388e2fdec8..f1493ab3d0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { InMemoryDatabase } from './Database'; -import { MemoryTaskBroker, TaskAgent } from './TaskBroker'; +import { MemoryDatabase } from './MemoryDatabase'; +import { MemoryTaskBroker, TaskAgent } from './MemoryTaskBroker'; describe('MemoryTaskBroker', () => { - const storage = new InMemoryDatabase(); + const storage = new MemoryDatabase(); const broker = new MemoryTaskBroker(storage); it('should claim a dispatched work item', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts similarity index 94% rename from plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index 6d6fb0a72f..d29aeb9be6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -21,12 +21,12 @@ import { TaskBroker, DispatchResult, } from './types'; -import { InMemoryDatabase } from './Database'; +import { MemoryDatabase } from './MemoryDatabase'; export class TaskAgent implements Task { private heartbeartInterval?: ReturnType; - static create(state: TaskState, storage: InMemoryDatabase) { + static create(state: TaskState, storage: MemoryDatabase) { const agent = new TaskAgent(state, storage); agent.start(); return agent; @@ -35,7 +35,7 @@ export class TaskAgent implements Task { // Runs heartbeat internally private constructor( private readonly state: TaskState, - private readonly storage: InMemoryDatabase, + private readonly storage: MemoryDatabase, ) {} get spec() { From 4f67e8aa21c9d1455137990a493614731d7d7333 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 12:51:05 +0100 Subject: [PATCH 12/78] Fix typo --- .../scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index d29aeb9be6..a5d90898f5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -78,7 +78,7 @@ function defer() { } export class MemoryTaskBroker implements TaskBroker { - constructor(private readonly storage: InMemoryDatabase) {} + constructor(private readonly storage: MemoryDatabase) {} private deferredDispatch = defer(); async claim(): Promise { From 1c1ff59ebac9a8b150da6d47a1e07ac1a8f860d8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 22 Jan 2021 16:58:41 +0100 Subject: [PATCH 13/78] Add new apis using task broker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../scaffolder/tasks/MemoryDatabase.test.ts | 21 ++++ .../src/scaffolder/tasks/MemoryDatabase.ts | 50 +++++++- .../src/scaffolder/tasks/MemoryTaskBroker.ts | 54 ++++++++- .../src/scaffolder/tasks/TaskWorker.ts | 110 ++++++++++++++++++ .../src/scaffolder/tasks/index.ts | 19 +++ .../src/scaffolder/tasks/types.ts | 11 +- .../scaffolder-backend/src/service/router.ts | 73 ++++++++++++ 7 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/index.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts new file mode 100644 index 0000000000..c5522cb38a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +describe('MemoryDatabase', () => { + it('should be tested', async () => { + expect(1).toBe(2); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts index 65e64bf2a8..69fb6a1567 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts @@ -13,22 +13,62 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DbTaskRow, Status, TaskSpec } from './types'; + +import { DbTaskRow, DbTaskEventRow, Status, TaskSpec } from './types'; import { v4 as uuid } from 'uuid'; -import { DateTime } from 'luxon'; export interface Database { get(taskId: string): Promise; createTask(task: TaskSpec): Promise; claimTask(): Promise; - heartBeat(runId: string): Promise; + heartbeat(runId: string): Promise; setStatus(taskId: string, status: Status): Promise; } +type EmitOptions = { + taskId: string; + runId: string; + event: string; +}; + +type ReadOptions = { + taskId: string; + after?: number | undefined; +}; + export class MemoryDatabase implements Database { private readonly store = new Map(); + private readonly events = new Array(); - async heartBeat(runId: string): Promise { + async emit({ taskId, runId, event }: EmitOptions) { + this.events.push({ + id: this.events.length, + taskId, + runId, + event, + createdAt: new Date().toISOString(), + }); + } + + async getEvents({ + taskId, + after, + }: ReadOptions): Promise<{ events: DbTaskEventRow[] }> { + const events = this.events.filter(event => { + if (event.taskId !== taskId) { + return false; + } + if (after !== undefined) { + if (event.id <= after) { + return false; + } + } + return true; + }); + return { events }; + } + + async heartbeat(runId: string): Promise { let task: DbTaskRow | undefined; for (const t of this.store.values()) { @@ -43,7 +83,7 @@ export class MemoryDatabase implements Database { this.store.set(task.taskId, { ...task, - lastHeartbeat: DateTime.local().toString(), + lastHeartbeat: new Date().toISOString(), }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index a5d90898f5..2491c9a091 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -20,6 +20,7 @@ import { TaskSpec, TaskBroker, DispatchResult, + DbTaskEventRow, } from './types'; import { MemoryDatabase } from './MemoryDatabase'; @@ -43,14 +44,22 @@ export class TaskAgent implements Task { } async emitLog(message: string): Promise { - throw new Error('Method not implemented.'); + await this.storage.emit({ + taskId: this.state.taskId, + runId: this.state.runId, + event: message, + }); } async complete(result: CompletedTaskState): Promise { - this.storage.setStatus( + await this.storage.setStatus( this.state.taskId, result === 'FAILED' ? 'FAILED' : 'COMPLETED', ); + + if (this.heartbeartInterval) { + clearInterval(this.heartbeartInterval); + } } private start() { @@ -58,7 +67,7 @@ export class TaskAgent implements Task { if (!this.state.runId) { throw new Error('no run id provided'); } - this.storage.heartBeat(this.state.runId); + this.storage.heartbeat(this.state.runId); }, 1000); } } @@ -66,7 +75,7 @@ export class TaskAgent implements Task { interface TaskState { spec: TaskSpec; taskId: string; - runId: string | undefined; + runId: string; } function defer() { @@ -87,7 +96,7 @@ export class MemoryTaskBroker implements TaskBroker { if (pendingTask) { return TaskAgent.create( { - runId: pendingTask.runId, + runId: pendingTask.runId!, taskId: pendingTask.taskId, spec: pendingTask.spec, }, @@ -107,6 +116,41 @@ export class MemoryTaskBroker implements TaskBroker { }; } + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: (result: { events: DbTaskEventRow[] }) => void, + ): () => void { + const { taskId } = options; + + let cancelled = false; + const unsubscribe = () => { + cancelled = true; + }; + + (async () => { + let after = options.after; + while (!cancelled) { + const result = await this.storage.getEvents({ taskId, after: after }); + const { events } = result; + if (events.length) { + after = events[events.length - 1].id; + try { + callback(result); + } catch (error) { + console.log('DEBUG: error =', error); + } + } + + await new Promise(resolve => setTimeout(resolve, 1000)); + } + })(); + + return unsubscribe; + } + private waitForDispatch() { return this.deferredDispatch.promise; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts new file mode 100644 index 0000000000..dc15b3db65 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskBroker, Task } from './types'; +import { Logger } from 'winston'; +import Docker from 'dockerode'; +import { CatalogEntityClient } from '../../lib/catalog'; +import { + FilePreparer, + parseLocationAnnotation, + PreparerBuilder, + TemplaterBuilder, + PublisherBuilder, +} from '../stages'; + +type Options = { + logger: Logger; + taskBroker: TaskBroker; + workingDirectory: string; + dockerClient: Docker; + entityClient: CatalogEntityClient; + preparers: PreparerBuilder; + templaters: TemplaterBuilder; + publishers: PublisherBuilder; +}; + +export class TaskWorker { + constructor(private readonly options: Options) {} + + start() { + (async () => { + for (;;) { + const task = await this.options.taskBroker.claim(); + await this.runOneTask(task); + } + })(); + } + + async runOneTask(task: Task) { + const { + dockerClient, + preparers, + templaters, + publishers, + workingDirectory, + logger, + } = this.options; + + try { + task.emitLog('Task claimed, waiting ...'); + // Give us some time to curl observe + await new Promise(resolve => setTimeout(resolve, 5000)); + + const { values, template } = task.spec; + task.emitLog('Prepare the skeleton'); + const { protocol, location: pullPath } = parseLocationAnnotation( + task.spec.template, + ); + + const preparer = + protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); + const templater = templaters.get(template); + const publisher = publishers.get(values.storePath); + + const skeletonDir = await preparer.prepare(task.spec.template, { + logger, + workingDirectory: workingDirectory, + }); + + task.emitLog('Run the templater'); + const { resultDir } = await templater.run({ + directory: skeletonDir, + dockerClient, + logStream: process.stdout, // yay + values: values, + }); + + task.emitLog('Publish template'); + logger.info('Will now store the template'); + + logger.info('Totally storing the template now'); + await new Promise(resolve => setTimeout(resolve, 5000)); + // const result = await publisher.publish({ + // values: values, + // directory: resultDir, + // logger, + // }); + // task.emitLog(`Result: ${JSON.stringify(result)}`); + + task.emitLog(`Completely done now!`); + + await task.complete('COMPLETED'); + } catch (error) { + await task.complete('FAILED'); + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts new file mode 100644 index 0000000000..1cc9842fed --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { MemoryDatabase } from './MemoryDatabase'; +export { MemoryTaskBroker } from './MemoryTaskBroker'; +export { TaskWorker } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index e8cea06005..12cb9fee1f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { TemplaterValues } from '..'; + export type Status = | 'OPEN' | 'PROCESSING' @@ -35,13 +38,15 @@ export type DbTaskRow = { export type DbTaskEventRow = { id: number; - runId?: number; - stageName: string; + runId: string; + taskId: string; + event: string; createdAt: string; }; export type TaskSpec = { - metadata: string; + template: TemplateEntityV1alpha1; + values: TemplaterValues; }; export type DispatchResult = { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 9098e5d05e..706027f62e 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -33,6 +33,11 @@ import { import { CatalogEntityClient } from '../lib/catalog'; import { validate, ValidatorResult } from 'jsonschema'; import parseGitUrl from 'git-url-parse'; +import { + MemoryTaskBroker, + MemoryDatabase, + TaskWorker, +} from '../scaffolder/tasks'; export interface RouterOptions { preparers: PreparerBuilder; @@ -63,6 +68,18 @@ export async function createRouter( const logger = parentLogger.child({ plugin: 'scaffolder' }); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); + const taskBroker = new MemoryTaskBroker(new MemoryDatabase()); + const worker = new TaskWorker({ + logger, + taskBroker, + workingDirectory: 'todo', + dockerClient, + entityClient, + preparers, + publishers, + templaters, + }); + worker.start(); router .get('/v1/job/:jobId', ({ params }, res) => { @@ -88,6 +105,62 @@ export async function createRouter( error: job.error, }); }) + // curl -X POST -d '{"templateName":"springboot-template","values": {"storePath":"https://github.com/jhaals/foo", "component_id":"woop", "description": "apa", "owner": "me" }}' -H 'Content-Type: application/json' localhost:7000/api/scaffolder/v2/tasks + .post('/v2/tasks', async (req, res) => { + const templateName: string = req.body.templateName; + const values: TemplaterValues = { + ...req.body.values, + destination: { + git: parseGitUrl(req.body.values.storePath), + }, + }; + const template = await entityClient.findTemplate(templateName); + + const validationResult: ValidatorResult = validate( + values, + template.spec.schema, + ); + + if (!validationResult.valid) { + res.status(400).json({ errors: validationResult.errors }); + return; + } + const result = await taskBroker.dispatch({ + template, + values, + }); + + res.status(201).json({ id: result.taskId }); + }) + + .get('/v2/tasks/:taskId/eventstream', async (req, res) => { + const { taskId } = req.params; + const after = Number(req.query.after) || undefined; + logger.info('event stream opened'); + + // Mandatory headers and http status to keep connection open + res.writeHead(200, { + Connection: 'keep-alive', + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }); + // After client opens connection send all nests as string + const unsubscribe = taskBroker.observe( + { taskId, after }, + ({ events }) => { + for (const event of events) { + res.write(`event:${JSON.stringify(event)}\n\n`); + } + }, + ); + // When client closes connection we update the clients list + // avoiding the disconnected one + req.on('close', () => { + unsubscribe(); + logger.info('event stream closed'); + }); + }) + .post('/v1/jobs', async (req, res) => { const templateName: string = req.body.templateName; const values: TemplaterValues = { From dfea89fa98562ef30713eba9db5942d8351f3e70 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 25 Jan 2021 10:12:15 +0100 Subject: [PATCH 14/78] Pass taskSpec in tests --- .../scaffolder/tasks/MemoryTaskBroker.test.ts | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts index f1493ab3d0..aa23044749 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { TemplaterValues } from '../stages/templater/types'; import { MemoryDatabase } from './MemoryDatabase'; import { MemoryTaskBroker, TaskAgent } from './MemoryTaskBroker'; @@ -21,10 +23,13 @@ describe('MemoryTaskBroker', () => { const storage = new MemoryDatabase(); const broker = new MemoryTaskBroker(storage); + const taskSpec = { + values: {} as TemplaterValues, + template: {} as TemplateEntityV1alpha1, + }; + it('should claim a dispatched work item', async () => { - await broker.dispatch({ - metadata: '', - }); + await broker.dispatch(taskSpec); await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); }); @@ -33,14 +38,23 @@ describe('MemoryTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); - await broker.dispatch({ metadata: 'foo' }); + await broker.dispatch(taskSpec); await expect(promise).resolves.toEqual(expect.any(TaskAgent)); }); it('should dispatch multiple items and claim them in order', async () => { - await broker.dispatch({ metadata: 'a' }); - await broker.dispatch({ metadata: 'b' }); - await broker.dispatch({ metadata: 'c' }); + await broker.dispatch({ + values: { owner: 'a' } as TemplaterValues, + template: {} as TemplateEntityV1alpha1, + }); + await broker.dispatch({ + values: { owner: 'b' } as TemplaterValues, + template: {} as TemplateEntityV1alpha1, + }); + await broker.dispatch({ + values: { owner: 'c' } as TemplaterValues, + template: {} as TemplateEntityV1alpha1, + }); const taskA = await broker.claim(); const taskB = await broker.claim(); @@ -48,13 +62,13 @@ describe('MemoryTaskBroker', () => { await expect(taskA).toEqual(expect.any(TaskAgent)); await expect(taskB).toEqual(expect.any(TaskAgent)); await expect(taskC).toEqual(expect.any(TaskAgent)); - await expect(taskA.spec.metadata).toBe('a'); - await expect(taskB.spec.metadata).toBe('b'); - await expect(taskC.spec.metadata).toBe('c'); + await expect(taskA.spec.values.owner).toBe('a'); + await expect(taskB.spec.values.owner).toBe('b'); + await expect(taskC.spec.values.owner).toBe('c'); }); it('should complete a task', async () => { - const dispatchResult = await broker.dispatch({ metadata: 'foo' }); + const dispatchResult = await broker.dispatch(taskSpec); const task = await broker.claim(); await task.complete('COMPLETED'); const taskRow = await storage.get(dispatchResult.taskId); @@ -62,7 +76,7 @@ describe('MemoryTaskBroker', () => { }); it('should fail a task', async () => { - const dispatchResult = await broker.dispatch({ metadata: 'foo' }); + const dispatchResult = await broker.dispatch(taskSpec); const task = await broker.claim(); await task.complete('FAILED'); const taskRow = await storage.get(dispatchResult.taskId); From 7d13f22c3db614d9be40b6652e42fdb07bb5185f Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 25 Jan 2021 13:16:28 +0100 Subject: [PATCH 15/78] Add eventType for closing observer stream Co-authored-by: Patrik Oldsberg --- .../src/scaffolder/tasks/MemoryDatabase.ts | 22 +++++++++++++------ .../scaffolder/tasks/MemoryTaskBroker.test.ts | 8 +++---- .../src/scaffolder/tasks/MemoryTaskBroker.ts | 12 +++++++--- .../src/scaffolder/tasks/TaskWorker.ts | 4 ++-- .../src/scaffolder/tasks/types.ts | 16 ++++++++------ .../scaffolder-backend/src/service/router.ts | 4 ++++ 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts index 69fb6a1567..b2d1cf667b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { DbTaskRow, DbTaskEventRow, Status, TaskSpec } from './types'; +import { + DbTaskRow, + DbTaskEventRow, + Status, + TaskSpec, + TaskEventType, +} from './types'; import { v4 as uuid } from 'uuid'; export interface Database { @@ -28,7 +34,8 @@ export interface Database { type EmitOptions = { taskId: string; runId: string; - event: string; + body: string; + type: TaskEventType; }; type ReadOptions = { @@ -40,12 +47,13 @@ export class MemoryDatabase implements Database { private readonly store = new Map(); private readonly events = new Array(); - async emit({ taskId, runId, event }: EmitOptions) { + async emit({ taskId, runId, body, type }: EmitOptions) { this.events.push({ id: this.events.length, taskId, runId, - event, + body, + type, createdAt: new Date().toISOString(), }); } @@ -89,10 +97,10 @@ export class MemoryDatabase implements Database { async claimTask(): Promise { for (const t of this.store.values()) { - if (t.status === 'OPEN') { + if (t.status === 'open') { const task: DbTaskRow = { ...t, - status: 'PROCESSING', + status: 'processing', runId: uuid(), }; this.store.set(t.taskId, task); @@ -106,7 +114,7 @@ export class MemoryDatabase implements Database { const taskRow = { taskId: uuid(), spec, - status: 'OPEN' as Status, + status: 'open' as Status, retryCount: 0, createdAt: new Date().toISOString(), }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts index aa23044749..1ac416ad4f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts @@ -70,16 +70,16 @@ describe('MemoryTaskBroker', () => { it('should complete a task', async () => { const dispatchResult = await broker.dispatch(taskSpec); const task = await broker.claim(); - await task.complete('COMPLETED'); + await task.complete('completed'); const taskRow = await storage.get(dispatchResult.taskId); - expect(taskRow.status).toBe('COMPLETED'); + expect(taskRow.status).toBe('completed'); }); it('should fail a task', async () => { const dispatchResult = await broker.dispatch(taskSpec); const task = await broker.claim(); - await task.complete('FAILED'); + await task.complete('failed'); const taskRow = await storage.get(dispatchResult.taskId); - expect(taskRow.status).toBe('FAILED'); + expect(taskRow.status).toBe('failed'); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index 2491c9a091..c22bb0bddb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -47,16 +47,22 @@ export class TaskAgent implements Task { await this.storage.emit({ taskId: this.state.taskId, runId: this.state.runId, - event: message, + body: message, + type: 'log', }); } async complete(result: CompletedTaskState): Promise { await this.storage.setStatus( this.state.taskId, - result === 'FAILED' ? 'FAILED' : 'COMPLETED', + result === 'failed' ? 'failed' : 'completed', ); - + this.storage.emit({ + taskId: this.state.taskId, + runId: this.state.runId, + body: `Run completed with status: ${result}`, + type: 'completion', + }); if (this.heartbeartInterval) { clearInterval(this.heartbeartInterval); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index dc15b3db65..641a8cd9b1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -102,9 +102,9 @@ export class TaskWorker { task.emitLog(`Completely done now!`); - await task.complete('COMPLETED'); + await task.complete('completed'); } catch (error) { - await task.complete('FAILED'); + await task.complete('failed'); } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 12cb9fee1f..47226dc6e9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -18,13 +18,13 @@ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { TemplaterValues } from '..'; export type Status = - | 'OPEN' - | 'PROCESSING' - | 'FAILED' - | 'CANCELLED' - | 'COMPLETED'; + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; -export type CompletedTaskState = 'FAILED' | 'COMPLETED'; +export type CompletedTaskState = 'failed' | 'completed'; export type DbTaskRow = { taskId: string; @@ -36,11 +36,13 @@ export type DbTaskRow = { runId?: string; }; +export type TaskEventType = 'completion' | 'log'; export type DbTaskEventRow = { id: number; runId: string; taskId: string; - event: string; + body: string; + type: TaskEventType; createdAt: string; }; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 706027f62e..9514d215d0 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -150,6 +150,10 @@ export async function createRouter( ({ events }) => { for (const event of events) { res.write(`event:${JSON.stringify(event)}\n\n`); + if (event.type === 'completion') { + unsubscribe(); + res.end(); + } } }, ); From fef53431df1c3745bfb210eb6b8cf8a637b75213 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 25 Jan 2021 14:02:51 +0100 Subject: [PATCH 16/78] create logger and stream that emit events --- .../src/scaffolder/tasks/TaskWorker.ts | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 641a8cd9b1..d500fc8e5c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -14,9 +14,11 @@ * limitations under the License. */ -import { TaskBroker, Task } from './types'; +import { PassThrough } from 'stream'; import { Logger } from 'winston'; +import * as winston from 'winston'; import Docker from 'dockerode'; +import { TaskBroker, Task } from './types'; import { CatalogEntityClient } from '../../lib/catalog'; import { FilePreparer, @@ -59,6 +61,24 @@ export class TaskWorker { logger, } = this.options; + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', data => { + const message = data.toString().trim(); + if (message?.length > 1) task.emitLog(message); + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + try { task.emitLog('Task claimed, waiting ...'); // Give us some time to curl observe @@ -76,7 +96,7 @@ export class TaskWorker { const publisher = publishers.get(values.storePath); const skeletonDir = await preparer.prepare(task.spec.template, { - logger, + logger: taskLogger, workingDirectory: workingDirectory, }); @@ -84,7 +104,7 @@ export class TaskWorker { const { resultDir } = await templater.run({ directory: skeletonDir, dockerClient, - logStream: process.stdout, // yay + logStream: stream, values: values, }); @@ -100,8 +120,6 @@ export class TaskWorker { // }); // task.emitLog(`Result: ${JSON.stringify(result)}`); - task.emitLog(`Completely done now!`); - await task.complete('completed'); } catch (error) { await task.complete('failed'); From 8623025c9b789b85bdf87b9cdf9cc94995f55859 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 26 Jan 2021 12:00:40 +0100 Subject: [PATCH 17/78] Wip wip --- .../src/scaffolder/stages/legacy.ts | 106 +++++++++++++++++ .../scaffolder/tasks/MemoryTaskBroker.test.ts | 2 +- .../src/scaffolder/tasks/TemplateConverter.ts | 110 ++++++++++++++++++ .../src/scaffolder/tasks/types.ts | 11 +- .../scaffolder-backend/src/service/router.ts | 2 + 5 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts new file mode 100644 index 0000000000..c9bd89c3dd --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -0,0 +1,106 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { TemplateActionRegistry } from '../TemplateConverter'; +import { FilePreparer } from './prepare'; +import Docker from 'dockerode'; + +type Options = { + logger: Logger; + config: Config; + dockerClient: Docker; +}; + +export function registerLegacyActions( + registry: TemplateActionRegistry, + options: Options, +) { + registry.register({ + id: 'legacy:prepare', + async handler(ctx) { + const { logger } = ctx; + console.log(ctx); + logger.info('Task claimed, waiting ...'); + // Give us some time to curl observe + await new Promise(resolve => setTimeout(resolve, 5000)); + + logger.info('Prepare the skeleton'); + + const { protocol, pullPath } = ctx.parameters; + + const preparer = + protocol === 'file' + ? new FilePreparer() + : preparers.get(pullPath as string); + + await preparer.prepare(task.spec.template, { + logger, + ctx.workspaceDir, + }); + ctx.output('catalogInfoUrl', 'httpderp://asdasd'); + }, + }); + + // try { + // const { values, template } = task.spec; + // task.emitLog('Prepare the skeleton'); + // const { protocol, location: pullPath } = parseLocationAnnotation( + // task.spec.template, + // ); + // const preparer = + // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); + // const templater = templaters.get(template); + // const publisher = publishers.get(values.storePath); + + // const skeletonDir = await preparer.prepare(task.spec.template, { + // logger: taskLogger, + // workingDirectory: workingDirectory, + // }); + + registry.register({ + id: 'legacy:template', + async handler(ctx) { + const { logger } = ctx; + + const templater = templaters.get(ctx.parameters.templater as string); + + logger.info('Run the templater'); + const { resultDir } = await templater.run({ + directory: ctx.workspaceDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.parameters.values as TemplaterValues, + }); + }, + }); + // task.emitLog('Publish template'); + // logger.info('Will now store the template'); + + // logger.info('Totally storing the template now'); + // await new Promise(resolve => setTimeout(resolve, 5000)); + // // const result = await publisher.publish({ + // // values: values, + // // directory: resultDir, + // // logger, + // // }); + // // task.emitLog(`Result: ${JSON.stringify(result)}`); + + // await task.complete('completed'); + // } catch (error) { + // await task.complete('failed'); + // } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts index 1ac416ad4f..686a6d9132 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts @@ -15,7 +15,7 @@ */ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { TemplaterValues } from '../stages/templater/types'; +import { TemplaterValues } from './actions/templater/types'; import { MemoryDatabase } from './MemoryDatabase'; import { MemoryTaskBroker, TaskAgent } from './MemoryTaskBroker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts new file mode 100644 index 0000000000..e435812fe5 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonValue } from '@backstage/config'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import type { Writable } from 'stream'; +import { + getTemplaterKey, + parseLocationAnnotation, + TemplaterValues, +} from '../jobs/actions'; +import { TaskSpec } from './types'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; + +function templateEntityToSpec( + template: TemplateEntityV1alpha1, + values: TemplaterValues, +): TaskSpec { + const steps: TaskSpec['steps'] = []; + + const { protocol, location: pullPath } = parseLocationAnnotation(template); + const templater = getTemplaterKey(template); + + steps.push({ + id: 'prepare', + name: 'Prepare', + action: 'legacy:prepare', + parameters: { + protocol, + pullPath, + }, + }); + + steps.push({ + id: 'template', + name: 'Template', + action: 'legacy:template', + parameters: { + templater, + values, + }, + }); + + steps.push({ + id: 'publish', + name: 'Publishing', + action: 'publish', + parameters: { + values, + directory, + }, + }); + + return { steps }; +} + +type ActionContext = { + logger: Logger; + logStream: Writable; + + workspaceDir: string; + parameters: { [name: string]: JsonValue }; + output(name: string, value: JsonValue): void; +}; + +type TemplateAction = { + id: string; + handler: (ctx: ActionContext) => Promise; +}; + +export class TemplateActionRegistry { + private readonly actions = new Map(); + + register(action: TemplateAction) { + if (this.actions.has(action.id)) { + throw new ConflictError( + `Template action with id ${action.id} as already been registered`, + ); + } + this.actions.set(action.id, action); + } + + // validate + // ensure that action exist. + // template variables exist. + + get(actionId: string): TemplateAction { + const action = this.actions.get(actionId); + if (!action) { + throw new NotFoundError( + `Template action with id ${actionId} is not registered.`, + ); + } + return action; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 47226dc6e9..620b6e7243 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,8 +14,7 @@ * limitations under the License. */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { TemplaterValues } from '..'; +import { JsonObject } from '@backstage/config'; export type Status = | 'open' @@ -47,8 +46,12 @@ export type DbTaskEventRow = { }; export type TaskSpec = { - template: TemplateEntityV1alpha1; - values: TemplaterValues; + steps: Array<{ + id: string; + name: string; + action: string; + parameters?: JsonObject; + }>; }; export type DispatchResult = { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 9514d215d0..fc40d45ce2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -38,6 +38,8 @@ import { MemoryDatabase, TaskWorker, } from '../scaffolder/tasks'; +import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateConverter'; +import { LOCATION_ANNOTATION } from '@backstage/catalog-model'; export interface RouterOptions { preparers: PreparerBuilder; From cf896972ef873b4a94f669d10b9a8f5a3e186e00 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Jan 2021 13:16:50 +0100 Subject: [PATCH 18/78] Wire up task registry with legacy tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../src/scaffolder/stages/legacy.ts | 105 ++++++------ .../src/scaffolder/tasks/MemoryTaskBroker.ts | 8 + .../src/scaffolder/tasks/TaskWorker.ts | 155 ++++++++++-------- .../src/scaffolder/tasks/TemplateConverter.ts | 31 ++-- .../src/scaffolder/tasks/types.ts | 6 +- .../scaffolder-backend/src/service/helpers.ts | 44 +++++ .../scaffolder-backend/src/service/router.ts | 23 ++- 7 files changed, 242 insertions(+), 130 deletions(-) create mode 100644 plugins/scaffolder-backend/src/service/helpers.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index c9bd89c3dd..6244e40d68 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,21 +14,25 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import { TemplateActionRegistry } from '../TemplateConverter'; -import { FilePreparer } from './prepare'; +import { TemplateActionRegistry } from '../tasks/TemplateConverter'; +import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; +import { TemplaterBuilder, TemplaterValues } from './templater'; +import { PublisherBuilder } from './publish'; type Options = { - logger: Logger; - config: Config; dockerClient: Docker; + preparers: PreparerBuilder; + templaters: TemplaterBuilder; + publishers: PublisherBuilder; }; export function registerLegacyActions( registry: TemplateActionRegistry, options: Options, ) { + const { dockerClient, preparers, templaters, publishers } = options; + registry.register({ id: 'legacy:prepare', async handler(ctx) { @@ -41,36 +45,18 @@ export function registerLegacyActions( logger.info('Prepare the skeleton'); const { protocol, pullPath } = ctx.parameters; - + const url = pullPath as string; const preparer = - protocol === 'file' - ? new FilePreparer() - : preparers.get(pullPath as string); + protocol === 'file' ? new FilePreparer() : preparers.get(url); - await preparer.prepare(task.spec.template, { - logger, - ctx.workspaceDir, + await preparer.prepare({ + url, + logger: ctx.logger, + workspacePath: ctx.workspacePath, }); - ctx.output('catalogInfoUrl', 'httpderp://asdasd'); }, }); - // try { - // const { values, template } = task.spec; - // task.emitLog('Prepare the skeleton'); - // const { protocol, location: pullPath } = parseLocationAnnotation( - // task.spec.template, - // ); - // const preparer = - // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); - // const templater = templaters.get(template); - // const publisher = publishers.get(values.storePath); - - // const skeletonDir = await preparer.prepare(task.spec.template, { - // logger: taskLogger, - // workingDirectory: workingDirectory, - // }); - registry.register({ id: 'legacy:template', async handler(ctx) { @@ -79,28 +65,55 @@ export function registerLegacyActions( const templater = templaters.get(ctx.parameters.templater as string); logger.info('Run the templater'); - const { resultDir } = await templater.run({ - directory: ctx.workspaceDir, + await templater.run({ + workspacePath: ctx.workspacePath, dockerClient, logStream: ctx.logStream, values: ctx.parameters.values as TemplaterValues, }); }, }); - // task.emitLog('Publish template'); - // logger.info('Will now store the template'); - // logger.info('Totally storing the template now'); - // await new Promise(resolve => setTimeout(resolve, 5000)); - // // const result = await publisher.publish({ - // // values: values, - // // directory: resultDir, - // // logger, - // // }); - // // task.emitLog(`Result: ${JSON.stringify(result)}`); - - // await task.complete('completed'); - // } catch (error) { - // await task.complete('failed'); - // } + registry.register({ + id: 'legacy:publish', + async handler(ctx) { + const { values } = ctx.parameters; + if ( + typeof values !== 'object' || + values === null || + Array.isArray(values) + ) { + throw new Error( + `Invalid values passed to publish, got ${typeof values}`, + ); + } + const storePath = values.storePath as unknown; + if (typeof storePath !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof storePath}`, + ); + } + const owner = values.owner as unknown; + if (typeof owner !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof owner}`, + ); + } + const publisher = publishers.get(storePath); + ctx.logger.info('Will now store the template'); + const { remoteUrl, catalogInfoUrl } = await publisher.publish({ + values: { + ...values, + owner, + storePath, + }, + workspacePath: ctx.workspacePath, + logger: ctx.logger, + }); + ctx.output('remoteUrl', remoteUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index c22bb0bddb..c5920fe836 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -43,6 +43,14 @@ export class TaskAgent implements Task { return this.state.spec; } + get runId() { + return this.state.runId; + } + + get taskId() { + return this.state.taskId; + } + async emitLog(message: string): Promise { await this.storage.emit({ taskId: this.state.taskId, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index d500fc8e5c..6eb1c4763c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -17,26 +17,17 @@ import { PassThrough } from 'stream'; import { Logger } from 'winston'; import * as winston from 'winston'; -import Docker from 'dockerode'; +import { JsonValue } from '@backstage/config'; import { TaskBroker, Task } from './types'; -import { CatalogEntityClient } from '../../lib/catalog'; -import { - FilePreparer, - parseLocationAnnotation, - PreparerBuilder, - TemplaterBuilder, - PublisherBuilder, -} from '../stages'; +import { TemplateActionRegistry } from './TemplateConverter'; +import fs from 'fs-extra'; +import path from 'path'; type Options = { logger: Logger; taskBroker: TaskBroker; workingDirectory: string; - dockerClient: Docker; - entityClient: CatalogEntityClient; - preparers: PreparerBuilder; - templaters: TemplaterBuilder; - publishers: PublisherBuilder; + actionRegistry: TemplateActionRegistry; }; export class TaskWorker { @@ -52,66 +43,90 @@ export class TaskWorker { } async runOneTask(task: Task) { - const { - dockerClient, - preparers, - templaters, - publishers, - workingDirectory, - logger, - } = this.options; - - const taskLogger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: {}, - }); - - const stream = new PassThrough(); - stream.on('data', data => { - const message = data.toString().trim(); - if (message?.length > 1) task.emitLog(message); - }); - - taskLogger.add(new winston.transports.Stream({ stream })); - try { - task.emitLog('Task claimed, waiting ...'); + const { actionRegistry, logger } = this.options; + + // bbl LUUUNCH + // taskID and runId not part of task? O_o + const workspacePath = await this.createWorkPath(task.taskId, task.runId); + + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', data => { + const message = data.toString().trim(); + if (message?.length > 1) task.emitLog(message); + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + // Give us some time to curl observe + task.emitLog('Task claimed, waiting ...'); await new Promise(resolve => setTimeout(resolve, 5000)); + task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); - const { values, template } = task.spec; - task.emitLog('Prepare the skeleton'); - const { protocol, location: pullPath } = parseLocationAnnotation( - task.spec.template, - ); + const outputs: { [name: string]: JsonValue } = {}; - const preparer = - protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); - const templater = templaters.get(template); - const publisher = publishers.get(values.storePath); + for (const step of task.spec.steps) { + task.emitLog(`Beginning step ${step.name}`); - const skeletonDir = await preparer.prepare(task.spec.template, { - logger: taskLogger, - workingDirectory: workingDirectory, - }); + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } - task.emitLog('Run the templater'); - const { resultDir } = await templater.run({ - directory: skeletonDir, - dockerClient, - logStream: stream, - values: values, - }); + // TODO: substitute any placeholders with output from previous steps + const parameters = step.parameters!; - task.emitLog('Publish template'); - logger.info('Will now store the template'); + await action.handler({ + logger, + logStream: stream, + parameters, + workspacePath, + output(name: string, value: JsonValue) { + outputs[name] = value; + }, + }); - logger.info('Totally storing the template now'); + task.emitLog(`Finished step ${step.name}`); + } + + // const { values, template } = task.spec; + // task.emitLog('Prepare the skeleton'); + // const { protocol, location: pullPath } = parseLocationAnnotation( + // task.spec.template, + // ); + + // const preparer = + // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); + // const templater = templaters.get(template); + // const publisher = publishers.get(values.storePath); + + // const skeletonDir = await preparer.prepare(task.spec.template, { + // logger: taskLogger, + // workingDirectory: workingDirectory, + // }); + + // task.emitLog('Run the templater'); + // const { resultDir } = await templater.run({ + // directory: skeletonDir, + // dockerClient, + // logStream: stream, + // values: values, + // }); + + // task.emitLog('Publish template'); + // logger.info('Will now store the template'); + + logger.info('So done right now'); await new Promise(resolve => setTimeout(resolve, 5000)); // const result = await publisher.publish({ // values: values, @@ -125,4 +140,14 @@ export class TaskWorker { await task.complete('failed'); } } + + async createWorkPath(taskId: string, runId: string): Promise { + const workspacePath = path.join( + this.options.workingDirectory, + taskId, + runId, + ); + fs.ensureDir(workspacePath); + return workspacePath; + } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index e435812fe5..bd06dbaa91 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -14,25 +14,37 @@ * limitations under the License. */ +import { resolve as resolvePath } from 'path'; import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { Logger } from 'winston'; import type { Writable } from 'stream'; -import { - getTemplaterKey, - parseLocationAnnotation, - TemplaterValues, -} from '../jobs/actions'; + import { TaskSpec } from './types'; import { ConflictError, NotFoundError } from '@backstage/backend-common'; +import { + getTemplaterKey, + joinGitUrlPath, + parseLocationAnnotation, + TemplaterValues, +} from '../stages'; -function templateEntityToSpec( +export function templateEntityToSpec( template: TemplateEntityV1alpha1, values: TemplaterValues, ): TaskSpec { const steps: TaskSpec['steps'] = []; - const { protocol, location: pullPath } = parseLocationAnnotation(template); + const { protocol, location } = parseLocationAnnotation(template); + + let url: string; + if (protocol === 'file') { + const path = resolvePath(location, template.spec.path || '.'); + + url = `file://${path}`; + } else { + url = joinGitUrlPath(location, template.spec.path); + } const templater = getTemplaterKey(template); steps.push({ @@ -41,7 +53,7 @@ function templateEntityToSpec( action: 'legacy:prepare', parameters: { protocol, - pullPath, + url, }, }); @@ -61,7 +73,6 @@ function templateEntityToSpec( action: 'publish', parameters: { values, - directory, }, }); @@ -72,7 +83,7 @@ type ActionContext = { logger: Logger; logStream: Writable; - workspaceDir: string; + workspacePath: string; parameters: { [name: string]: JsonValue }; output(name: string, value: JsonValue): void; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 620b6e7243..496385d4ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { JsonObject } from '@backstage/config'; +import { JsonValue } from '@backstage/config'; export type Status = | 'open' @@ -50,7 +50,7 @@ export type TaskSpec = { id: string; name: string; action: string; - parameters?: JsonObject; + parameters?: { [name: string]: JsonValue }; }>; }; @@ -60,6 +60,8 @@ export type DispatchResult = { export interface Task { spec: TaskSpec; + taskId: string; + runId: string; emitLog(message: string): Promise; complete(result: CompletedTaskState): Promise; } diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts new file mode 100644 index 0000000000..dd3d43c6d1 --- /dev/null +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import fs from 'fs-extra'; +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +export async function getWorkingDirectory( + config: Config, + logger: Logger, +): Promise { + if (!config.has('backend.workingDirectory')) { + return os.tmpdir(); + } + + const workingDirectory = config.getString('backend.workingDirectory'); + try { + // Check if working directory exists and is writable + await fs.access(workingDirectory, fs.constants.F_OK | fs.constants.W_OK); + logger.info(`using working directory: ${workingDirectory}`); + } catch (err) { + logger.error( + `working directory ${workingDirectory} ${ + err.code === 'ENOENT' ? 'does not exist' : 'is not writable' + }`, + ); + throw err; + } + return workingDirectory; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fc40d45ce2..a72a5c5d48 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -38,8 +38,13 @@ import { MemoryDatabase, TaskWorker, } from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateConverter'; +import { + TemplateActionRegistry, + templateEntityToSpec, +} from '../scaffolder/tasks/TemplateConverter'; import { LOCATION_ANNOTATION } from '@backstage/catalog-model'; +import { registerLegacyActions } from '../scaffolder/stages/legacy'; +import { getWorkingDirectory } from './helpers'; export interface RouterOptions { preparers: PreparerBuilder; @@ -69,18 +74,24 @@ export async function createRouter( } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); + const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); const taskBroker = new MemoryTaskBroker(new MemoryDatabase()); + const actionRegistry = new TemplateActionRegistry(); const worker = new TaskWorker({ logger, taskBroker, - workingDirectory: 'todo', + actionRegistry, + workingDirectory, + }); + + registerLegacyActions(actionRegistry, { dockerClient, - entityClient, preparers, publishers, templaters, }); + worker.start(); router @@ -127,10 +138,8 @@ export async function createRouter( res.status(400).json({ errors: validationResult.errors }); return; } - const result = await taskBroker.dispatch({ - template, - values, - }); + const taskSpec = templateEntityToSpec(template, values); + const result = await taskBroker.dispatch(taskSpec); res.status(201).json({ id: result.taskId }); }) From abff060e67dbacacdf3343a20df927390fd8b802 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Jan 2021 15:56:29 +0100 Subject: [PATCH 19/78] Use url from parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../scaffolder-backend/src/scaffolder/stages/legacy.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index 6244e40d68..a3fe01c96d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -40,17 +40,15 @@ export function registerLegacyActions( console.log(ctx); logger.info('Task claimed, waiting ...'); // Give us some time to curl observe - await new Promise(resolve => setTimeout(resolve, 5000)); + await new Promise(resolve => setTimeout(resolve, 1000)); logger.info('Prepare the skeleton'); - - const { protocol, pullPath } = ctx.parameters; - const url = pullPath as string; + const { protocol, url } = ctx.parameters; const preparer = - protocol === 'file' ? new FilePreparer() : preparers.get(url); + protocol === 'file' ? new FilePreparer() : preparers.get(url as string); await preparer.prepare({ - url, + url: url as string, logger: ctx.logger, workspacePath: ctx.workspacePath, }); From 7b7230f75bdd1d7a4c7513f11e9451b58f1f41f1 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Jan 2021 15:57:37 +0100 Subject: [PATCH 20/78] Add getWorkspaceName --- .../src/scaffolder/tasks/MemoryTaskBroker.ts | 8 +-- .../src/scaffolder/tasks/TaskWorker.ts | 54 +++---------------- .../src/scaffolder/tasks/types.ts | 3 +- 3 files changed, 10 insertions(+), 55 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index c5920fe836..6937c535dc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -43,12 +43,8 @@ export class TaskAgent implements Task { return this.state.spec; } - get runId() { - return this.state.runId; - } - - get taskId() { - return this.state.taskId; + async getWorkspaceName() { + return `${this.state.taskId}_${this.state.runId}`; } async emitLog(message: string): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 6eb1c4763c..c80a1b81bf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -19,9 +19,9 @@ import { Logger } from 'winston'; import * as winston from 'winston'; import { JsonValue } from '@backstage/config'; import { TaskBroker, Task } from './types'; -import { TemplateActionRegistry } from './TemplateConverter'; import fs from 'fs-extra'; import path from 'path'; +import { TemplateActionRegistry } from './TemplateConverter'; type Options = { logger: Logger; @@ -46,9 +46,11 @@ export class TaskWorker { try { const { actionRegistry, logger } = this.options; - // bbl LUUUNCH - // taskID and runId not part of task? O_o - const workspacePath = await this.createWorkPath(task.taskId, task.runId); + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + await fs.ensureDir(workspacePath); const taskLogger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', @@ -99,55 +101,13 @@ export class TaskWorker { task.emitLog(`Finished step ${step.name}`); } - // const { values, template } = task.spec; - // task.emitLog('Prepare the skeleton'); - // const { protocol, location: pullPath } = parseLocationAnnotation( - // task.spec.template, - // ); - - // const preparer = - // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); - // const templater = templaters.get(template); - // const publisher = publishers.get(values.storePath); - - // const skeletonDir = await preparer.prepare(task.spec.template, { - // logger: taskLogger, - // workingDirectory: workingDirectory, - // }); - - // task.emitLog('Run the templater'); - // const { resultDir } = await templater.run({ - // directory: skeletonDir, - // dockerClient, - // logStream: stream, - // values: values, - // }); - - // task.emitLog('Publish template'); - // logger.info('Will now store the template'); - logger.info('So done right now'); await new Promise(resolve => setTimeout(resolve, 5000)); - // const result = await publisher.publish({ - // values: values, - // directory: resultDir, - // logger, - // }); - // task.emitLog(`Result: ${JSON.stringify(result)}`); await task.complete('completed'); } catch (error) { + task.emitLog(error); await task.complete('failed'); } } - - async createWorkPath(taskId: string, runId: string): Promise { - const workspacePath = path.join( - this.options.workingDirectory, - taskId, - runId, - ); - fs.ensureDir(workspacePath); - return workspacePath; - } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 496385d4ea..d3c08085fd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -60,10 +60,9 @@ export type DispatchResult = { export interface Task { spec: TaskSpec; - taskId: string; - runId: string; emitLog(message: string): Promise; complete(result: CompletedTaskState): Promise; + getWorkspaceName(): Promise; } export interface TaskBroker { From 95c2719809adec52fb1979757559e1172e1e3073 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Jan 2021 15:58:20 +0100 Subject: [PATCH 21/78] Rename publish to legacy:publish --- .../src/scaffolder/tasks/TemplateConverter.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index bd06dbaa91..86f722e475 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -70,7 +70,7 @@ export function templateEntityToSpec( steps.push({ id: 'publish', name: 'Publishing', - action: 'publish', + action: 'legacy:publish', parameters: { values, }, @@ -105,10 +105,6 @@ export class TemplateActionRegistry { this.actions.set(action.id, action); } - // validate - // ensure that action exist. - // template variables exist. - get(actionId: string): TemplateAction { const action = this.actions.get(actionId); if (!action) { From 605eb7f59401429fa5ab48ec83f4acf0a313dbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 Jan 2021 15:36:36 +0100 Subject: [PATCH 22/78] scaffolder: add the bare minimum of a db abstraction --- plugins/scaffolder-backend/knexfile.js | 25 +++++ .../migrations/20210120143715_init.js | 100 ++++++++++++++++++ plugins/scaffolder-backend/package.json | 3 +- .../scaffolder-backend/src/tasks/Database.ts | 70 ++++++++++++ plugins/scaffolder-backend/src/tasks/types.ts | 37 +++++++ 5 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 plugins/scaffolder-backend/knexfile.js create mode 100644 plugins/scaffolder-backend/migrations/20210120143715_init.js create mode 100644 plugins/scaffolder-backend/src/tasks/Database.ts create mode 100644 plugins/scaffolder-backend/src/tasks/types.ts diff --git a/plugins/scaffolder-backend/knexfile.js b/plugins/scaffolder-backend/knexfile.js new file mode 100644 index 0000000000..f469df4c08 --- /dev/null +++ b/plugins/scaffolder-backend/knexfile.js @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = { + development: { + client: 'sqlite3', + connection: ':memory:', + migrations: { + directory: 'migrations', + }, + }, +}; diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js new file mode 100644 index 0000000000..d2535bfa86 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -0,0 +1,100 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('tasks', table => { + table.comment('The table of scaffolder tasks'); + table.uuid('id').primary().notNullable().comment('The ID of the task'); + table + .text('context') + .notNullable() + .comment('A JSON object with task specific context information'); + table + .text('spec') + .notNullable() + .comment('A JSON encoded task specification'); + table + .text('status') + .notNullable() + .comment('The current status of the task'); + table + .integer('run_id') + .nullable() + .comment('The current run ID of the task'); + table + .dateTime('created_at') + .notNullable() + .comment('The timestamp when this task was created'); + table + .dateTime('last_heartbeat_at') + .nullable() + .comment('The last timestamp when a heartbeat was received'); + table + .integer('retry_count') + .notNullable() + .defaultTo(0) + .comment('The number of times that this task has been attempted'); + }); + await knex.schema.createTable('task_events', table => { + table.comment('The event stream a given task'); + table + .bigIncrements('id') + .primary() + .notNullable() + .comment('The ID of the event'); + table + .uuid('task_id') + .references('id') + .inTable('tasks') + .notNullable() + .onDelete('CASCADE') + .comment('The task that generated the event'); + table + .integer('run_id') + .nullable() + .comment('The run ID of the task that this event applies to'); + table + .text('stage_name') + .nullable() + .comment('The stage of the task that this event applies to'); + table.text('event_type').notNullable().comment('The type of event'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this event was generated'); + table.text('text').notNullable().comment('The text of the event'); + table.index(['task_id'], 'task_events_task_id_idx'); + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('task_events', table => { + table.dropIndex([], 'task_events_task_id_idx'); + }); + } + await knex.schema.dropTable('task_events'); + await knex.schema.dropTable('tasks'); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 97ab723e81..5d024c2798 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -53,7 +53,7 @@ "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", - "luxon": "^1.25.0", + "knex": "^0.21.6", "morgan": "^1.10.0", "p-queue": "^6.3.0", "uuid": "^8.2.0", @@ -73,6 +73,7 @@ }, "files": [ "dist", + "migrations", "config.d.ts" ], "configSchema": "config.d.ts" diff --git a/plugins/scaffolder-backend/src/tasks/Database.ts b/plugins/scaffolder-backend/src/tasks/Database.ts new file mode 100644 index 0000000000..c135380f87 --- /dev/null +++ b/plugins/scaffolder-backend/src/tasks/Database.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, resolvePackagePath } from '@backstage/backend-common'; +import Knex from 'knex'; +import { Logger } from 'winston'; +import { Database, Transaction } from './types'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-scaffolder-backend', + 'migrations', +); + +export class CommonDatabase implements Database { + static async create(knex: Knex, logger: Logger): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return new CommonDatabase(knex, logger); + } + + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} + + async transaction(fn: (tx: Transaction) => Promise): Promise { + try { + let result: T | undefined = undefined; + + await this.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.logger.debug(`Error during transaction, ${e}`); + + if ( + /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || + /unique constraint/.test(e.message) + ) { + throw new ConflictError(`Rejected due to a conflicting entity`, e); + } + + throw e; + } + } +} diff --git a/plugins/scaffolder-backend/src/tasks/types.ts b/plugins/scaffolder-backend/src/tasks/types.ts new file mode 100644 index 0000000000..27cbe7f7f1 --- /dev/null +++ b/plugins/scaffolder-backend/src/tasks/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The core database implementation. + */ +export interface Database { + /** + * Runs a transaction. + * + * The callback is expected to make calls back into this class. When it + * completes, the transaction is closed. + * + * @param fn The callback that implements the transaction + */ + transaction(fn: (tx: Transaction) => Promise): Promise; +} + +/** + * An abstraction for transactions of the underlying database technology. + */ +export type Transaction = { + rollback(): Promise; +}; From 71dc78b01ff6d64c644da18986be7b5d450b8042 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Feb 2021 14:06:09 +0100 Subject: [PATCH 23/78] Wire up database and rename database classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- packages/backend/src/plugins/scaffolder.ts | 2 + .../backend/src/plugins/scaffolder.ts | 2 + .../migrations/20210120143715_init.js | 14 +- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 248 ++++++++++++++++++ ...tabase.test.ts => MemoryTaskStore.test.ts} | 0 .../{MemoryDatabase.ts => MemoryTaskStore.ts} | 44 +--- ...oker.test.ts => StorageTaskBroker.test.ts} | 12 +- ...moryTaskBroker.ts => StorageTaskBroker.ts} | 18 +- .../src/scaffolder/tasks/TaskWorker.ts | 3 +- .../src/scaffolder/tasks/index.ts | 5 +- .../src/scaffolder/tasks/types.ts | 30 ++- .../scaffolder-backend/src/service/router.ts | 14 +- 12 files changed, 328 insertions(+), 64 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts rename plugins/scaffolder-backend/src/scaffolder/tasks/{MemoryDatabase.test.ts => MemoryTaskStore.test.ts} (100%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{MemoryDatabase.ts => MemoryTaskStore.ts} (73%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{MemoryTaskBroker.test.ts => StorageTaskBroker.test.ts} (90%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{MemoryTaskBroker.ts => StorageTaskBroker.ts} (90%) diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 4e2257a46c..723165ff82 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -30,6 +30,7 @@ import Docker from 'dockerode'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -54,5 +55,6 @@ export default async function createPlugin({ config, dockerClient, entityClient, + database, }); } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index c8bd3e5012..d68f90ce08 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -14,6 +14,7 @@ import Docker from 'dockerode'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -38,5 +39,6 @@ export default async function createPlugin({ config, dockerClient, entityClient, + database, }); } diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js index d2535bfa86..bb65e1e2fb 100644 --- a/plugins/scaffolder-backend/migrations/20210120143715_init.js +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -23,10 +23,6 @@ exports.up = async function up(knex) { await knex.schema.createTable('tasks', table => { table.comment('The table of scaffolder tasks'); table.uuid('id').primary().notNullable().comment('The ID of the task'); - table - .text('context') - .notNullable() - .comment('A JSON object with task specific context information'); table .text('spec') .notNullable() @@ -41,6 +37,7 @@ exports.up = async function up(knex) { .comment('The current run ID of the task'); table .dateTime('created_at') + .defaultTo(knex.fn.now()) .notNullable() .comment('The timestamp when this task was created'); table @@ -53,6 +50,7 @@ exports.up = async function up(knex) { .defaultTo(0) .comment('The number of times that this task has been attempted'); }); + await knex.schema.createTable('task_events', table => { table.comment('The event stream a given task'); table @@ -72,16 +70,16 @@ exports.up = async function up(knex) { .nullable() .comment('The run ID of the task that this event applies to'); table - .text('stage_name') - .nullable() - .comment('The stage of the task that this event applies to'); + .text('body') + .notNullable() + .comment('The JSON encoded body of the event'); table.text('event_type').notNullable().comment('The type of event'); table .dateTime('created_at') .defaultTo(knex.fn.now()) .notNullable() .comment('The timestamp when this event was generated'); - table.text('text').notNullable().comment('The text of the event'); + table.index(['task_id'], 'task_events_task_id_idx'); }); }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts new file mode 100644 index 0000000000..e6de1735d8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -0,0 +1,248 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/config'; +import { + ConflictError, + NotFoundError, + resolvePackagePath, +} from '@backstage/backend-common'; +import Knex, { Transaction } from 'knex'; +import { Logger } from 'winston'; +import { v4 as uuid } from 'uuid'; +import { + DbTaskEventRow, + DbTaskRow, + Status, + TaskEventType, + TaskSpec, + TaskStore, + TaskStoreEmitOptions, + TaskStoreGetEventsOptions, +} from './types'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-scaffolder-backend', + 'migrations', +); + +export type RawDbTaskRow = { + id: string; + spec: string; + status: Status; + last_heartbeat_at?: string; + retry_count: number; + created_at: string; + run_id?: string; +}; + +export type RawDbTaskEventRow = { + id: number; + run_id: string; + task_id: string; + body: string; + event_type: TaskEventType; + created_at: string; +}; + +export class DatabaseTaskStore implements TaskStore { + static async create(knex: Knex): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return new DatabaseTaskStore(knex); + } + + constructor(private readonly db: Knex) {} + + async get(taskId: string): Promise { + const [result] = await this.db('tasks') + .where({ id: taskId }) + .select(); + if (!result) { + throw new NotFoundError(`No task with id '${taskId}' found`); + } + try { + const spec = JSON.parse(result.spec); + return { + id: result.id, + spec, + status: result.status, + lastHeartbeat: result.last_heartbeat_at, + retryCount: result.retry_count, + createdAt: result.created_at, + runId: result.run_id, + }; + } catch (error) { + throw new Error(`Failed to parse spec of task '${taskId}', ${error}`); + } + } + + async createTask(spec: TaskSpec): Promise<{ taskId: string }> { + const taskId = uuid(); + await this.db('tasks').insert({ + id: taskId, + spec: JSON.stringify(spec), + status: 'open', + retry_count: 0, + }); + return { taskId }; + } + + async claimTask(): Promise { + return this.db.transaction(async tx => { + const [task] = await tx('tasks') + .where({ + status: 'open', + }) + .limit(1) + .select(); + + if (!task) { + return undefined; + } + + const runId = uuid(); + const updateCount = await tx('tasks') + .where({ id: task.id, status: 'open' }) + .update({ + status: 'processing', + run_id: runId, + }); + + if (updateCount < 1) { + return undefined; + } + + try { + const spec = JSON.parse(task.spec); + return { + id: task.id, + spec, + status: 'processing', + lastHeartbeat: task.last_heartbeat_at, + retryCount: task.retry_count, + createdAt: task.created_at, + runId: runId, + }; + } catch (error) { + throw new Error(`Failed to parse spec of task '${task.id}', ${error}`); + } + }); + } + + async heartbeat(runId: string): Promise { + const updateCount = await this.db('tasks') + .where({ run_id: runId, status: 'processing' }) + .update({ + last_heartbeat_at: this.db.fn.now(), + }); + if (updateCount === 0) { + throw new Error(`No running task with runId ${runId} found`); + } + } + + async setStatus(runId: string, status: Status): Promise { + let oldStatus: string; + if (status === 'failed' || status === 'completed') { + oldStatus = 'processing'; + } else { + throw new Error( + `Invalid status update of run '${runId}' to status '${status}'`, + ); + } + await this.db.transaction(async tx => { + const [task] = await tx('tasks') + .where({ + run_id: runId, + }) + .limit(1) + .select(); + + if (!task) { + throw new Error(`No task with runId ${runId} found`); + } + if (task.status !== oldStatus) { + throw new ConflictError( + `Refusing to update status of run '${runId}' to status '${status}' ` + + `as it is currently '${task.status}', expected '${oldStatus}'`, + ); + } + const updateCount = await tx('tasks') + .where({ + run_id: runId, + status: oldStatus, + }) + .update({ + status, + }); + if (updateCount !== 1) { + throw new Error( + `Failed to update status to '${status}' for runId ${runId}`, + ); + } + }); + } + + async emit({ + taskId, + runId, + body, + type, + }: TaskStoreEmitOptions): Promise { + const serliazedBody = JSON.stringify(body); + await this.db('task_events').insert({ + task_id: taskId, + run_id: runId, + event_type: type, + body: serliazedBody, + }); + } + + async getEvents({ + taskId, + after, + }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { + let query = this.db('task_events').where({ + task_id: taskId, + }); + if (typeof after === 'number') { + query = query + .where('task_events.id', '>', after) + .orWhere({ event_type: 'completion' }); + } + + const rawEvents = await query.select(); + const events = rawEvents.map(event => { + try { + const body = JSON.parse(event.body) as JsonObject; + return { + id: event.id, + runId: event.run_id, + taskId: event.task_id, + body, + type: event.event_type, + createdAt: event.created_at, + }; + } catch (error) { + throw new Error( + `Failed to parse event body from event taskId=${taskId} id=${event.id}, ${error}`, + ); + } + }); + return { events }; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.test.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts similarity index 73% rename from plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts index b2d1cf667b..e181edf524 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryDatabase.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts @@ -19,35 +19,17 @@ import { DbTaskEventRow, Status, TaskSpec, - TaskEventType, + TaskStore, + TaskStoreGetEventsOptions, + TaskStoreEmitOptions, } from './types'; import { v4 as uuid } from 'uuid'; -export interface Database { - get(taskId: string): Promise; - createTask(task: TaskSpec): Promise; - claimTask(): Promise; - heartbeat(runId: string): Promise; - setStatus(taskId: string, status: Status): Promise; -} - -type EmitOptions = { - taskId: string; - runId: string; - body: string; - type: TaskEventType; -}; - -type ReadOptions = { - taskId: string; - after?: number | undefined; -}; - -export class MemoryDatabase implements Database { +export class MemoryTaskStore implements TaskStore { private readonly store = new Map(); private readonly events = new Array(); - async emit({ taskId, runId, body, type }: EmitOptions) { + async emit({ taskId, runId, body, type }: TaskStoreEmitOptions) { this.events.push({ id: this.events.length, taskId, @@ -61,7 +43,7 @@ export class MemoryDatabase implements Database { async getEvents({ taskId, after, - }: ReadOptions): Promise<{ events: DbTaskEventRow[] }> { + }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { const events = this.events.filter(event => { if (event.taskId !== taskId) { return false; @@ -89,7 +71,7 @@ export class MemoryDatabase implements Database { throw new Error('No task with matching runId found'); } - this.store.set(task.taskId, { + this.store.set(task.id, { ...task, lastHeartbeat: new Date().toISOString(), }); @@ -103,23 +85,23 @@ export class MemoryDatabase implements Database { status: 'processing', runId: uuid(), }; - this.store.set(t.taskId, task); + this.store.set(t.id, task); return task; } } return undefined; } - async createTask(spec: TaskSpec): Promise { + async createTask(spec: TaskSpec): Promise<{ taskId: string }> { const taskRow = { - taskId: uuid(), + id: uuid(), spec, status: 'open' as Status, retryCount: 0, createdAt: new Date().toISOString(), }; - this.store.set(taskRow.taskId, taskRow); - return taskRow; + this.store.set(taskRow.id, taskRow); + return { taskId: taskRow.id }; } async get(taskId: string): Promise { @@ -135,6 +117,6 @@ export class MemoryDatabase implements Database { if (!task) { throw new Error(`no task found`); } - this.store.set(task.taskId, { ...task, status }); + this.store.set(task.id, { ...task, status }); } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts similarity index 90% rename from plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 686a6d9132..5bbe91ea87 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -15,13 +15,13 @@ */ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { TemplaterValues } from './actions/templater/types'; -import { MemoryDatabase } from './MemoryDatabase'; -import { MemoryTaskBroker, TaskAgent } from './MemoryTaskBroker'; +import { TemplaterValues } from '../stages'; +import { MemoryTaskStore } from './MemoryTaskStore'; +import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; -describe('MemoryTaskBroker', () => { - const storage = new MemoryDatabase(); - const broker = new MemoryTaskBroker(storage); +describe('StorageTaskBroker', () => { + const storage = new MemoryTaskStore(); + const broker = new StorageTaskBroker(storage); const taskSpec = { values: {} as TemplaterValues, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts similarity index 90% rename from plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 6937c535dc..6eedc023f8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -18,16 +18,16 @@ import { CompletedTaskState, Task, TaskSpec, + TaskStore, TaskBroker, DispatchResult, DbTaskEventRow, } from './types'; -import { MemoryDatabase } from './MemoryDatabase'; export class TaskAgent implements Task { private heartbeartInterval?: ReturnType; - static create(state: TaskState, storage: MemoryDatabase) { + static create(state: TaskState, storage: TaskStore) { const agent = new TaskAgent(state, storage); agent.start(); return agent; @@ -36,7 +36,7 @@ export class TaskAgent implements Task { // Runs heartbeat internally private constructor( private readonly state: TaskState, - private readonly storage: MemoryDatabase, + private readonly storage: TaskStore, ) {} get spec() { @@ -51,20 +51,20 @@ export class TaskAgent implements Task { await this.storage.emit({ taskId: this.state.taskId, runId: this.state.runId, - body: message, + body: { message }, type: 'log', }); } async complete(result: CompletedTaskState): Promise { await this.storage.setStatus( - this.state.taskId, + this.state.runId, result === 'failed' ? 'failed' : 'completed', ); this.storage.emit({ taskId: this.state.taskId, runId: this.state.runId, - body: `Run completed with status: ${result}`, + body: { message: `Run completed with status: ${result}` }, type: 'completion', }); if (this.heartbeartInterval) { @@ -96,8 +96,8 @@ function defer() { return { promise, resolve }; } -export class MemoryTaskBroker implements TaskBroker { - constructor(private readonly storage: MemoryDatabase) {} +export class StorageTaskBroker implements TaskBroker { + constructor(private readonly storage: TaskStore) {} private deferredDispatch = defer(); async claim(): Promise { @@ -107,7 +107,7 @@ export class MemoryTaskBroker implements TaskBroker { return TaskAgent.create( { runId: pendingTask.runId!, - taskId: pendingTask.taskId, + taskId: pendingTask.id, spec: pendingTask.spec, }, this.storage, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index c80a1b81bf..64cfdc6f96 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -73,6 +73,7 @@ export class TaskWorker { // Give us some time to curl observe task.emitLog('Task claimed, waiting ...'); await new Promise(resolve => setTimeout(resolve, 5000)); + console.log('DEBUG: task.spec =', JSON.stringify(task.spec, null, 2)); task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); const outputs: { [name: string]: JsonValue } = {}; @@ -106,7 +107,7 @@ export class TaskWorker { await task.complete('completed'); } catch (error) { - task.emitLog(error); + task.emitLog(String(error.stack)); await task.complete('failed'); } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 1cc9842fed..7b6552a3e0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ -export { MemoryDatabase } from './MemoryDatabase'; -export { MemoryTaskBroker } from './MemoryTaskBroker'; +export { MemoryTaskStore } from './MemoryTaskStore'; +export { DatabaseTaskStore } from './DatabaseTaskStore'; +export { StorageTaskBroker } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index d3c08085fd..f9081af6b6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { JsonValue } from '@backstage/config'; +import { JsonValue, JsonObject } from '@backstage/config'; export type Status = | 'open' @@ -26,7 +26,7 @@ export type Status = export type CompletedTaskState = 'failed' | 'completed'; export type DbTaskRow = { - taskId: string; + id: string; spec: TaskSpec; status: Status; lastHeartbeat?: string; @@ -40,7 +40,7 @@ export type DbTaskEventRow = { id: number; runId: string; taskId: string; - body: string; + body: JsonObject; type: TaskEventType; createdAt: string; }; @@ -69,3 +69,27 @@ export interface TaskBroker { claim(): Promise; dispatch(spec: TaskSpec): Promise; } + +export type TaskStoreEmitOptions = { + taskId: string; + runId: string; + body: JsonObject; + type: TaskEventType; +}; + +export type TaskStoreGetEventsOptions = { + taskId: string; + after?: number | undefined; +}; +export interface TaskStore { + get(taskId: string): Promise; + createTask(task: TaskSpec): Promise<{ taskId: string }>; + claimTask(): Promise; + heartbeat(runId: string): Promise; + setStatus(runId: string, status: Status): Promise; + emit({ taskId, runId, body, type }: TaskStoreEmitOptions): Promise; + getEvents({ + taskId, + after, + }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a72a5c5d48..23af538df8 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -34,17 +34,17 @@ import { CatalogEntityClient } from '../lib/catalog'; import { validate, ValidatorResult } from 'jsonschema'; import parseGitUrl from 'git-url-parse'; import { - MemoryTaskBroker, - MemoryDatabase, + DatabaseTaskStore, + StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; import { TemplateActionRegistry, templateEntityToSpec, } from '../scaffolder/tasks/TemplateConverter'; -import { LOCATION_ANNOTATION } from '@backstage/catalog-model'; import { registerLegacyActions } from '../scaffolder/stages/legacy'; import { getWorkingDirectory } from './helpers'; +import { PluginDatabaseManager } from '@backstage/backend-common'; export interface RouterOptions { preparers: PreparerBuilder; @@ -55,6 +55,7 @@ export interface RouterOptions { config: Config; dockerClient: Docker; entityClient: CatalogEntityClient; + database: PluginDatabaseManager; } export async function createRouter( @@ -71,12 +72,17 @@ export async function createRouter( config, dockerClient, entityClient, + database, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); - const taskBroker = new MemoryTaskBroker(new MemoryDatabase()); + + const databaseTaskStore = await DatabaseTaskStore.create( + await database.getClient(), + ); + const taskBroker = new StorageTaskBroker(databaseTaskStore); const actionRegistry = new TemplateActionRegistry(); const worker = new TaskWorker({ logger, From 5da5e6b2243516a448fbbe88fd806b550ba2f0ed Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Feb 2021 14:07:19 +0100 Subject: [PATCH 24/78] Delete MemoryTaskStore This is replaced by running sqlite in memory. --- .../scaffolder/tasks/MemoryTaskStore.test.ts | 21 --- .../src/scaffolder/tasks/MemoryTaskStore.ts | 122 ------------------ .../src/scaffolder/tasks/index.ts | 1 - 3 files changed, 144 deletions(-) delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts deleted file mode 100644 index c5522cb38a..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -describe('MemoryDatabase', () => { - it('should be tested', async () => { - expect(1).toBe(2); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts deleted file mode 100644 index e181edf524..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskStore.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - DbTaskRow, - DbTaskEventRow, - Status, - TaskSpec, - TaskStore, - TaskStoreGetEventsOptions, - TaskStoreEmitOptions, -} from './types'; -import { v4 as uuid } from 'uuid'; - -export class MemoryTaskStore implements TaskStore { - private readonly store = new Map(); - private readonly events = new Array(); - - async emit({ taskId, runId, body, type }: TaskStoreEmitOptions) { - this.events.push({ - id: this.events.length, - taskId, - runId, - body, - type, - createdAt: new Date().toISOString(), - }); - } - - async getEvents({ - taskId, - after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { - const events = this.events.filter(event => { - if (event.taskId !== taskId) { - return false; - } - if (after !== undefined) { - if (event.id <= after) { - return false; - } - } - return true; - }); - return { events }; - } - - async heartbeat(runId: string): Promise { - let task: DbTaskRow | undefined; - - for (const t of this.store.values()) { - if (t.runId === runId) { - task = t; - } - } - - if (!task) { - throw new Error('No task with matching runId found'); - } - - this.store.set(task.id, { - ...task, - lastHeartbeat: new Date().toISOString(), - }); - } - - async claimTask(): Promise { - for (const t of this.store.values()) { - if (t.status === 'open') { - const task: DbTaskRow = { - ...t, - status: 'processing', - runId: uuid(), - }; - this.store.set(t.id, task); - return task; - } - } - return undefined; - } - - async createTask(spec: TaskSpec): Promise<{ taskId: string }> { - const taskRow = { - id: uuid(), - spec, - status: 'open' as Status, - retryCount: 0, - createdAt: new Date().toISOString(), - }; - this.store.set(taskRow.id, taskRow); - return { taskId: taskRow.id }; - } - - async get(taskId: string): Promise { - const task = this.store.get(taskId); - if (task) { - return task; - } - throw new Error(`could not found task ${taskId}`); - } - - async setStatus(taskId: string, status: Status): Promise { - const task = this.store.get(taskId); - if (!task) { - throw new Error(`no task found`); - } - this.store.set(task.id, { ...task, status }); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 7b6552a3e0..c85135426e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -export { MemoryTaskStore } from './MemoryTaskStore'; export { DatabaseTaskStore } from './DatabaseTaskStore'; export { StorageTaskBroker } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; From 88c31ab16ba778b43c5c0e6fb6aad4a3f68cf8c0 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 09:27:18 +0100 Subject: [PATCH 25/78] Add tests. Extend storage with fetch staletasks method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- plugins/scaffolder-backend/package.json | 3 +- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 35 +++-- .../tasks/StorageTaskBroker.test.ts | 145 ++++++++++++++---- .../src/scaffolder/tasks/StorageTaskBroker.ts | 17 +- .../src/scaffolder/tasks/types.ts | 1 + .../src/service/router.test.ts | 25 ++- .../scaffolder-backend/src/service/router.ts | 6 +- 7 files changed, 182 insertions(+), 50 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 5d024c2798..bdecf6196e 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -58,7 +58,8 @@ "p-queue": "^6.3.0", "uuid": "^8.2.0", "winston": "^3.2.1", - "yaml": "^1.10.0" + "yaml": "^1.10.0", + "luxon": "^1.25.0" }, "devDependencies": { "@backstage/cli": "^0.5.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index e6de1735d8..1d99cd2958 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -20,8 +20,7 @@ import { NotFoundError, resolvePackagePath, } from '@backstage/backend-common'; -import Knex, { Transaction } from 'knex'; -import { Logger } from 'winston'; +import Knex from 'knex'; import { v4 as uuid } from 'uuid'; import { DbTaskEventRow, @@ -121,6 +120,7 @@ export class DatabaseTaskStore implements TaskStore { .update({ status: 'processing', run_id: runId, + last_heartbeat_at: this.db.fn.now(), }); if (updateCount < 1) { @@ -155,6 +155,18 @@ export class DatabaseTaskStore implements TaskStore { } } + async listStaleTasks(): Promise<{ tasks: DbTaskRow }> { + const rows = await this.db('tasks') + .where('status', 'processing') + .andWhere( + 'last_heartbeat_at', + '<', + this.db.type === 'sqlite' + ? this.db.raw("datetime('now', '-2 seconds')") + : this.db.raw("dateadd('second', -2, now())"), + ); + } + async setStatus(runId: string, status: Status): Promise { let oldStatus: string; if (status === 'failed' || status === 'completed') { @@ -216,16 +228,17 @@ export class DatabaseTaskStore implements TaskStore { taskId, after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { - let query = this.db('task_events').where({ - task_id: taskId, - }); - if (typeof after === 'number') { - query = query - .where('task_events.id', '>', after) - .orWhere({ event_type: 'completion' }); - } + const rawEvents = await this.db('task_events') + .where({ + task_id: taskId, + }) + .andWhere(builder => { + if (typeof after === 'number') { + builder.where('id', '>', after).orWhere('event_type', 'completion'); + } + }) + .select(); - const rawEvents = await query.select(); const events = rawEvents.map(event => { try { const body = JSON.parse(event.body) as JsonObject; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 5bbe91ea87..ebbac1db5f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -14,47 +14,54 @@ * limitations under the License. */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { TemplaterValues } from '../stages'; -import { MemoryTaskStore } from './MemoryTaskStore'; +import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +import { TaskStore, TaskSpec, DbTaskEventRow } from './types'; + +async function createStore(): Promise { + const manager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ).forPlugin('scaffolder'); + return await DatabaseTaskStore.create(await manager.getClient()); +} describe('StorageTaskBroker', () => { - const storage = new MemoryTaskStore(); - const broker = new StorageTaskBroker(storage); + let storage: TaskStore; - const taskSpec = { - values: {} as TemplaterValues, - template: {} as TemplateEntityV1alpha1, - }; + beforeAll(async () => { + storage = await createStore(); + }); it('should claim a dispatched work item', async () => { - await broker.dispatch(taskSpec); + const broker = new StorageTaskBroker(storage); + await broker.dispatch({ steps: [] }); await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); }); it('should wait for a dispatched work item', async () => { + const broker = new StorageTaskBroker(storage); const promise = broker.claim(); await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); - await broker.dispatch(taskSpec); + await broker.dispatch({ steps: [] }); await expect(promise).resolves.toEqual(expect.any(TaskAgent)); }); it('should dispatch multiple items and claim them in order', async () => { - await broker.dispatch({ - values: { owner: 'a' } as TemplaterValues, - template: {} as TemplateEntityV1alpha1, - }); - await broker.dispatch({ - values: { owner: 'b' } as TemplaterValues, - template: {} as TemplateEntityV1alpha1, - }); - await broker.dispatch({ - values: { owner: 'c' } as TemplaterValues, - template: {} as TemplateEntityV1alpha1, - }); + const broker = new StorageTaskBroker(storage); + await broker.dispatch({ steps: [{ id: 'a' }] } as TaskSpec); + await broker.dispatch({ steps: [{ id: 'b' }] } as TaskSpec); + await broker.dispatch({ steps: [{ id: 'c' }] } as TaskSpec); const taskA = await broker.claim(); const taskB = await broker.claim(); @@ -62,13 +69,14 @@ describe('StorageTaskBroker', () => { await expect(taskA).toEqual(expect.any(TaskAgent)); await expect(taskB).toEqual(expect.any(TaskAgent)); await expect(taskC).toEqual(expect.any(TaskAgent)); - await expect(taskA.spec.values.owner).toBe('a'); - await expect(taskB.spec.values.owner).toBe('b'); - await expect(taskC.spec.values.owner).toBe('c'); + await expect(taskA.spec.steps[0].id).toBe('a'); + await expect(taskB.spec.steps[0].id).toBe('b'); + await expect(taskC.spec.steps[0].id).toBe('c'); }); it('should complete a task', async () => { - const dispatchResult = await broker.dispatch(taskSpec); + const broker = new StorageTaskBroker(storage); + const dispatchResult = await broker.dispatch({ steps: [] }); const task = await broker.claim(); await task.complete('completed'); const taskRow = await storage.get(dispatchResult.taskId); @@ -76,10 +84,91 @@ describe('StorageTaskBroker', () => { }); it('should fail a task', async () => { - const dispatchResult = await broker.dispatch(taskSpec); + const broker = new StorageTaskBroker(storage); + const dispatchResult = await broker.dispatch({ steps: [] }); const task = await broker.claim(); await task.complete('failed'); const taskRow = await storage.get(dispatchResult.taskId); expect(taskRow.status).toBe('failed'); }); + + it('multiple brokers should be able to observe a single task', async () => { + const broker1 = new StorageTaskBroker(storage); + const broker2 = new StorageTaskBroker(storage); + + const { taskId } = await broker1.dispatch({ steps: [] }); + + const logPromise = new Promise(resolve => { + const observedEvents = new Array(); + + broker2.observe({ taskId, after: undefined }, (_err, { events }) => { + observedEvents.push(...events); + if (events.some(e => e.type === 'completion')) { + resolve(observedEvents); + } + }); + }); + const task = await broker1.claim(); + await task.emitLog('log 1'); + await task.emitLog('log 2'); + await task.emitLog('log 3'); + await task.complete('completed'); + + const logs = await logPromise; + expect(logs.map(l => l.body.message)).toEqual([ + 'log 1', + 'log 2', + 'log 3', + 'Run completed with status: completed', + ]); + + const afterLogs = await new Promise(resolve => { + broker2.observe({ taskId, after: logs[1].id }, (_err, { events }) => + resolve(events.map(e => e.body.message as string)), + ); + }); + expect(afterLogs).toEqual([ + 'log 3', + 'Run completed with status: completed', + ]); + }); + + it('should heartbeat', async () => { + const broker = new StorageTaskBroker(storage); + const { taskId } = await broker.dispatch({ steps: [] }); + const task = await broker.claim(); + + const initialTask = await storage.get(taskId); + + for (;;) { + const maybeTask = await storage.get(taskId); + if (maybeTask.lastHeartbeat !== initialTask.lastHeartbeat) { + break; + } + await new Promise(resolve => setTimeout(resolve, 50)); + } + await task.complete('completed'); + expect.assertions(0); + }); + + it('should be cancelled if heartbeat stops', async () => { + const broker = new StorageTaskBroker(storage); + const { taskId } = await broker.dispatch({ steps: [] }); + console.log('DEBUG: taskId =', taskId); + const task = await broker.claim(); + clearInterval((task as any).heartbeatInterval); + + setTimeout(() => { + storage.listStaleTasks(); + }, 4000); + + for (;;) { + const maybeTask = await storage.get(taskId); + if (maybeTask.status === 'cancelled') { + break; + } + await new Promise(resolve => setTimeout(resolve, 50)); + } + expect.assertions(0); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 6eedc023f8..8c0d63ee71 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -25,7 +25,7 @@ import { } from './types'; export class TaskAgent implements Task { - private heartbeartInterval?: ReturnType; + private heartbeatInterval?: ReturnType; static create(state: TaskState, storage: TaskStore) { const agent = new TaskAgent(state, storage); @@ -67,13 +67,13 @@ export class TaskAgent implements Task { body: { message: `Run completed with status: ${result}` }, type: 'completion', }); - if (this.heartbeartInterval) { - clearInterval(this.heartbeartInterval); + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); } } private start() { - this.heartbeartInterval = setInterval(() => { + this.heartbeatInterval = setInterval(() => { if (!this.state.runId) { throw new Error('no run id provided'); } @@ -131,7 +131,10 @@ export class StorageTaskBroker implements TaskBroker { taskId: string; after: number | undefined; }, - callback: (result: { events: DbTaskEventRow[] }) => void, + callback: ( + error: Error | undefined, + result: { events: DbTaskEventRow[] }, + ) => void, ): () => void { const { taskId } = options; @@ -148,9 +151,9 @@ export class StorageTaskBroker implements TaskBroker { if (events.length) { after = events[events.length - 1].id; try { - callback(result); + callback(undefined, result); } catch (error) { - console.log('DEBUG: error =', error); + callback(error, { events: [] }); } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index f9081af6b6..dd9a1ad9ad 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -86,6 +86,7 @@ export interface TaskStore { createTask(task: TaskSpec): Promise<{ taskId: string }>; claimTask(): Promise; heartbeat(runId: string): Promise; + listStaleTasks(): Promise<{ tasks: DbTaskRow }>; setStatus(runId: string, status: Status): Promise; emit({ taskId, runId, body, type }: TaskStoreEmitOptions): Promise; getEvents({ diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 31f62febd2..8a0fa6cba6 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -16,6 +16,7 @@ const mockAccess = jest.fn(); jest.doMock('fs-extra', () => ({ + access: mockAccess, promises: { access: mockAccess, }, @@ -27,7 +28,11 @@ jest.doMock('fs-extra', () => ({ remove: jest.fn(), })); -import { getVoidLogger } from '@backstage/backend-common'; +import { + SingleConnectionDatabaseManager, + PluginDatabaseManager, + getVoidLogger, +} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -41,6 +46,19 @@ const generateEntityClient: any = (template: any) => ({ findTemplate: () => Promise.resolve(template), }); +function createDatabase(): PluginDatabaseManager { + return SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ).forPlugin('scaffolder'); +} + describe('createRouter - working directory', () => { const mockPrepare = jest.fn(); const mockPreparers = new Preparers(); @@ -78,7 +96,6 @@ describe('createRouter - working directory', () => { }; const mockedEntityClient = generateEntityClient(template); - it('should throw an error when working directory does not exist or is not writable', async () => { mockAccess.mockImplementation(() => { throw new Error('access error'); @@ -93,6 +110,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }), ).rejects.toThrow('access error'); }); @@ -106,6 +124,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }); const app = express().use(router); @@ -134,6 +153,7 @@ describe('createRouter - working directory', () => { config: new ConfigReader({}), dockerClient: new Docker(), entityClient: mockedEntityClient, + database: createDatabase(), }); const app = express().use(router); @@ -203,6 +223,7 @@ describe('createRouter', () => { config: new ConfigReader({}), dockerClient: new Docker(), entityClient: generateEntityClient(template), + database: createDatabase(), }); app = express().use(router); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 23af538df8..6eb112fa3c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -164,7 +164,11 @@ export async function createRouter( // After client opens connection send all nests as string const unsubscribe = taskBroker.observe( { taskId, after }, - ({ events }) => { + (error, { events }) => { + logger.error( + `Received error from event stream when observing task ${taskId}`, + error, + ); for (const event of events) { res.write(`event:${JSON.stringify(event)}\n\n`); if (event.type === 'completion') { From 6608f9f5ee4cab89e5f658924db7b3ed975ff712 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 16:10:13 +0100 Subject: [PATCH 26/78] Drop runId, update TaskStore interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../migrations/20210120143715_init.js | 17 +--- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 85 +++++++++--------- .../tasks/StorageTaskBroker.test.ts | 57 ++++++------ .../src/scaffolder/tasks/StorageTaskBroker.ts | 86 +++++++++++++------ .../src/scaffolder/tasks/TaskWorker.ts | 1 - .../src/scaffolder/tasks/types.ts | 39 ++++++--- .../scaffolder-backend/src/service/router.ts | 4 +- 7 files changed, 168 insertions(+), 121 deletions(-) diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js index bb65e1e2fb..248b4f932f 100644 --- a/plugins/scaffolder-backend/migrations/20210120143715_init.js +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -31,10 +31,6 @@ exports.up = async function up(knex) { .text('status') .notNullable() .comment('The current status of the task'); - table - .integer('run_id') - .nullable() - .comment('The current run ID of the task'); table .dateTime('created_at') .defaultTo(knex.fn.now()) @@ -44,11 +40,6 @@ exports.up = async function up(knex) { .dateTime('last_heartbeat_at') .nullable() .comment('The last timestamp when a heartbeat was received'); - table - .integer('retry_count') - .notNullable() - .defaultTo(0) - .comment('The number of times that this task has been attempted'); }); await knex.schema.createTable('task_events', table => { @@ -65,17 +56,13 @@ exports.up = async function up(knex) { .notNullable() .onDelete('CASCADE') .comment('The task that generated the event'); - table - .integer('run_id') - .nullable() - .comment('The run ID of the task that this event applies to'); table .text('body') .notNullable() .comment('The JSON encoded body of the event'); table.text('event_type').notNullable().comment('The type of event'); table - .dateTime('created_at') + .timestamp('created_at', { precision: 9 }) .defaultTo(knex.fn.now()) .notNullable() .comment('The timestamp when this event was generated'); @@ -90,7 +77,7 @@ exports.up = async function up(knex) { exports.down = async function down(knex) { if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('task_events', table => { - table.dropIndex([], 'task_events_task_id_idx'); + table.dropIndex([], 'ctask_events_task_id_idx'); }); } await knex.schema.dropTable('task_events'); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 1d99cd2958..cf1bc18621 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -43,14 +43,11 @@ export type RawDbTaskRow = { spec: string; status: Status; last_heartbeat_at?: string; - retry_count: number; created_at: string; - run_id?: string; }; export type RawDbTaskEventRow = { id: number; - run_id: string; task_id: string; body: string; event_type: TaskEventType; @@ -80,10 +77,8 @@ export class DatabaseTaskStore implements TaskStore { id: result.id, spec, status: result.status, - lastHeartbeat: result.last_heartbeat_at, - retryCount: result.retry_count, + lastHeartbeatAt: result.last_heartbeat_at, createdAt: result.created_at, - runId: result.run_id, }; } catch (error) { throw new Error(`Failed to parse spec of task '${taskId}', ${error}`); @@ -96,7 +91,6 @@ export class DatabaseTaskStore implements TaskStore { id: taskId, spec: JSON.stringify(spec), status: 'open', - retry_count: 0, }); return { taskId }; } @@ -114,12 +108,10 @@ export class DatabaseTaskStore implements TaskStore { return undefined; } - const runId = uuid(); const updateCount = await tx('tasks') .where({ id: task.id, status: 'open' }) .update({ status: 'processing', - run_id: runId, last_heartbeat_at: this.db.fn.now(), }); @@ -133,10 +125,8 @@ export class DatabaseTaskStore implements TaskStore { id: task.id, spec, status: 'processing', - lastHeartbeat: task.last_heartbeat_at, - retryCount: task.retry_count, + lastHeartbeatAt: task.last_heartbeat_at, createdAt: task.created_at, - runId: runId, }; } catch (error) { throw new Error(`Failed to parse spec of task '${task.id}', ${error}`); @@ -144,58 +134,76 @@ export class DatabaseTaskStore implements TaskStore { }); } - async heartbeat(runId: string): Promise { + async heartbeatTask(taskId: string): Promise { const updateCount = await this.db('tasks') - .where({ run_id: runId, status: 'processing' }) + .where({ id: taskId, status: 'processing' }) .update({ last_heartbeat_at: this.db.fn.now(), }); if (updateCount === 0) { - throw new Error(`No running task with runId ${runId} found`); + throw new Error(`No running task with taskId ${taskId} found`); } } - async listStaleTasks(): Promise<{ tasks: DbTaskRow }> { - const rows = await this.db('tasks') + async listStaleTasks({ + timeoutS, + }: { + timeoutS: number; + }): Promise<{ + tasks: { taskId: string }[]; + }> { + const rawRows = await this.db('tasks') .where('status', 'processing') .andWhere( 'last_heartbeat_at', - '<', - this.db.type === 'sqlite' - ? this.db.raw("datetime('now', '-2 seconds')") - : this.db.raw("dateadd('second', -2, now())"), + '<=', + this.db.client.config.client === 'sqlite3' + ? this.db.raw(`datetime('now', '-${Number(timeoutS)} seconds')`) + : this.db.raw(`dateadd('second', -${Number(timeoutS)}, now())`), ); + const tasks = rawRows.map(row => ({ + taskId: row.id, + })); + return { tasks }; } - async setStatus(runId: string, status: Status): Promise { + async completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise { let oldStatus: string; if (status === 'failed' || status === 'completed') { oldStatus = 'processing'; } else { throw new Error( - `Invalid status update of run '${runId}' to status '${status}'`, + `Invalid status update of run '${taskId}' to status '${status}'`, ); } await this.db.transaction(async tx => { const [task] = await tx('tasks') .where({ - run_id: runId, + id: taskId, }) .limit(1) .select(); if (!task) { - throw new Error(`No task with runId ${runId} found`); + throw new Error(`No task with taskId ${taskId} found`); } if (task.status !== oldStatus) { throw new ConflictError( - `Refusing to update status of run '${runId}' to status '${status}' ` + + `Refusing to update status of run '${taskId}' to status '${status}' ` + `as it is currently '${task.status}', expected '${oldStatus}'`, ); } const updateCount = await tx('tasks') .where({ - run_id: runId, + id: taskId, status: oldStatus, }) .update({ @@ -203,28 +211,28 @@ export class DatabaseTaskStore implements TaskStore { }); if (updateCount !== 1) { throw new Error( - `Failed to update status to '${status}' for runId ${runId}`, + `Failed to update status to '${status}' for taskId ${taskId}`, ); } + + await tx('task_events').insert({ + task_id: taskId, + event_type: 'completion', + body: JSON.stringify(eventBody), + }); }); } - async emit({ - taskId, - runId, - body, - type, - }: TaskStoreEmitOptions): Promise { + async emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise { const serliazedBody = JSON.stringify(body); await this.db('task_events').insert({ task_id: taskId, - run_id: runId, - event_type: type, + event_type: 'log', body: serliazedBody, }); } - async getEvents({ + async listEvents({ taskId, after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { @@ -244,8 +252,7 @@ export class DatabaseTaskStore implements TaskStore { const body = JSON.parse(event.body) as JsonObject; return { id: event.id, - runId: event.run_id, - taskId: event.task_id, + taskId, body, type: event.event_type, createdAt: event.created_at, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index ebbac1db5f..c45fa46335 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -14,13 +14,16 @@ * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { + getVoidLogger, + SingleConnectionDatabaseManager, +} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; -import { TaskStore, TaskSpec, DbTaskEventRow } from './types'; +import { TaskSpec, DbTaskEventRow } from './types'; -async function createStore(): Promise { +async function createStore(): Promise { const manager = SingleConnectionDatabaseManager.fromConfig( new ConfigReader({ backend: { @@ -35,20 +38,21 @@ async function createStore(): Promise { } describe('StorageTaskBroker', () => { - let storage: TaskStore; + let storage: DatabaseTaskStore; beforeAll(async () => { storage = await createStore(); }); + const logger = getVoidLogger(); it('should claim a dispatched work item', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); await broker.dispatch({ steps: [] }); await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); }); it('should wait for a dispatched work item', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); const promise = broker.claim(); await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); @@ -58,7 +62,7 @@ describe('StorageTaskBroker', () => { }); it('should dispatch multiple items and claim them in order', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); await broker.dispatch({ steps: [{ id: 'a' }] } as TaskSpec); await broker.dispatch({ steps: [{ id: 'b' }] } as TaskSpec); await broker.dispatch({ steps: [{ id: 'c' }] } as TaskSpec); @@ -75,16 +79,16 @@ describe('StorageTaskBroker', () => { }); it('should complete a task', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); const dispatchResult = await broker.dispatch({ steps: [] }); const task = await broker.claim(); await task.complete('completed'); const taskRow = await storage.get(dispatchResult.taskId); expect(taskRow.status).toBe('completed'); - }); + }, 10000); it('should fail a task', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); const dispatchResult = await broker.dispatch({ steps: [] }); const task = await broker.claim(); await task.complete('failed'); @@ -93,8 +97,8 @@ describe('StorageTaskBroker', () => { }); it('multiple brokers should be able to observe a single task', async () => { - const broker1 = new StorageTaskBroker(storage); - const broker2 = new StorageTaskBroker(storage); + const broker1 = new StorageTaskBroker(storage, logger); + const broker2 = new StorageTaskBroker(storage, logger); const { taskId } = await broker1.dispatch({ steps: [] }); @@ -115,7 +119,7 @@ describe('StorageTaskBroker', () => { await task.complete('completed'); const logs = await logPromise; - expect(logs.map(l => l.body.message)).toEqual([ + expect(logs.map(l => l.body.message, logger)).toEqual([ 'log 1', 'log 2', 'log 3', @@ -134,7 +138,7 @@ describe('StorageTaskBroker', () => { }); it('should heartbeat', async () => { - const broker = new StorageTaskBroker(storage); + const broker = new StorageTaskBroker(storage, logger); const { taskId } = await broker.dispatch({ steps: [] }); const task = await broker.claim(); @@ -142,7 +146,7 @@ describe('StorageTaskBroker', () => { for (;;) { const maybeTask = await storage.get(taskId); - if (maybeTask.lastHeartbeat !== initialTask.lastHeartbeat) { + if (maybeTask.lastHeartbeatAt !== initialTask.lastHeartbeatAt) { break; } await new Promise(resolve => setTimeout(resolve, 50)); @@ -151,24 +155,29 @@ describe('StorageTaskBroker', () => { expect.assertions(0); }); - it('should be cancelled if heartbeat stops', async () => { - const broker = new StorageTaskBroker(storage); + it('should be update the status to failed if heartbeat fails', async () => { + const broker = new StorageTaskBroker(storage, logger); const { taskId } = await broker.dispatch({ steps: [] }); - console.log('DEBUG: taskId =', taskId); const task = await broker.claim(); - clearInterval((task as any).heartbeatInterval); - setTimeout(() => { - storage.listStaleTasks(); - }, 4000); + jest + .spyOn((task as any).storage, 'heartbeatTask') + .mockRejectedValue(new Error('nah m8')); + + const intervalId = setInterval(() => { + broker.vacuumTasks({ timeoutS: 2 }).catch(fail); + }, 500); for (;;) { const maybeTask = await storage.get(taskId); - if (maybeTask.status === 'cancelled') { + if (maybeTask.status === 'failed') { break; } await new Promise(resolve => setTimeout(resolve, 50)); } - expect.assertions(0); + + clearInterval(intervalId); + + expect(task.done).toBe(true); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 8c0d63ee71..48c0d5e5ba 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { Logger } from 'winston'; import { CompletedTaskState, Task, @@ -25,11 +25,13 @@ import { } from './types'; export class TaskAgent implements Task { - private heartbeatInterval?: ReturnType; + private isDone = false; - static create(state: TaskState, storage: TaskStore) { - const agent = new TaskAgent(state, storage); - agent.start(); + private heartbeatTimeoutId?: ReturnType; + + static create(state: TaskState, storage: TaskStore, logger: Logger) { + const agent = new TaskAgent(state, storage, logger); + agent.startTimeout(); return agent; } @@ -37,6 +39,7 @@ export class TaskAgent implements Task { private constructor( private readonly state: TaskState, private readonly storage: TaskStore, + private readonly logger: Logger, ) {} get spec() { @@ -44,40 +47,45 @@ export class TaskAgent implements Task { } async getWorkspaceName() { - return `${this.state.taskId}_${this.state.runId}`; + return this.state.taskId; + } + + get done() { + return this.isDone; } async emitLog(message: string): Promise { - await this.storage.emit({ + await this.storage.emitLogEvent({ taskId: this.state.taskId, - runId: this.state.runId, body: { message }, - type: 'log', }); } async complete(result: CompletedTaskState): Promise { - await this.storage.setStatus( - this.state.runId, - result === 'failed' ? 'failed' : 'completed', - ); - this.storage.emit({ + await this.storage.completeTask({ taskId: this.state.taskId, - runId: this.state.runId, - body: { message: `Run completed with status: ${result}` }, - type: 'completion', + status: result === 'failed' ? 'failed' : 'completed', + eventBody: { message: `Run completed with status: ${result}` }, }); - if (this.heartbeatInterval) { - clearInterval(this.heartbeatInterval); + this.isDone = true; + if (this.heartbeatTimeoutId) { + clearTimeout(this.heartbeatTimeoutId); } } - private start() { - this.heartbeatInterval = setInterval(() => { - if (!this.state.runId) { - throw new Error('no run id provided'); + private startTimeout() { + this.heartbeatTimeoutId = setTimeout(async () => { + try { + await this.storage.heartbeatTask(this.state.taskId); + this.startTimeout(); + } catch (error) { + this.isDone = true; + + this.logger.error( + `Heartbeat for task ${this.state.taskId} failed`, + error, + ); } - this.storage.heartbeat(this.state.runId); }, 1000); } } @@ -85,7 +93,6 @@ export class TaskAgent implements Task { interface TaskState { spec: TaskSpec; taskId: string; - runId: string; } function defer() { @@ -97,7 +104,10 @@ function defer() { } export class StorageTaskBroker implements TaskBroker { - constructor(private readonly storage: TaskStore) {} + constructor( + private readonly storage: TaskStore, + private readonly logger: Logger, + ) {} private deferredDispatch = defer(); async claim(): Promise { @@ -106,11 +116,11 @@ export class StorageTaskBroker implements TaskBroker { if (pendingTask) { return TaskAgent.create( { - runId: pendingTask.runId!, taskId: pendingTask.id, spec: pendingTask.spec, }, this.storage, + this.logger, ); } @@ -146,7 +156,7 @@ export class StorageTaskBroker implements TaskBroker { (async () => { let after = options.after; while (!cancelled) { - const result = await this.storage.getEvents({ taskId, after: after }); + const result = await this.storage.listEvents({ taskId, after: after }); const { events } = result; if (events.length) { after = events[events.length - 1].id; @@ -164,6 +174,26 @@ export class StorageTaskBroker implements TaskBroker { return unsubscribe; } + async vacuumTasks(timeoutS: { timeoutS: number }): Promise { + const { tasks } = await this.storage.listStaleTasks(timeoutS); + await Promise.all( + tasks.map(async task => { + try { + await this.storage.completeTask({ + taskId: task.taskId, + status: 'failed', + eventBody: { + message: + 'The task was cancelled because the task worker lost connection to the task broker', + }, + }); + } catch (error) { + this.logger.warn(`Failed to cancel task '${task.taskId}', ${error}`); + } + }), + ); + } + private waitForDispatch() { return this.deferredDispatch.promise; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 64cfdc6f96..f87d2df934 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -73,7 +73,6 @@ export class TaskWorker { // Give us some time to curl observe task.emitLog('Task claimed, waiting ...'); await new Promise(resolve => setTimeout(resolve, 5000)); - console.log('DEBUG: task.spec =', JSON.stringify(task.spec, null, 2)); task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); const outputs: { [name: string]: JsonValue } = {}; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index dd9a1ad9ad..0c2592109b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -29,16 +29,13 @@ export type DbTaskRow = { id: string; spec: TaskSpec; status: Status; - lastHeartbeat?: string; - retryCount: number; createdAt: string; - runId?: string; + lastHeartbeatAt?: string; }; export type TaskEventType = 'completion' | 'log'; export type DbTaskEventRow = { id: number; - runId: string; taskId: string; body: JsonObject; type: TaskEventType; @@ -60,6 +57,7 @@ export type DispatchResult = { export interface Task { spec: TaskSpec; + done: boolean; emitLog(message: string): Promise; complete(result: CompletedTaskState): Promise; getWorkspaceName(): Promise; @@ -68,13 +66,22 @@ export interface Task { export interface TaskBroker { claim(): Promise; dispatch(spec: TaskSpec): Promise; + vacuumTasks(timeoutS: { timeoutS: number }): Promise; + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { events: DbTaskEventRow[] }, + ) => void, + ): () => void; } export type TaskStoreEmitOptions = { taskId: string; - runId: string; body: JsonObject; - type: TaskEventType; }; export type TaskStoreGetEventsOptions = { @@ -82,14 +89,22 @@ export type TaskStoreGetEventsOptions = { after?: number | undefined; }; export interface TaskStore { - get(taskId: string): Promise; createTask(task: TaskSpec): Promise<{ taskId: string }>; claimTask(): Promise; - heartbeat(runId: string): Promise; - listStaleTasks(): Promise<{ tasks: DbTaskRow }>; - setStatus(runId: string, status: Status): Promise; - emit({ taskId, runId, body, type }: TaskStoreEmitOptions): Promise; - getEvents({ + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + heartbeatTask(taskId: string): Promise; + listStaleTasks(options: { + timeoutS: number; + }): Promise<{ + tasks: { taskId: string }[]; + }>; + + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + listEvents({ taskId, after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 6eb112fa3c..8da6f02507 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -82,7 +82,7 @@ export async function createRouter( const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), ); - const taskBroker = new StorageTaskBroker(databaseTaskStore); + const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const worker = new TaskWorker({ logger, @@ -124,7 +124,7 @@ export async function createRouter( error: job.error, }); }) - // curl -X POST -d '{"templateName":"springboot-template","values": {"storePath":"https://github.com/jhaals/foo", "component_id":"woop", "description": "apa", "owner": "me" }}' -H 'Content-Type: application/json' localhost:7000/api/scaffolder/v2/tasks + .post('/v2/tasks', async (req, res) => { const templateName: string = req.body.templateName; const values: TemplaterValues = { From 9b0bba5439c7fe7383f897756b6394d807a62393 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 16:25:45 +0100 Subject: [PATCH 27/78] Reshuffle api endpoints --- .../scaffolder-backend/src/service/router.ts | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 8da6f02507..16cefa5297 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -124,68 +124,6 @@ export async function createRouter( error: job.error, }); }) - - .post('/v2/tasks', async (req, res) => { - const templateName: string = req.body.templateName; - const values: TemplaterValues = { - ...req.body.values, - destination: { - git: parseGitUrl(req.body.values.storePath), - }, - }; - const template = await entityClient.findTemplate(templateName); - - const validationResult: ValidatorResult = validate( - values, - template.spec.schema, - ); - - if (!validationResult.valid) { - res.status(400).json({ errors: validationResult.errors }); - return; - } - const taskSpec = templateEntityToSpec(template, values); - const result = await taskBroker.dispatch(taskSpec); - - res.status(201).json({ id: result.taskId }); - }) - - .get('/v2/tasks/:taskId/eventstream', async (req, res) => { - const { taskId } = req.params; - const after = Number(req.query.after) || undefined; - logger.info('event stream opened'); - - // Mandatory headers and http status to keep connection open - res.writeHead(200, { - Connection: 'keep-alive', - 'Cache-Control': 'no-cache', - 'Content-Type': 'text/event-stream', - }); - // After client opens connection send all nests as string - const unsubscribe = taskBroker.observe( - { taskId, after }, - (error, { events }) => { - logger.error( - `Received error from event stream when observing task ${taskId}`, - error, - ); - for (const event of events) { - res.write(`event:${JSON.stringify(event)}\n\n`); - if (event.type === 'completion') { - unsubscribe(); - res.end(); - } - } - }, - ); - // When client closes connection we update the clients list - // avoiding the disconnected one - req.on('close', () => { - unsubscribe(); - logger.info('event stream closed'); - }); - }) - .post('/v1/jobs', async (req, res) => { const templateName: string = req.body.templateName; const values: TemplaterValues = { @@ -282,6 +220,68 @@ export async function createRouter( res.status(201).json({ id: job.id }); }); + // NOTE: The v2 API is unstable + router + .post('/v2/tasks', async (req, res) => { + const templateName: string = req.body.templateName; + const values: TemplaterValues = { + ...req.body.values, + destination: { + git: parseGitUrl(req.body.values.storePath), + }, + }; + const template = await entityClient.findTemplate(templateName); + + const validationResult: ValidatorResult = validate( + values, + template.spec.schema, + ); + + if (!validationResult.valid) { + res.status(400).json({ errors: validationResult.errors }); + return; + } + const taskSpec = templateEntityToSpec(template, values); + const result = await taskBroker.dispatch(taskSpec); + + res.status(201).json({ id: result.taskId }); + }) + .get('/v2/tasks/:taskId/eventstream', async (req, res) => { + const { taskId } = req.params; + const after = Number(req.query.after) || undefined; + logger.info('event stream opened'); + + // Mandatory headers and http status to keep connection open + res.writeHead(200, { + Connection: 'keep-alive', + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }); + // After client opens connection send all nests as string + const unsubscribe = taskBroker.observe( + { taskId, after }, + (error, { events }) => { + logger.error( + `Received error from event stream when observing task ${taskId}`, + error, + ); + for (const event of events) { + res.write(`event:${JSON.stringify(event)}\n\n`); + if (event.type === 'completion') { + unsubscribe(); + res.end(); + } + } + }, + ); + // When client closes connection we update the clients list + // avoiding the disconnected one + req.on('close', () => { + unsubscribe(); + logger.info('event stream closed'); + }); + }); + const app = express(); app.set('logger', logger); app.use('/', router); From 615103a631993ec21745e6b3d93652f74c48cd4d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 16:33:00 +0100 Subject: [PATCH 28/78] Add changesets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .changeset/afraid-teachers-cross.md | 29 +++++++++++++++++++++++++++++ .changeset/large-terms-invite.md | 7 +++++++ 2 files changed, 36 insertions(+) create mode 100644 .changeset/afraid-teachers-cross.md create mode 100644 .changeset/large-terms-invite.md diff --git a/.changeset/afraid-teachers-cross.md b/.changeset/afraid-teachers-cross.md new file mode 100644 index 0000000000..8fc376441f --- /dev/null +++ b/.changeset/afraid-teachers-cross.md @@ -0,0 +1,29 @@ +--- +'@backstage/create-app': patch +--- + +Pass on plugin database management instance that is now required by the scaffolder plugin. + +To apply this change to an existing application, add the following to `src/plugins/scaffolder.ts`: + +```diff +export default async function createPlugin({ + logger, + config, ++ database, +}: PluginEnvironment) { + +// ...omitted... + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + entityClient, ++ database, + }); +} +``` diff --git a/.changeset/large-terms-invite.md b/.changeset/large-terms-invite.md new file mode 100644 index 0000000000..6653ac8e1e --- /dev/null +++ b/.changeset/large-terms-invite.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Introduced `v2` Scaffolder REST API, which uses an implementation that is database backed, making the scaffolder instances stateless. The `createRouter` function now requires a `PluginDatabaseManager` instance to be passed in, commonly available as `database` in the plugin environment in the backend. + +This API should be considered unstable until used by the scaffolder frontend. From a173d47527b0ceb960c3a2a815b59edf2ebb6e2e Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 16:45:28 +0100 Subject: [PATCH 29/78] Delete knexfile --- plugins/scaffolder-backend/knexfile.js | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 plugins/scaffolder-backend/knexfile.js diff --git a/plugins/scaffolder-backend/knexfile.js b/plugins/scaffolder-backend/knexfile.js deleted file mode 100644 index f469df4c08..0000000000 --- a/plugins/scaffolder-backend/knexfile.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module.exports = { - development: { - client: 'sqlite3', - connection: ':memory:', - migrations: { - directory: 'migrations', - }, - }, -}; From c386de896d65dbe05db2a1c23d0bc2a68a4aa864 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 2 Feb 2021 16:46:11 +0100 Subject: [PATCH 30/78] Remove unused dependencies --- plugins/scaffolder-backend/package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index bdecf6196e..9f45c218dc 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -55,11 +55,9 @@ "jsonschema": "^1.2.6", "knex": "^0.21.6", "morgan": "^1.10.0", - "p-queue": "^6.3.0", "uuid": "^8.2.0", "winston": "^3.2.1", - "yaml": "^1.10.0", - "luxon": "^1.25.0" + "yaml": "^1.10.0" }, "devDependencies": { "@backstage/cli": "^0.5.0", From 6b26c9f41eb059968db05998d5f4bbf50d75fbb4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 21:57:41 +0100 Subject: [PATCH 31/78] circleci: migrate to new composability API --- .changeset/curly-crabs-compete.md | 5 +++++ plugins/circleci/dev/index.tsx | 4 ++-- plugins/circleci/src/components/Router.tsx | 19 +++++++++++++++---- plugins/circleci/src/index.ts | 12 ++++++++++-- plugins/circleci/src/plugin.test.ts | 4 ++-- plugins/circleci/src/plugin.ts | 11 ++++++++++- 6 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 .changeset/curly-crabs-compete.md diff --git a/.changeset/curly-crabs-compete.md b/.changeset/curly-crabs-compete.md new file mode 100644 index 0000000000..552a41a9f1 --- /dev/null +++ b/.changeset/curly-crabs-compete.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-circleci': patch +--- + +Migrated to new composability API, exporting the plugin instance as `circleCIPlugin`, the entity page content as `EntityCircleCIContent`, and entity conditional as `isCircleCIAvailable`. diff --git a/plugins/circleci/dev/index.tsx b/plugins/circleci/dev/index.tsx index 812a5585d4..19a32c0d04 100644 --- a/plugins/circleci/dev/index.tsx +++ b/plugins/circleci/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { circleCIPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(circleCIPlugin).render(); diff --git a/plugins/circleci/src/components/Router.tsx b/plugins/circleci/src/components/Router.tsx index 5fe9a20eff..b3a0d2c3e2 100644 --- a/plugins/circleci/src/components/Router.tsx +++ b/plugins/circleci/src/components/Router.tsx @@ -21,15 +21,25 @@ import { BuildWithStepsPage } from './BuildWithStepsPage/'; import { BuildsPage } from './BuildsPage'; import { CIRCLECI_ANNOTATION } from '../constants'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { MissingAnnotationEmptyState } from '@backstage/core'; export const isPluginApplicableToEntity = (entity: Entity) => Boolean(entity.metadata.annotations?.[CIRCLECI_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isPluginApplicableToEntity(entity)) { + return ; + } + + return ( } /> /> ); +}; diff --git a/plugins/circleci/src/index.ts b/plugins/circleci/src/index.ts index c6cb140208..1e8b213279 100644 --- a/plugins/circleci/src/index.ts +++ b/plugins/circleci/src/index.ts @@ -14,8 +14,16 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + circleCIPlugin, + circleCIPlugin as plugin, + EntityCircleCIContent, +} from './plugin'; export * from './api'; export * from './route-refs'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isPluginApplicableToEntity, + isPluginApplicableToEntity as isCircleCIAvailable, +} from './components/Router'; export { CIRCLECI_ANNOTATION } from './constants'; diff --git a/plugins/circleci/src/plugin.test.ts b/plugins/circleci/src/plugin.test.ts index 821a503257..bdd637027c 100644 --- a/plugins/circleci/src/plugin.test.ts +++ b/plugins/circleci/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { circleCIPlugin } from './plugin'; describe('circleci', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(circleCIPlugin).toBeDefined(); }); }); diff --git a/plugins/circleci/src/plugin.ts b/plugins/circleci/src/plugin.ts index c5bb122ed3..4e0d485726 100644 --- a/plugins/circleci/src/plugin.ts +++ b/plugins/circleci/src/plugin.ts @@ -18,10 +18,12 @@ import { createPlugin, createApiFactory, discoveryApiRef, + createRoutableExtension, } from '@backstage/core'; import { circleCIApiRef, CircleCIApi } from './api'; +import { circleCIRouteRef } from './route-refs'; -export const plugin = createPlugin({ +export const circleCIPlugin = createPlugin({ id: 'circleci', apis: [ createApiFactory({ @@ -31,3 +33,10 @@ export const plugin = createPlugin({ }), ], }); + +export const EntityCircleCIContent = circleCIPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: circleCIRouteRef, + }), +); From 302795d107c2a2dedd521bf644b2d85283311f76 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 22:16:45 +0100 Subject: [PATCH 32/78] cloudbuild: migrate to now composability API --- .changeset/four-rings-push.md | 5 +++ plugins/cloudbuild/dev/index.tsx | 4 +-- plugins/cloudbuild/package.json | 1 + .../cloudbuild/src/components/Cards/Cards.tsx | 24 ++++++++------ plugins/cloudbuild/src/components/Router.tsx | 20 +++++++++--- plugins/cloudbuild/src/index.ts | 14 ++++++-- plugins/cloudbuild/src/plugin.test.ts | 4 +-- plugins/cloudbuild/src/plugin.ts | 32 ++++++++++++++++++- 8 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 .changeset/four-rings-push.md diff --git a/.changeset/four-rings-push.md b/.changeset/four-rings-push.md new file mode 100644 index 0000000000..500dd0e812 --- /dev/null +++ b/.changeset/four-rings-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cloudbuild': patch +--- + +Migrate to new composability API, exporting the plugin instance as `cloudbuildPlugin`, the entity content as `EntityCloudbuildContent`, the entity conditional as `isCloudbuildAvailable`, and entity cards as `EntityLatestCloudbuildRunCard` and `EntityLatestCloudbuildsForBranchCard`. diff --git a/plugins/cloudbuild/dev/index.tsx b/plugins/cloudbuild/dev/index.tsx index 264d6f801f..26fb65d151 100644 --- a/plugins/cloudbuild/dev/index.tsx +++ b/plugins/cloudbuild/dev/index.tsx @@ -14,6 +14,6 @@ * limitations under the License. */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { cloudbuildPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(cloudbuildPlugin).render(); diff --git a/plugins/cloudbuild/package.json b/plugins/cloudbuild/package.json index 3e5bbef3a9..7e9e13816b 100644 --- a/plugins/cloudbuild/package.json +++ b/plugins/cloudbuild/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.0", + "@backstage/plugin-catalog-react": "^0.0.1", "@backstage/core": "^0.5.0", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", diff --git a/plugins/cloudbuild/src/components/Cards/Cards.tsx b/plugins/cloudbuild/src/components/Cards/Cards.tsx index bbfb0b59c3..75ae33027e 100644 --- a/plugins/cloudbuild/src/components/Cards/Cards.tsx +++ b/plugins/cloudbuild/src/components/Cards/Cards.tsx @@ -17,6 +17,7 @@ import React, { useEffect } from 'react'; import { useWorkflowRuns } from '../useWorkflowRuns'; import { WorkflowRun, WorkflowRunsTable } from '../WorkflowRunsTable'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { WorkflowRunStatus } from '../WorkflowRunStatus'; import { Link, Theme, makeStyles, LinearProgress } from '@material-ui/core'; import { @@ -72,12 +73,13 @@ const WidgetContent = ({ }; export const LatestWorkflowRunCard = ({ - entity, branch = 'master', }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; }) => { + const { entity } = useEntity(); const errorApi = useApi(errorApiRef); const projectId = entity?.metadata.annotations?.[CLOUDBUILD_ANNOTATION] || ''; @@ -104,13 +106,17 @@ export const LatestWorkflowRunCard = ({ }; export const LatestWorkflowsForBranchCard = ({ - entity, branch = 'master', }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; -}) => ( - - - -); +}) => { + const { entity } = useEntity(); + + return ( + + + + ); +}; diff --git a/plugins/cloudbuild/src/components/Router.tsx b/plugins/cloudbuild/src/components/Router.tsx index af4959f4be..c31947a531 100644 --- a/plugins/cloudbuild/src/components/Router.tsx +++ b/plugins/cloudbuild/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Routes, Route } from 'react-router'; import { rootRouteRef, buildRouteRef } from '../plugin'; import { WorkflowRunDetails } from './WorkflowRunDetails'; @@ -25,11 +26,19 @@ import { MissingAnnotationEmptyState } from '@backstage/core'; export const isPluginApplicableToEntity = (entity: Entity) => Boolean(entity.metadata.annotations?.[CLOUDBUILD_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - // TODO(shmidt-i): move warning to a separate standardized component - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isPluginApplicableToEntity(entity)) { + // TODO(shmidt-i): move warning to a separate standardized component + return ; + } + return ( ) ); +}; diff --git a/plugins/cloudbuild/src/index.ts b/plugins/cloudbuild/src/index.ts index 616bb6b4e7..a846792fa2 100644 --- a/plugins/cloudbuild/src/index.ts +++ b/plugins/cloudbuild/src/index.ts @@ -13,8 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { plugin } from './plugin'; +export { + cloudbuildPlugin, + cloudbuildPlugin as plugin, + EntityCloudbuildContent, + EntityLatestCloudbuildRunCard, + EntityLatestCloudbuildsForBranchCard, +} from './plugin'; export * from './api'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isPluginApplicableToEntity, + isPluginApplicableToEntity as isCloudbuildAvailable, +} from './components/Router'; export * from './components/Cards'; export { CLOUDBUILD_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/cloudbuild/src/plugin.test.ts b/plugins/cloudbuild/src/plugin.test.ts index dae16f1d1b..ea042a901a 100644 --- a/plugins/cloudbuild/src/plugin.test.ts +++ b/plugins/cloudbuild/src/plugin.test.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { plugin } from './plugin'; +import { cloudbuildPlugin } from './plugin'; describe('cloudbuild', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(cloudbuildPlugin).toBeDefined(); }); }); diff --git a/plugins/cloudbuild/src/plugin.ts b/plugins/cloudbuild/src/plugin.ts index ee997cc7ec..f5aaf4696a 100644 --- a/plugins/cloudbuild/src/plugin.ts +++ b/plugins/cloudbuild/src/plugin.ts @@ -18,6 +18,8 @@ import { createRouteRef, createApiFactory, googleAuthApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { cloudbuildApiRef, CloudbuildClient } from './api'; @@ -31,7 +33,7 @@ export const buildRouteRef = createRouteRef({ title: 'Cloudbuild Run', }); -export const plugin = createPlugin({ +export const cloudbuildPlugin = createPlugin({ id: 'cloudbuild', apis: [ createApiFactory({ @@ -42,4 +44,32 @@ export const plugin = createPlugin({ }, }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityCloudbuildContent = cloudbuildPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); + +export const EntityLatestCloudbuildRunCard = cloudbuildPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowRunCard), + }, + }), +); + +export const EntityLatestCloudbuildsForBranchCard = cloudbuildPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowsForBranchCard), + }, + }), +); From f54025dde83bb62a251e5b5afa4c2a94a21f8c74 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 22:18:45 +0100 Subject: [PATCH 33/78] lighthouse: grab entity from context --- plugins/lighthouse/src/Router.tsx | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/plugins/lighthouse/src/Router.tsx b/plugins/lighthouse/src/Router.tsx index 0634a0aaa0..643d30cdc2 100644 --- a/plugins/lighthouse/src/Router.tsx +++ b/plugins/lighthouse/src/Router.tsx @@ -16,6 +16,7 @@ import React from 'react'; import { Route, Routes } from 'react-router-dom'; +import { useEntity } from '@backstage/plugin-catalog-react'; import AuditList from './components/AuditList'; import AuditView, { AuditViewContent } from './components/AuditView'; import CreateAudit, { CreateAuditContent } from './components/CreateAudit'; @@ -35,15 +36,27 @@ export const Router = () => ( ); -export const EmbeddedRouter = ({ entity }: { entity: Entity }) => - !isLighthouseAvailable(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const EmbeddedRouter = (_props: Props) => { + const { entity } = useEntity(); + + if (!isLighthouseAvailable(entity)) { + return ( + + ); + } + + return ( } /> } /> } /> ); +}; From b7fb8984c08a133b786c8fea46e26a172827f85b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 22:24:42 +0100 Subject: [PATCH 34/78] org: update cards to grab entity from context --- .../Group/GroupProfile/GroupProfileCard.tsx | 42 ++++++++++--------- .../Group/MembersList/MembersListCard.tsx | 9 ++-- .../Cards/OwnershipCard/OwnershipCard.tsx | 11 +++-- .../User/UserProfileCard/UserProfileCard.tsx | 7 ++-- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx index f5a1acd5e6..3b8d93b121 100644 --- a/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx +++ b/plugins/org/src/components/Cards/Group/GroupProfile/GroupProfileCard.tsx @@ -21,7 +21,7 @@ import { RELATION_PARENT_OF, } from '@backstage/catalog-model'; import { Avatar, InfoCard } from '@backstage/core'; -import { entityRouteParams } from '@backstage/plugin-catalog-react'; +import { useEntity, entityRouteParams } from '@backstage/plugin-catalog-react'; import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; import AccountTreeIcon from '@material-ui/icons/AccountTree'; import EmailIcon from '@material-ui/icons/Email'; @@ -33,26 +33,29 @@ import { generatePath, Link as RouterLink } from 'react-router-dom'; const GroupLink = ({ groupName, index = 0, - entity, }: { groupName: string; index?: number; - entity: Entity; -}) => ( - <> - {index >= 1 ? ', ' : ''} - - [{groupName}] - - -); + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}) => { + const { entity } = useEntity(); + return ( + <> + {index >= 1 ? ', ' : ''} + + [{groupName}] + + + ); +}; const CardTitle = ({ title }: { title: string }) => ( @@ -61,12 +64,13 @@ const CardTitle = ({ title }: { title: string }) => ( ); export const GroupProfileCard = ({ - entity: group, variant, }: { - entity: GroupEntity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: GroupEntity; variant: string; }) => { + const group = useEntity().entity as GroupEntity; const { metadata: { name, description }, spec: { profile }, diff --git a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx index f644e7db7e..7f8ede2eb7 100644 --- a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx +++ b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx @@ -21,6 +21,7 @@ import { } from '@backstage/catalog-model'; import { Avatar, InfoCard, Progress, useApi } from '@backstage/core'; import { + useEntity, catalogApiRef, entityRouteParams, } from '@backstage/plugin-catalog-react'; @@ -105,11 +106,11 @@ const MemberComponent = ({ ); }; -export const MembersListCard = ({ - entity: groupEntity, -}: { - entity: GroupEntity; +export const MembersListCard = (_props: { + /** @deprecated The entity is now grabbed from context instead */ + entity?: GroupEntity; }) => { + const groupEntity = useEntity().entity as GroupEntity; const { metadata: { name: groupName }, spec: { profile }, diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 60f06bd53a..80e80bddcc 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -16,7 +16,11 @@ import { Entity } from '@backstage/catalog-model'; import { InfoCard, Progress, useApi } from '@backstage/core'; -import { catalogApiRef, isOwnerOf } from '@backstage/plugin-catalog-react'; +import { + catalogApiRef, + isOwnerOf, + useEntity, +} from '@backstage/plugin-catalog-react'; import { pageTheme } from '@backstage/theme'; import { Box, @@ -113,12 +117,13 @@ const EntityCountTile = ({ }; export const OwnershipCard = ({ - entity, variant, }: { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; variant: string; }) => { + const { entity } = useEntity(); const catalogApi = useApi(catalogApiRef); const { loading, diff --git a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx index bdd0cd15f6..1cf7c968b3 100644 --- a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx +++ b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx @@ -19,7 +19,7 @@ import { UserEntity, } from '@backstage/catalog-model'; import { Avatar, InfoCard } from '@backstage/core'; -import { entityRouteParams } from '@backstage/plugin-catalog-react'; +import { useEntity, entityRouteParams } from '@backstage/plugin-catalog-react'; import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; import EmailIcon from '@material-ui/icons/Email'; import GroupIcon from '@material-ui/icons/Group'; @@ -60,12 +60,13 @@ const CardTitle = ({ title }: { title?: string }) => ) : null; export const UserProfileCard = ({ - entity: user, variant, }: { - entity: UserEntity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: UserEntity; variant: string; }) => { + const user = useEntity().entity as UserEntity; const { metadata: { name: metaName }, spec: { profile }, From f5f45744e36e09b9ac21442292780bf9a6f59826 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 23:11:22 +0100 Subject: [PATCH 35/78] github-actions: migrate to new composability API --- .changeset/selfish-years-pump.md | 5 +++ plugins/github-actions/dev/index.tsx | 4 +- plugins/github-actions/package.json | 1 + .../src/components/Cards/Cards.tsx | 21 ++++++---- .../Cards/RecentWorkflowRunsCard.tsx | 6 ++- .../github-actions/src/components/Router.tsx | 20 +++++++-- plugins/github-actions/src/index.ts | 15 ++++++- plugins/github-actions/src/plugin.test.ts | 4 +- plugins/github-actions/src/plugin.ts | 41 ++++++++++++++++++- 9 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 .changeset/selfish-years-pump.md diff --git a/.changeset/selfish-years-pump.md b/.changeset/selfish-years-pump.md new file mode 100644 index 0000000000..48994a1d9e --- /dev/null +++ b/.changeset/selfish-years-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-github-actions': patch +--- + +Migrate to new composability API, exporting the plugin instance as `githubActionsPlugin`, the entity content as `EntityGitHubActionsContent`, entity conditional as `isGitHubActionsAvailable`, and entity cards as `EntityLatestGitHubActionRunCard`, `EntityLatestGitHubActionsForBranchCard`, and `EntityRecentGitHubActionsRunsCard`. diff --git a/plugins/github-actions/dev/index.tsx b/plugins/github-actions/dev/index.tsx index 812a5585d4..9156224202 100644 --- a/plugins/github-actions/dev/index.tsx +++ b/plugins/github-actions/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { githubActionsPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(githubActionsPlugin).render(); diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index beb35b3cb0..33378c2022 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.0", + "@backstage/plugin-catalog-react": "^0.0.1", "@backstage/core": "^0.5.0", "@backstage/integration": "^0.3.1", "@backstage/theme": "^0.2.2", diff --git a/plugins/github-actions/src/components/Cards/Cards.tsx b/plugins/github-actions/src/components/Cards/Cards.tsx index f1c27cb666..202bf5aa92 100644 --- a/plugins/github-actions/src/components/Cards/Cards.tsx +++ b/plugins/github-actions/src/components/Cards/Cards.tsx @@ -17,6 +17,7 @@ import React, { useEffect } from 'react'; import { useWorkflowRuns } from '../useWorkflowRuns'; import { WorkflowRun, WorkflowRunsTable } from '../WorkflowRunsTable'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { readGitHubIntegrationConfigs } from '@backstage/integration'; import { WorkflowRunStatus } from '../WorkflowRunStatus'; import { @@ -81,11 +82,11 @@ const WidgetContent = ({ }; export const LatestWorkflowRunCard = ({ - entity, branch = 'master', // Display the card full height suitable for variant, }: Props) => { + const { entity } = useEntity(); const config = useApi(configApiRef); const errorApi = useApi(errorApiRef); // TODO: Get github hostname from metadata annotation @@ -121,17 +122,21 @@ export const LatestWorkflowRunCard = ({ }; type Props = { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch: string; variant?: string; }; export const LatestWorkflowsForBranchCard = ({ - entity, branch = 'master', variant, -}: Props) => ( - - - -); +}: Props) => { + const { entity } = useEntity(); + + return ( + + + + ); +}; diff --git a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx index aa30423c3e..4425bc8364 100644 --- a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx +++ b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.tsx @@ -23,6 +23,7 @@ import { useApi, } from '@backstage/core'; import { readGitHubIntegrationConfigs } from '@backstage/integration'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Button, Link } from '@material-ui/core'; import React, { useEffect } from 'react'; import { generatePath, Link as RouterLink } from 'react-router-dom'; @@ -33,7 +34,8 @@ import { WorkflowRunStatus } from '../WorkflowRunStatus'; const firstLine = (message: string): string => message.split('\n')[0]; export type Props = { - entity: Entity; + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; branch?: string; dense?: boolean; limit?: number; @@ -41,12 +43,12 @@ export type Props = { }; export const RecentWorkflowRunsCard = ({ - entity, branch, dense = false, limit = 5, variant, }: Props) => { + const { entity } = useEntity(); const config = useApi(configApiRef); const errorApi = useApi(errorApiRef); // TODO: Get github hostname from metadata annotation diff --git a/plugins/github-actions/src/components/Router.tsx b/plugins/github-actions/src/components/Router.tsx index e7430d40ed..e867b05b18 100644 --- a/plugins/github-actions/src/components/Router.tsx +++ b/plugins/github-actions/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Routes, Route } from 'react-router'; import { rootRouteRef, buildRouteRef } from '../plugin'; import { WorkflowRunDetails } from './WorkflowRunDetails'; @@ -25,10 +26,20 @@ import { MissingAnnotationEmptyState } from '@backstage/core'; export const isPluginApplicableToEntity = (entity: Entity) => Boolean(entity.metadata.annotations?.[GITHUB_ACTIONS_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => - !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isPluginApplicableToEntity(entity)) { + return ( + + ); + } + return ( ) ); +}; diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 24fe6fc90d..6dd3b9724e 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -14,8 +14,19 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + githubActionsPlugin, + githubActionsPlugin as plugin, + EntityGitHubActionsContent, + EntityLatestGitHubActionRunCard, + EntityLatestGitHubActionsForBranchCard, + EntityRecentGitHubActionsRunsCard, +} from './plugin'; export * from './api'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isPluginApplicableToEntity, + isPluginApplicableToEntity as isGitHubActionsAvailable, +} from './components/Router'; export * from './components/Cards'; export { GITHUB_ACTIONS_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/github-actions/src/plugin.test.ts b/plugins/github-actions/src/plugin.test.ts index fa1075cbc8..53bc96c77a 100644 --- a/plugins/github-actions/src/plugin.test.ts +++ b/plugins/github-actions/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { githubActionsPlugin } from './plugin'; describe('github-actions', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(githubActionsPlugin).toBeDefined(); }); }); diff --git a/plugins/github-actions/src/plugin.ts b/plugins/github-actions/src/plugin.ts index 39ffc468b1..71c05745d4 100644 --- a/plugins/github-actions/src/plugin.ts +++ b/plugins/github-actions/src/plugin.ts @@ -20,6 +20,8 @@ import { createRouteRef, createApiFactory, githubAuthApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { githubActionsApiRef, GithubActionsClient } from './api'; @@ -34,7 +36,7 @@ export const buildRouteRef = createRouteRef({ title: 'GitHub Actions Workflow Run', }); -export const plugin = createPlugin({ +export const githubActionsPlugin = createPlugin({ id: 'github-actions', apis: [ createApiFactory({ @@ -44,4 +46,41 @@ export const plugin = createPlugin({ new GithubActionsClient({ configApi, githubAuthApi }), }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityGitHubActionsContent = githubActionsPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); + +export const EntityLatestGitHubActionRunCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowRunCard), + }, + }), +); + +export const EntityLatestGitHubActionsForBranchCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.LatestWorkflowsForBranchCard), + }, + }), +); + +export const EntityRecentGitHubActionsRunsCard = githubActionsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/Cards').then(m => m.RecentWorkflowRunsCard), + }, + }), +); From 025c0c7bf8b8ecfaf368243a8e21c52ed3bfe643 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 2 Feb 2021 23:30:10 +0100 Subject: [PATCH 36/78] jenkins: migrate to new composability API --- .changeset/popular-cars-eat.md | 5 +++++ plugins/jenkins/dev/index.tsx | 4 ++-- plugins/jenkins/src/components/Router.tsx | 18 ++++++++++++++---- plugins/jenkins/src/index.ts | 13 +++++++++++-- plugins/jenkins/src/plugin.test.ts | 4 ++-- plugins/jenkins/src/plugin.ts | 21 ++++++++++++++++++++- 6 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 .changeset/popular-cars-eat.md diff --git a/.changeset/popular-cars-eat.md b/.changeset/popular-cars-eat.md new file mode 100644 index 0000000000..6d3bcdfab2 --- /dev/null +++ b/.changeset/popular-cars-eat.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins': patch +--- + +Migrate to new composability API, exporting the plugin instance as `jenkinsPlugin`, the entity content as `EntityJenkinsContent`, the entity conditional as `isJenkinsAvailable`, and the entity card as `EntityLatestJenkinsRunCard`. diff --git a/plugins/jenkins/dev/index.tsx b/plugins/jenkins/dev/index.tsx index 812a5585d4..8beb6eabc1 100644 --- a/plugins/jenkins/dev/index.tsx +++ b/plugins/jenkins/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { jenkinsPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(jenkinsPlugin).render(); diff --git a/plugins/jenkins/src/components/Router.tsx b/plugins/jenkins/src/components/Router.tsx index 91bcde2a8b..e310cd4193 100644 --- a/plugins/jenkins/src/components/Router.tsx +++ b/plugins/jenkins/src/components/Router.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; import { Route, Routes } from 'react-router'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { buildRouteRef, rootRouteRef } from '../plugin'; import { DetailedViewPage } from './BuildWithStepsPage/'; import { JENKINS_ANNOTATION } from '../constants'; @@ -25,10 +26,19 @@ import { CITable } from './BuildsPage/lib/CITable'; export const isPluginApplicableToEntity = (entity: Entity) => Boolean(entity.metadata.annotations?.[JENKINS_ANNOTATION]); -export const Router = ({ entity }: { entity: Entity }) => { - return !isPluginApplicableToEntity(entity) ? ( - - ) : ( +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const Router = (_props: Props) => { + const { entity } = useEntity(); + + if (!isPluginApplicableToEntity(entity)) { + return ; + } + + return ( } /> } /> diff --git a/plugins/jenkins/src/index.ts b/plugins/jenkins/src/index.ts index 30fa0c70a5..6d0e7837d3 100644 --- a/plugins/jenkins/src/index.ts +++ b/plugins/jenkins/src/index.ts @@ -14,8 +14,17 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + jenkinsPlugin, + jenkinsPlugin as plugin, + EntityJenkinsContent, + EntityLatestJenkinsRunCard, +} from './plugin'; export { LatestRunCard } from './components/Cards'; -export { Router, isPluginApplicableToEntity } from './components/Router'; +export { + Router, + isPluginApplicableToEntity, + isPluginApplicableToEntity as isJenkinsAvailable, +} from './components/Router'; export { JENKINS_ANNOTATION } from './constants'; export * from './api'; diff --git a/plugins/jenkins/src/plugin.test.ts b/plugins/jenkins/src/plugin.test.ts index 8c3f057ecb..0498b80b90 100644 --- a/plugins/jenkins/src/plugin.test.ts +++ b/plugins/jenkins/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { jenkinsPlugin } from './plugin'; describe('jenkins', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(jenkinsPlugin).toBeDefined(); }); }); diff --git a/plugins/jenkins/src/plugin.ts b/plugins/jenkins/src/plugin.ts index 23c54743a3..85001d9bbc 100644 --- a/plugins/jenkins/src/plugin.ts +++ b/plugins/jenkins/src/plugin.ts @@ -19,6 +19,8 @@ import { createRouteRef, createApiFactory, discoveryApiRef, + createRoutableExtension, + createComponentExtension, } from '@backstage/core'; import { jenkinsApiRef, JenkinsApi } from './api'; @@ -32,7 +34,7 @@ export const buildRouteRef = createRouteRef({ title: 'Jenkins run', }); -export const plugin = createPlugin({ +export const jenkinsPlugin = createPlugin({ id: 'jenkins', apis: [ createApiFactory({ @@ -41,4 +43,21 @@ export const plugin = createPlugin({ factory: ({ discoveryApi }) => new JenkinsApi({ discoveryApi }), }), ], + routes: { + entityContent: rootRouteRef, + }, }); + +export const EntityJenkinsContent = jenkinsPlugin.provide( + createRoutableExtension({ + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); +export const EntityLatestJenkinsRunCard = jenkinsPlugin.provide( + createComponentExtension({ + component: { + lazy: () => import('./components/Cards').then(m => m.LatestRunCard), + }, + }), +); From f20f0465141f676f42b39ed467f9eaac5439fe2f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 3 Feb 2021 00:03:07 +0100 Subject: [PATCH 37/78] github-actions: fix card tests --- .../src/components/Cards/RecentWorkflowRunsCard.test.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx index 0deb4ae39d..18cc59439b 100644 --- a/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx +++ b/plugins/github-actions/src/components/Cards/RecentWorkflowRunsCard.test.tsx @@ -22,6 +22,7 @@ import { ConfigApi, ConfigReader, } from '@backstage/core'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; import { lightTheme } from '@backstage/theme'; import { ThemeProvider } from '@material-ui/core'; import { render } from '@testing-library/react'; @@ -84,7 +85,9 @@ describe('', () => { configApi, )} > - + + + , From b4358c46e0b13e3bfd318d5b345aa55e1acbad39 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 3 Feb 2021 10:03:50 +0100 Subject: [PATCH 38/78] org: fix entity card tests --- .../Cards/Group/MembersList/MembersListCard.test.tsx | 11 +++++++++-- .../User/UserProfileCard/UserProfileCard.test.tsx | 7 ++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx index 6815fd0e55..28c55b376b 100644 --- a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx +++ b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.test.tsx @@ -16,7 +16,11 @@ import { Entity, GroupEntity } from '@backstage/catalog-model'; import { ApiProvider, ApiRegistry } from '@backstage/core'; -import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; +import { + CatalogApi, + catalogApiRef, + EntityProvider, +} from '@backstage/plugin-catalog-react'; import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; import React from 'react'; import { MembersListCard } from './MembersListCard'; @@ -78,7 +82,10 @@ describe('MemberTab Test', () => { const rendered = await renderWithEffects( wrapInTestApp( - + + + + , , ), ); diff --git a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx index 77708e6f27..7c7d26c228 100644 --- a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx +++ b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.test.tsx @@ -15,6 +15,7 @@ */ import { UserEntity } from '@backstage/catalog-model'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; import React from 'react'; import { UserProfileCard } from './UserProfileCard'; @@ -48,7 +49,11 @@ describe('UserSummary Test', () => { it('Display Profile Card', async () => { const rendered = await renderWithEffects( - wrapInTestApp(), + wrapInTestApp( + + + , + ), ); expect(rendered.getByText('calum-leavy@example.com')).toBeInTheDocument(); From 693ae8c6258d5cd923e89a981e8f2fae2a07eaff Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 3 Feb 2021 10:41:06 +0100 Subject: [PATCH 39/78] Remove date precision --- plugins/scaffolder-backend/migrations/20210120143715_init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js index 248b4f932f..256ec7d423 100644 --- a/plugins/scaffolder-backend/migrations/20210120143715_init.js +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -62,7 +62,7 @@ exports.up = async function up(knex) { .comment('The JSON encoded body of the event'); table.text('event_type').notNullable().comment('The type of event'); table - .timestamp('created_at', { precision: 9 }) + .timestamp('created_at') .defaultTo(knex.fn.now()) .notNullable() .comment('The timestamp when this event was generated'); From 1955c62398793a3c385de98e8667fb9e2b0d3ffe Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 3 Feb 2021 10:47:44 +0100 Subject: [PATCH 40/78] Delete unused database code --- .../scaffolder-backend/src/tasks/Database.ts | 70 ------------------- plugins/scaffolder-backend/src/tasks/types.ts | 37 ---------- 2 files changed, 107 deletions(-) delete mode 100644 plugins/scaffolder-backend/src/tasks/Database.ts delete mode 100644 plugins/scaffolder-backend/src/tasks/types.ts diff --git a/plugins/scaffolder-backend/src/tasks/Database.ts b/plugins/scaffolder-backend/src/tasks/Database.ts deleted file mode 100644 index c135380f87..0000000000 --- a/plugins/scaffolder-backend/src/tasks/Database.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ConflictError, resolvePackagePath } from '@backstage/backend-common'; -import Knex from 'knex'; -import { Logger } from 'winston'; -import { Database, Transaction } from './types'; - -const migrationsDir = resolvePackagePath( - '@backstage/plugin-scaffolder-backend', - 'migrations', -); - -export class CommonDatabase implements Database { - static async create(knex: Knex, logger: Logger): Promise { - await knex.migrate.latest({ - directory: migrationsDir, - }); - return new CommonDatabase(knex, logger); - } - - constructor( - private readonly database: Knex, - private readonly logger: Logger, - ) {} - - async transaction(fn: (tx: Transaction) => Promise): Promise { - try { - let result: T | undefined = undefined; - - await this.database.transaction( - async tx => { - // We can't return here, as knex swallows the return type in case the transaction is rolled back: - // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 - result = await fn(tx); - }, - { - // If we explicitly trigger a rollback, don't fail. - doNotRejectOnRollback: true, - }, - ); - - return result!; - } catch (e) { - this.logger.debug(`Error during transaction, ${e}`); - - if ( - /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || - /unique constraint/.test(e.message) - ) { - throw new ConflictError(`Rejected due to a conflicting entity`, e); - } - - throw e; - } - } -} diff --git a/plugins/scaffolder-backend/src/tasks/types.ts b/plugins/scaffolder-backend/src/tasks/types.ts deleted file mode 100644 index 27cbe7f7f1..0000000000 --- a/plugins/scaffolder-backend/src/tasks/types.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The core database implementation. - */ -export interface Database { - /** - * Runs a transaction. - * - * The callback is expected to make calls back into this class. When it - * completes, the transaction is closed. - * - * @param fn The callback that implements the transaction - */ - transaction(fn: (tx: Transaction) => Promise): Promise; -} - -/** - * An abstraction for transactions of the underlying database technology. - */ -export type Transaction = { - rollback(): Promise; -}; From e5520064eabd725c038c878220522bd1aa2b3db9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 3 Feb 2021 10:48:34 +0100 Subject: [PATCH 41/78] Drop console log --- plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index a3fe01c96d..cb06f5a1e7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -37,7 +37,6 @@ export function registerLegacyActions( id: 'legacy:prepare', async handler(ctx) { const { logger } = ctx; - console.log(ctx); logger.info('Task claimed, waiting ...'); // Give us some time to curl observe await new Promise(resolve => setTimeout(resolve, 1000)); From ff0eed8200e0a5886f8201bf0358a1e0ded3a9bf Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 3 Feb 2021 10:51:48 +0100 Subject: [PATCH 42/78] Remove sleep from prepare step --- plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index cb06f5a1e7..6daf1ba4f6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -36,16 +36,11 @@ export function registerLegacyActions( registry.register({ id: 'legacy:prepare', async handler(ctx) { - const { logger } = ctx; - logger.info('Task claimed, waiting ...'); - // Give us some time to curl observe - await new Promise(resolve => setTimeout(resolve, 1000)); - - logger.info('Prepare the skeleton'); const { protocol, url } = ctx.parameters; const preparer = protocol === 'file' ? new FilePreparer() : preparers.get(url as string); + ctx.logger.info('Prepare the skeleton'); await preparer.prepare({ url: url as string, logger: ctx.logger, From 586607e417877aaa34d92845731c969aab0ac4c3 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 4 Feb 2021 09:39:05 +0100 Subject: [PATCH 43/78] Order events by id Co-authored-by: blam --- .../scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index cf1bc18621..22697f5fdb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -245,6 +245,7 @@ export class DatabaseTaskStore implements TaskStore { builder.where('id', '>', after).orWhere('event_type', 'completion'); } }) + .orderBy('id') .select(); const events = rawEvents.map(event => { From 6ca31fb834ea620e2c4de23d26efd076ffe6646c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 4 Feb 2021 09:42:04 +0100 Subject: [PATCH 44/78] scaffolder: Flush events to eventstream Co-authored-by: blam --- .../scaffolder-backend/src/service/router.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 16cefa5297..a8a61972af 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -249,7 +249,7 @@ export async function createRouter( .get('/v2/tasks/:taskId/eventstream', async (req, res) => { const { taskId } = req.params; const after = Number(req.query.after) || undefined; - logger.info('event stream opened'); + logger.debug(`Event stream observing taskId '${taskId}' opened`); // Mandatory headers and http status to keep connection open res.writeHead(200, { @@ -257,28 +257,35 @@ export async function createRouter( 'Cache-Control': 'no-cache', 'Content-Type': 'text/event-stream', }); + // After client opens connection send all nests as string const unsubscribe = taskBroker.observe( { taskId, after }, (error, { events }) => { - logger.error( - `Received error from event stream when observing task ${taskId}`, - error, - ); + if (error) { + logger.error( + `Received error from event stream when observing taskId '${taskId}', ${error}`, + ); + } + for (const event of events) { - res.write(`event:${JSON.stringify(event)}\n\n`); + res.write( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ); if (event.type === 'completion') { unsubscribe(); - res.end(); + // Closing the event stream here would cause the frontend + // to automatically reconnect because it lost connection. } } + res.flush(); }, ); // When client closes connection we update the clients list // avoiding the disconnected one req.on('close', () => { unsubscribe(); - logger.info('event stream closed'); + logger.debug(`Event stream observing taskId '${taskId}' closed`); }); }); From accdfeb30b019ce7ece3b080d1a7779d17650672 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 00:38:45 +0100 Subject: [PATCH 45/78] gitops-profiles: migrate to new composability API --- .changeset/two-dogs-search.md | 5 ++++ plugins/gitops-profiles/dev/index.tsx | 4 +-- plugins/gitops-profiles/src/index.ts | 8 ++++- plugins/gitops-profiles/src/plugin.test.ts | 4 +-- plugins/gitops-profiles/src/plugin.ts | 34 ++++++++++++++++++++-- plugins/gitops-profiles/src/routes.ts | 1 + 6 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 .changeset/two-dogs-search.md diff --git a/.changeset/two-dogs-search.md b/.changeset/two-dogs-search.md new file mode 100644 index 0000000000..5f673c0cfb --- /dev/null +++ b/.changeset/two-dogs-search.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-gitops-profiles': patch +--- + +Migrated to new composability API, exporting the plugin instance as `gitopsProfilesPlugin` and pages as `GitopsProfilesClusterListPage`, `GitopsProfilesClusterPage`, and `GitopsProfilesCreatePage`. diff --git a/plugins/gitops-profiles/dev/index.tsx b/plugins/gitops-profiles/dev/index.tsx index 812a5585d4..c164e4005d 100644 --- a/plugins/gitops-profiles/dev/index.tsx +++ b/plugins/gitops-profiles/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { gitopsProfilesPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(gitopsProfilesPlugin).render(); diff --git a/plugins/gitops-profiles/src/index.ts b/plugins/gitops-profiles/src/index.ts index d67bc6a864..876c7c3263 100644 --- a/plugins/gitops-profiles/src/index.ts +++ b/plugins/gitops-profiles/src/index.ts @@ -14,5 +14,11 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + gitopsProfilesPlugin, + gitopsProfilesPlugin as plugin, + GitopsProfilesClusterListPage, + GitopsProfilesClusterPage, + GitopsProfilesCreatePage, +} from './plugin'; export * from './api'; diff --git a/plugins/gitops-profiles/src/plugin.test.ts b/plugins/gitops-profiles/src/plugin.test.ts index 19bc49eba7..fa26902f2b 100644 --- a/plugins/gitops-profiles/src/plugin.test.ts +++ b/plugins/gitops-profiles/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { gitopsProfilesPlugin } from './plugin'; describe('gitops-profiles', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(gitopsProfilesPlugin).toBeDefined(); }); }); diff --git a/plugins/gitops-profiles/src/plugin.ts b/plugins/gitops-profiles/src/plugin.ts index 45820644f7..98a0067969 100644 --- a/plugins/gitops-profiles/src/plugin.ts +++ b/plugins/gitops-profiles/src/plugin.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { createPlugin, createApiFactory } from '@backstage/core'; +import { + createPlugin, + createApiFactory, + createRoutableExtension, +} from '@backstage/core'; import ProfileCatalog from './components/ProfileCatalog'; import ClusterPage from './components/ClusterPage'; import ClusterList from './components/ClusterList'; @@ -25,7 +29,7 @@ import { } from './routes'; import { gitOpsApiRef, GitOpsRestApi } from './api'; -export const plugin = createPlugin({ +export const gitopsProfilesPlugin = createPlugin({ id: 'gitops-profiles', apis: [ createApiFactory(gitOpsApiRef, new GitOpsRestApi('http://localhost:3008')), @@ -35,4 +39,30 @@ export const plugin = createPlugin({ router.addRoute(gitOpsClusterDetailsRoute, ClusterPage); router.addRoute(gitOpsClusterCreateRoute, ProfileCatalog); }, + routes: { + listPage: gitOpsClusterListRoute, + detailsPage: gitOpsClusterDetailsRoute, + createPage: gitOpsClusterCreateRoute, + }, }); + +export const GitopsProfilesClusterListPage = gitopsProfilesPlugin.provide( + createRoutableExtension({ + component: () => import('./components/ClusterList').then(m => m.default), + mountPoint: gitOpsClusterListRoute, + }), +); + +export const GitopsProfilesClusterPage = gitopsProfilesPlugin.provide( + createRoutableExtension({ + component: () => import('./components/ClusterPage').then(m => m.default), + mountPoint: gitOpsClusterDetailsRoute, + }), +); + +export const GitopsProfilesCreatePage = gitopsProfilesPlugin.provide( + createRoutableExtension({ + component: () => import('./components/ProfileCatalog').then(m => m.default), + mountPoint: gitOpsClusterCreateRoute, + }), +); diff --git a/plugins/gitops-profiles/src/routes.ts b/plugins/gitops-profiles/src/routes.ts index f9bdefa806..28be85bd38 100644 --- a/plugins/gitops-profiles/src/routes.ts +++ b/plugins/gitops-profiles/src/routes.ts @@ -28,6 +28,7 @@ export const gitOpsClusterDetailsRoute = createRouteRef({ icon: NoIcon, path: '/gitops-cluster/:owner/:repo', title: 'GitOps Cluster details', + params: ['owner', 'repo'], }); export const gitOpsClusterCreateRoute = createRouteRef({ From b288a291ee393f15f711e84d3e86d8fa8929a81e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 00:52:33 +0100 Subject: [PATCH 46/78] pagerduty: migrate to new composability API --- .changeset/perfect-ladybugs-listen.md | 5 +++++ plugins/pagerduty/dev/index.tsx | 4 ++-- plugins/pagerduty/package.json | 1 + plugins/pagerduty/src/components/PagerDutyCard.tsx | 5 ++++- plugins/pagerduty/src/index.ts | 7 ++++++- plugins/pagerduty/src/plugin.test.ts | 4 ++-- plugins/pagerduty/src/plugin.ts | 12 +++++++++++- 7 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 .changeset/perfect-ladybugs-listen.md diff --git a/.changeset/perfect-ladybugs-listen.md b/.changeset/perfect-ladybugs-listen.md new file mode 100644 index 0000000000..2f9e65c1d9 --- /dev/null +++ b/.changeset/perfect-ladybugs-listen.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-pagerduty': patch +--- + +Migrated to new composability API, exporting the plugin instance as `pagerDutyPlugin`, entity card as `EntityPagerDutyCard`, and entity conditional as `isPagerDutyAvailable`. diff --git a/plugins/pagerduty/dev/index.tsx b/plugins/pagerduty/dev/index.tsx index 264d6f801f..ad46c0ca9e 100644 --- a/plugins/pagerduty/dev/index.tsx +++ b/plugins/pagerduty/dev/index.tsx @@ -14,6 +14,6 @@ * limitations under the License. */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { pagerDutyPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(pagerDutyPlugin).render(); diff --git a/plugins/pagerduty/package.json b/plugins/pagerduty/package.json index a5db20e9ab..afd41dcc46 100644 --- a/plugins/pagerduty/package.json +++ b/plugins/pagerduty/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.7.1", + "@backstage/plugin-catalog-react": "^0.0.2", "@backstage/core": "^0.6.0", "@backstage/theme": "^0.2.3", "@material-ui/core": "^4.11.0", diff --git a/plugins/pagerduty/src/components/PagerDutyCard.tsx b/plugins/pagerduty/src/components/PagerDutyCard.tsx index 5fb2cd7f4a..f7360c01d6 100644 --- a/plugins/pagerduty/src/components/PagerDutyCard.tsx +++ b/plugins/pagerduty/src/components/PagerDutyCard.tsx @@ -16,6 +16,7 @@ import React, { useState, useCallback } from 'react'; import { useApi, Progress, HeaderIconLinkRow } from '@backstage/core'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Button, makeStyles, @@ -56,11 +57,13 @@ export const isPluginApplicableToEntity = (entity: Entity) => Boolean(entity.metadata.annotations?.[PAGERDUTY_INTEGRATION_KEY]); type Props = { + /** @deprecated The entity is now grabbed from context instead */ entity: Entity; }; -export const PagerDutyCard = ({ entity }: Props) => { +export const PagerDutyCard = (_props: Props) => { const classes = useStyles(); + const { entity } = useEntity(); const api = useApi(pagerDutyApiRef); const [showDialog, setShowDialog] = useState(false); const [refreshIncidents, setRefreshIncidents] = useState(false); diff --git a/plugins/pagerduty/src/index.ts b/plugins/pagerduty/src/index.ts index 4ecd4edcc6..2dfedab164 100644 --- a/plugins/pagerduty/src/index.ts +++ b/plugins/pagerduty/src/index.ts @@ -13,9 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { plugin } from './plugin'; +export { + pagerDutyPlugin, + pagerDutyPlugin as plugin, + EntityPagerDutyCard, +} from './plugin'; export { isPluginApplicableToEntity, + isPluginApplicableToEntity as isPagerDutyAvailable, PagerDutyCard, } from './components/PagerDutyCard'; export { diff --git a/plugins/pagerduty/src/plugin.test.ts b/plugins/pagerduty/src/plugin.test.ts index 8d4545ac12..c1175ab384 100644 --- a/plugins/pagerduty/src/plugin.test.ts +++ b/plugins/pagerduty/src/plugin.test.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { plugin } from './plugin'; +import { pagerDutyPlugin } from './plugin'; describe('pagerduty', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(pagerDutyPlugin).toBeDefined(); }); }); diff --git a/plugins/pagerduty/src/plugin.ts b/plugins/pagerduty/src/plugin.ts index 34796f491e..fbe827b90d 100644 --- a/plugins/pagerduty/src/plugin.ts +++ b/plugins/pagerduty/src/plugin.ts @@ -19,6 +19,7 @@ import { createRouteRef, discoveryApiRef, configApiRef, + createComponentExtension, } from '@backstage/core'; import { pagerDutyApiRef, PagerDutyClient } from './api'; @@ -27,7 +28,7 @@ export const rootRouteRef = createRouteRef({ title: 'pagerduty', }); -export const plugin = createPlugin({ +export const pagerDutyPlugin = createPlugin({ id: 'pagerduty', apis: [ createApiFactory({ @@ -38,3 +39,12 @@ export const plugin = createPlugin({ }), ], }); + +export const EntityPagerDutyCard = pagerDutyPlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./components/PagerDutyCard').then(m => m.PagerDutyCard), + }, + }), +); From b3f0c38112be1afddc30fe4260ce3ed7dc160d18 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 12:02:28 +0100 Subject: [PATCH 47/78] search: migrate to new composability API --- .changeset/cyan-lies-flow.md | 5 +++++ plugins/search/dev/index.tsx | 4 ++-- plugins/search/src/index.ts | 12 ++++++++++-- plugins/search/src/plugin.test.ts | 4 ++-- plugins/search/src/plugin.ts | 22 ++++++++++++++++++---- 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 .changeset/cyan-lies-flow.md diff --git a/.changeset/cyan-lies-flow.md b/.changeset/cyan-lies-flow.md new file mode 100644 index 0000000000..e7da9fab0a --- /dev/null +++ b/.changeset/cyan-lies-flow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search': minor +--- + +Migrated to new composability API, exporting the plugin instance as `searchPlugin`, and page as `SearchPage`. Due to the old router component also being called `SearchPage`, this is a breaking change. The old page component is now exported as `Router`, which can be used to maintain the old behavior. diff --git a/plugins/search/dev/index.tsx b/plugins/search/dev/index.tsx index 264d6f801f..e6e97ead6d 100644 --- a/plugins/search/dev/index.tsx +++ b/plugins/search/dev/index.tsx @@ -14,6 +14,6 @@ * limitations under the License. */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { searchPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(searchPlugin).render(); diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts index 77ad7f9266..572d75bea5 100644 --- a/plugins/search/src/index.ts +++ b/plugins/search/src/index.ts @@ -13,5 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { plugin } from './plugin'; -export * from './components'; +export { searchPlugin, searchPlugin as plugin, SearchPage } from './plugin'; +export { + Filters, + FiltersButton, + SearchBar, + SearchPage as Router, + SearchResult, + SidebarSearch, +} from './components'; +export type { FiltersState } from './components'; diff --git a/plugins/search/src/plugin.test.ts b/plugins/search/src/plugin.test.ts index 92b8d5bcbf..902faeaf9e 100644 --- a/plugins/search/src/plugin.test.ts +++ b/plugins/search/src/plugin.test.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { plugin } from './plugin'; +import { searchPlugin } from './plugin'; describe('search', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(searchPlugin).toBeDefined(); }); }); diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index 44c7bcb042..37503b7751 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -13,17 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { createPlugin, createRouteRef } from '@backstage/core'; -import { SearchPage } from './components/SearchPage'; +import { + createPlugin, + createRouteRef, + createRoutableExtension, +} from '@backstage/core'; +import { SearchPage as SearchPageComponent } from './components/SearchPage'; export const rootRouteRef = createRouteRef({ path: '/search', title: 'search', }); -export const plugin = createPlugin({ +export const searchPlugin = createPlugin({ id: 'search', register({ router }) { - router.addRoute(rootRouteRef, SearchPage); + router.addRoute(rootRouteRef, SearchPageComponent); + }, + routes: { + root: rootRouteRef, }, }); + +export const SearchPage = searchPlugin.provide( + createRoutableExtension({ + component: () => import('./components/SearchPage').then(m => m.SearchPage), + mountPoint: rootRouteRef, + }), +); From 9ec66c3459d170d4bdd698e6f8b1d29b56ad9df0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 19:57:44 +0100 Subject: [PATCH 48/78] register-component: migrated to new composability API --- .changeset/funny-students-shout.md | 5 ++++ plugins/register-component/dev/index.tsx | 4 +-- plugins/register-component/src/index.ts | 6 ++++- plugins/register-component/src/plugin.test.ts | 4 +-- plugins/register-component/src/plugin.ts | 27 ++++++++++++++++--- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 .changeset/funny-students-shout.md diff --git a/.changeset/funny-students-shout.md b/.changeset/funny-students-shout.md new file mode 100644 index 0000000000..97b273c687 --- /dev/null +++ b/.changeset/funny-students-shout.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-register-component': patch +--- + +Migrated to new composability API, exporting the plugin instance as `registerComponentPlugin`, and page as `RegisterComponentPage`. diff --git a/plugins/register-component/dev/index.tsx b/plugins/register-component/dev/index.tsx index 812a5585d4..3304d0874f 100644 --- a/plugins/register-component/dev/index.tsx +++ b/plugins/register-component/dev/index.tsx @@ -15,6 +15,6 @@ */ import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { registerComponentPlugin } from '../src/plugin'; -createDevApp().registerPlugin(plugin).render(); +createDevApp().registerPlugin(registerComponentPlugin).render(); diff --git a/plugins/register-component/src/index.ts b/plugins/register-component/src/index.ts index ff7857cacd..56bcd06fde 100644 --- a/plugins/register-component/src/index.ts +++ b/plugins/register-component/src/index.ts @@ -14,5 +14,9 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + registerComponentPlugin, + registerComponentPlugin as plugin, + RegisterComponentPage, +} from './plugin'; export { Router } from './components/Router'; diff --git a/plugins/register-component/src/plugin.test.ts b/plugins/register-component/src/plugin.test.ts index 1e61060202..2b6ad48bcf 100644 --- a/plugins/register-component/src/plugin.test.ts +++ b/plugins/register-component/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { registerComponentPlugin } from './plugin'; describe('register-component', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(registerComponentPlugin).toBeDefined(); }); }); diff --git a/plugins/register-component/src/plugin.ts b/plugins/register-component/src/plugin.ts index 17f99917c8..d3727158e5 100644 --- a/plugins/register-component/src/plugin.ts +++ b/plugins/register-component/src/plugin.ts @@ -14,8 +14,29 @@ * limitations under the License. */ -import { createPlugin } from '@backstage/core'; +import { + createPlugin, + createRoutableExtension, + createRouteRef, +} from '@backstage/core'; -export const plugin = createPlugin({ - id: 'register-component', +const rootRouteRef = createRouteRef({ + title: 'Register Component', }); + +export const registerComponentPlugin = createPlugin({ + id: 'register-component', + routes: { + root: rootRouteRef, + }, +}); + +export const RegisterComponentPage = registerComponentPlugin.provide( + createRoutableExtension({ + component: () => + import('./components/RegisterComponentPage').then( + m => m.RegisterComponentPage, + ), + mountPoint: rootRouteRef, + }), +); From 41af18227fd4893a24f39cb8512556bee2e63e58 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 20:07:12 +0100 Subject: [PATCH 49/78] techdocs: migrate to new composability API --- .changeset/ninety-houses-shout.md | 5 +++++ plugins/techdocs/dev/index.tsx | 4 ++-- plugins/techdocs/src/Router.tsx | 10 +++++++++- plugins/techdocs/src/index.ts | 7 ++++++- plugins/techdocs/src/plugin.test.ts | 4 ++-- plugins/techdocs/src/plugin.ts | 21 ++++++++++++++++++++- 6 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 .changeset/ninety-houses-shout.md diff --git a/.changeset/ninety-houses-shout.md b/.changeset/ninety-houses-shout.md new file mode 100644 index 0000000000..845decc74c --- /dev/null +++ b/.changeset/ninety-houses-shout.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Migrated to new composability API, exporting the plugin instance as `techdocsPlugin`, the top-level page as `TechdocsPage`, and the entity content as `EntityTechdocsContent`. diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index 42fa9a9ec4..4cf74c9622 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -16,7 +16,7 @@ import { configApiRef, discoveryApiRef } from '@backstage/core'; import { createDevApp } from '@backstage/dev-utils'; -import { plugin } from '../src/plugin'; +import { techdocsPlugin } from '../src/plugin'; import { TechDocsDevStorageApi } from './api'; import { techdocsStorageApiRef } from '../src'; @@ -30,5 +30,5 @@ createDevApp() discoveryApi, }), }) - .registerPlugin(plugin) + .registerPlugin(techdocsPlugin) .render(); diff --git a/plugins/techdocs/src/Router.tsx b/plugins/techdocs/src/Router.tsx index 262f542ee6..b220912174 100644 --- a/plugins/techdocs/src/Router.tsx +++ b/plugins/techdocs/src/Router.tsx @@ -16,6 +16,7 @@ import React from 'react'; import { Entity } from '@backstage/catalog-model'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Route, Routes } from 'react-router-dom'; import { MissingAnnotationEmptyState } from '@backstage/core'; import { @@ -38,7 +39,14 @@ export const Router = () => { ); }; -export const EmbeddedDocsRouter = ({ entity }: { entity: Entity }) => { +type Props = { + /** @deprecated The entity is now grabbed from context instead */ + entity?: Entity; +}; + +export const EmbeddedDocsRouter = (_props: Props) => { + const { entity } = useEntity(); + const projectId = entity.metadata.annotations?.[TECHDOCS_ANNOTATION]; if (!projectId) { diff --git a/plugins/techdocs/src/index.ts b/plugins/techdocs/src/index.ts index 1726e0daf3..30c143e31f 100644 --- a/plugins/techdocs/src/index.ts +++ b/plugins/techdocs/src/index.ts @@ -14,7 +14,12 @@ * limitations under the License. */ -export { plugin } from './plugin'; +export { + techdocsPlugin, + techdocsPlugin as plugin, + TechdocsPage, + EntityTechdocsContent, +} from './plugin'; export { Router, EmbeddedDocsRouter } from './Router'; export * from './reader'; export * from './api'; diff --git a/plugins/techdocs/src/plugin.test.ts b/plugins/techdocs/src/plugin.test.ts index e750be9ac3..52f072627c 100644 --- a/plugins/techdocs/src/plugin.test.ts +++ b/plugins/techdocs/src/plugin.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { plugin } from './plugin'; +import { techdocsPlugin } from './plugin'; describe('techdocs', () => { it('should export plugin', () => { - expect(plugin).toBeDefined(); + expect(techdocsPlugin).toBeDefined(); }); }); diff --git a/plugins/techdocs/src/plugin.ts b/plugins/techdocs/src/plugin.ts index 982b749f2c..0b4063c237 100644 --- a/plugins/techdocs/src/plugin.ts +++ b/plugins/techdocs/src/plugin.ts @@ -35,6 +35,7 @@ import { createApiFactory, configApiRef, discoveryApiRef, + createRoutableExtension, } from '@backstage/core'; import { techdocsStorageApiRef, @@ -58,7 +59,7 @@ export const rootCatalogDocsRouteRef = createRouteRef({ title: 'Docs', }); -export const plugin = createPlugin({ +export const techdocsPlugin = createPlugin({ id: 'techdocs', apis: [ createApiFactory({ @@ -80,4 +81,22 @@ export const plugin = createPlugin({ }), }), ], + routes: { + root: rootRouteRef, + entityContent: rootCatalogDocsRouteRef, + }, }); + +export const TechdocsPage = techdocsPlugin.provide( + createRoutableExtension({ + component: () => import('./Router').then(m => m.Router), + mountPoint: rootRouteRef, + }), +); + +export const EntityTechdocsContent = techdocsPlugin.provide( + createRoutableExtension({ + component: () => import('./Router').then(m => m.EmbeddedDocsRouter), + mountPoint: rootCatalogDocsRouteRef, + }), +); From 7c789117881a4654822383ebfcdea88066b1a9bd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 4 Feb 2021 20:36:18 +0100 Subject: [PATCH 50/78] pagerduty: fix entity card tests --- .../src/components/PagerDutyCard.test.tsx | 17 +++++++++++++---- .../pagerduty/src/components/PagerDutyCard.tsx | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/plugins/pagerduty/src/components/PagerDutyCard.test.tsx b/plugins/pagerduty/src/components/PagerDutyCard.test.tsx index a6615aee82..a719a27f86 100644 --- a/plugins/pagerduty/src/components/PagerDutyCard.test.tsx +++ b/plugins/pagerduty/src/components/PagerDutyCard.test.tsx @@ -17,6 +17,7 @@ import React from 'react'; import { render, waitFor, fireEvent, act } from '@testing-library/react'; import { PagerDutyCard } from './PagerDutyCard'; import { Entity } from '@backstage/catalog-model'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; import { wrapInTestApp } from '@backstage/test-utils'; import { alertApiRef, @@ -80,7 +81,9 @@ describe('PageDutyCard', () => { const { getByText, queryByTestId } = render( wrapInTestApp( - + + + , ), ); @@ -99,7 +102,9 @@ describe('PageDutyCard', () => { const { getByText, queryByTestId } = render( wrapInTestApp( - + + + , ), ); @@ -114,7 +119,9 @@ describe('PageDutyCard', () => { const { getByText, queryByTestId } = render( wrapInTestApp( - + + + , ), ); @@ -134,7 +141,9 @@ describe('PageDutyCard', () => { const { getByText, queryByTestId, getByTestId, getByRole } = render( wrapInTestApp( - + + + , ), ); diff --git a/plugins/pagerduty/src/components/PagerDutyCard.tsx b/plugins/pagerduty/src/components/PagerDutyCard.tsx index f7360c01d6..e03fb0fdbe 100644 --- a/plugins/pagerduty/src/components/PagerDutyCard.tsx +++ b/plugins/pagerduty/src/components/PagerDutyCard.tsx @@ -58,7 +58,7 @@ export const isPluginApplicableToEntity = (entity: Entity) => type Props = { /** @deprecated The entity is now grabbed from context instead */ - entity: Entity; + entity?: Entity; }; export const PagerDutyCard = (_props: Props) => { From f9dacb504e1357bee73a05f5f2a810705c685534 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Feb 2021 04:38:07 +0000 Subject: [PATCH 51/78] chore(deps-dev): bump husky from 4.3.6 to 4.3.8 Bumps [husky](https://github.com/typicode/husky) from 4.3.6 to 4.3.8. - [Release notes](https://github.com/typicode/husky/releases) - [Commits](https://github.com/typicode/husky/compare/v4.3.6...v4.3.8) Signed-off-by: dependabot[bot] --- yarn.lock | 61 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/yarn.lock b/yarn.lock index 45f8a8b4fb..d5e90fa45a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2645,7 +2645,31 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/plugin-catalog@^0.2.0", "@backstage/plugin-catalog@^0.2.1": +"@backstage/plugin-catalog@^0.2.0": + version "0.3.0" + dependencies: + "@backstage/catalog-client" "^0.3.5" + "@backstage/catalog-model" "^0.7.1" + "@backstage/core" "^0.6.0" + "@backstage/plugin-catalog-react" "^0.0.2" + "@backstage/plugin-scaffolder" "^0.4.2" + "@backstage/theme" "^0.2.3" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@types/react" "^16.9" + classnames "^2.2.6" + git-url-parse "^11.4.4" + moment "^2.26.0" + react "^16.13.1" + react-dom "^16.13.1" + react-helmet "6.1.0" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-use "^15.3.3" + swr "^0.3.0" + +"@backstage/plugin-catalog@^0.2.1": version "0.3.0" dependencies: "@backstage/catalog-client" "^0.3.5" @@ -13795,12 +13819,12 @@ find-up@^5.0.0: locate-path "^6.0.0" path-exists "^4.0.0" -find-versions@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/find-versions/-/find-versions-3.2.0.tgz#10297f98030a786829681690545ef659ed1d254e" - integrity sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww== +find-versions@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/find-versions/-/find-versions-4.0.0.tgz#3c57e573bf97769b8cb8df16934b627915da4965" + integrity sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ== dependencies: - semver-regex "^2.0.0" + semver-regex "^3.1.2" find-yarn-workspace-root2@1.2.16: version "1.2.16" @@ -15406,17 +15430,17 @@ humanize-ms@^1.2.1: ms "^2.0.0" husky@^4.2.3: - version "4.3.6" - resolved "https://registry.npmjs.org/husky/-/husky-4.3.6.tgz#ebd9dd8b9324aa851f1587318db4cccb7665a13c" - integrity sha512-o6UjVI8xtlWRL5395iWq9LKDyp/9TE7XMOTvIpEVzW638UcGxTmV5cfel6fsk/jbZSTlvfGVJf2svFtybcIZag== + version "4.3.8" + resolved "https://registry.npmjs.org/husky/-/husky-4.3.8.tgz#31144060be963fd6850e5cc8f019a1dfe194296d" + integrity sha512-LCqqsB0PzJQ/AlCgfrfzRe3e3+NvmefAdKQhRYpxS4u6clblBoDdzzvHi8fmxKRzvMxPY/1WZWzomPZww0Anow== dependencies: chalk "^4.0.0" ci-info "^2.0.0" compare-versions "^3.6.0" cosmiconfig "^7.0.0" - find-versions "^3.2.0" + find-versions "^4.0.0" opencollective-postinstall "^2.0.2" - pkg-dir "^4.2.0" + pkg-dir "^5.0.0" please-upgrade-node "^3.2.0" slash "^3.0.0" which-pm-runs "^1.0.0" @@ -20738,6 +20762,13 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +pkg-dir@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" + integrity sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA== + dependencies: + find-up "^5.0.0" + pkg-up@3.1.0, pkg-up@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" @@ -23359,10 +23390,10 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -semver-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/semver-regex/-/semver-regex-2.0.0.tgz#a93c2c5844539a770233379107b38c7b4ac9d338" - integrity sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw== +semver-regex@^3.1.2: + version "3.1.2" + resolved "https://registry.npmjs.org/semver-regex/-/semver-regex-3.1.2.tgz#34b4c0d361eef262e07199dbef316d0f2ab11807" + integrity sha512-bXWyL6EAKOJa81XG1OZ/Yyuq+oT0b2YLlxx7c+mrdYPaPbnj6WgVULXhinMIeZGufuUBu/eVRqXEhiv4imfwxA== "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.6.0, semver@^5.7.0, semver@^5.7.1: version "5.7.1" From ee65c94555e786f83209355d66d451a85ec30709 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 5 Feb 2021 12:25:38 +0100 Subject: [PATCH 52/78] backend-common: ignore rate limiting errors in UrlReader integration tests --- packages/backend-common/src/reading/integration.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/reading/integration.test.ts b/packages/backend-common/src/reading/integration.test.ts index 18f87636a5..16fdf3ff58 100644 --- a/packages/backend-common/src/reading/integration.test.ts +++ b/packages/backend-common/src/reading/integration.test.ts @@ -71,7 +71,11 @@ function withRetries(count: number, fn: () => Promise) { error = err; } } - throw error; + if (!error.message.match(/rate limit|Too Many Requests/)) { + throw error; + } else { + console.warn('Request was rate limited', error); + } }; } From ba3a6a9be0128d1b952263187b0d5d3a59e0e12a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 5 Feb 2021 13:52:20 +0100 Subject: [PATCH 53/78] Scaffolder: fix typos and log output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw --- .../src/scaffolder/tasks/TemplateConverter.ts | 4 ++-- plugins/scaffolder-backend/src/service/router.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index 86f722e475..69788238cc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -99,7 +99,7 @@ export class TemplateActionRegistry { register(action: TemplateAction) { if (this.actions.has(action.id)) { throw new ConflictError( - `Template action with id ${action.id} as already been registered`, + `Template action with ID '${action.id}' has already been registered`, ); } this.actions.set(action.id, action); @@ -109,7 +109,7 @@ export class TemplateActionRegistry { const action = this.actions.get(actionId); if (!action) { throw new NotFoundError( - `Template action with id ${actionId} is not registered.`, + `Template action with ID '${actionId}' is not registered.`, ); } return action; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a8a61972af..5821edfe8c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -258,7 +258,7 @@ export async function createRouter( 'Content-Type': 'text/event-stream', }); - // After client opens connection send all nests as string + // After client opens connection send all events as string const unsubscribe = taskBroker.observe( { taskId, after }, (error, { events }) => { From 0e640893dfeb944a84c4c7ec993e1af5f3696253 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 5 Feb 2021 13:56:53 +0100 Subject: [PATCH 54/78] test-utils: add mountedRoutes option to wrapInTestApp --- .../src/testUtils/appWrappers.test.tsx | 27 +++++++++++++++ .../test-utils/src/testUtils/appWrappers.tsx | 34 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/packages/test-utils/src/testUtils/appWrappers.test.tsx b/packages/test-utils/src/testUtils/appWrappers.test.tsx index 29f8d97793..4899f07061 100644 --- a/packages/test-utils/src/testUtils/appWrappers.test.tsx +++ b/packages/test-utils/src/testUtils/appWrappers.test.tsx @@ -21,9 +21,11 @@ import { Route, Routes } from 'react-router'; import { withLogCollector } from '@backstage/test-utils-core'; import { useApi, + useRouteRef, errorApiRef, ApiProvider, ApiRegistry, + createRouteRef, } from '@backstage/core-api'; import { MockErrorApi } from './apis'; @@ -113,4 +115,29 @@ describe('wrapInTestApp', () => { expect(rendered.getByText('foo')).toBeInTheDocument(); expect(mockErrorApi.getErrors()).toEqual([{ error: new Error('NOPE') }]); }); + + it('should allow route refs to be mounted on specific paths', async () => { + const aRouteRef = createRouteRef({ title: 'A' }); + const bRouteRef = createRouteRef({ title: 'B', params: ['name'] }); + + const MyComponent = () => { + const a = useRouteRef(aRouteRef); + const b = useRouteRef(bRouteRef); + return ( +
+
Link A: {a()}
+
Link B: {b({ name: 'x' })}
+
+ ); + }; + + const rendered = await renderInTestApp(, { + mountedRoutes: { + '/my-a-path': aRouteRef, + '/my-b-path/:name': bRouteRef, + }, + }); + expect(rendered.getByText('Link A: /my-a-path')).toBeInTheDocument(); + expect(rendered.getByText('Link B: /my-b-path/x')).toBeInTheDocument(); + }); }); diff --git a/packages/test-utils/src/testUtils/appWrappers.tsx b/packages/test-utils/src/testUtils/appWrappers.tsx index 1f6c7622b8..cc67a5b707 100644 --- a/packages/test-utils/src/testUtils/appWrappers.tsx +++ b/packages/test-utils/src/testUtils/appWrappers.tsx @@ -21,6 +21,9 @@ import { lightTheme } from '@backstage/theme'; import privateExports, { defaultSystemIcons, BootErrorPageProps, + RouteRef, + createPlugin, + createRoutableExtension, } from '@backstage/core-api'; import { RenderResult } from '@testing-library/react'; import { renderWithEffects } from '@backstage/test-utils-core'; @@ -44,6 +47,22 @@ type TestAppOptions = { * Initial route entries to pass along as `initialEntries` to the router. */ routeEntries?: string[]; + + /** + * An object of paths to mount route ref on, with the key being the path and the value + * being the RouteRef that the path will be bound to. This allows the route refs to be + * used by `useRouteRef` in the rendered elements. + * + * @example + * wrapInTestApp(, { + * mountedRoutes: { + * '/my-path': myRouteRef, + * } + * }) + * // ... + * const link = useRouteRef(myRouteRef) + */ + mountedRoutes?: { [path: string]: RouteRef }; }; /** @@ -90,12 +109,27 @@ export function wrapInTestApp( Wrapper = () => Component as React.ReactElement; } + const routePlugin = createPlugin({ id: 'mock-route-plugin' }); + const routeElements = Object.entries(options.mountedRoutes ?? {}).map( + ([path, routeRef]) => { + const PageComponent = () =>
Mounted at {path}
; + const Page = routePlugin.provide( + createRoutableExtension({ + component: async () => PageComponent, + mountPoint: routeRef, + }), + ); + return } />; + }, + ); + const AppProvider = app.getProvider(); const AppRouter = app.getRouter(); return ( + {routeElements} {/* The path of * here is needed to be set as a catch all, so it will render the wrapper element * and work with nested routes if they exist too */} } /> From 95cd312d55659c34b3b4f5ea39903b8db001fafb Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 5 Feb 2021 14:10:41 +0100 Subject: [PATCH 55/78] scaffolder: Make updateCount errors ConflictError --- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 22697f5fdb..373a4486f1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -141,7 +141,7 @@ export class DatabaseTaskStore implements TaskStore { last_heartbeat_at: this.db.fn.now(), }); if (updateCount === 0) { - throw new Error(`No running task with taskId ${taskId} found`); + throw new ConflictError(`No running task with taskId ${taskId} found`); } } @@ -210,7 +210,7 @@ export class DatabaseTaskStore implements TaskStore { status, }); if (updateCount !== 1) { - throw new Error( + throw new ConflictError( `Failed to update status to '${status}' for taskId ${taskId}`, ); } From b3b766b79b2efd7f9e0201627c5e7fe10ffb1daa Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 5 Feb 2021 14:34:00 +0100 Subject: [PATCH 56/78] Scaffolder: Remove sleep --- plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index f87d2df934..c62495ca4c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -101,9 +101,6 @@ export class TaskWorker { task.emitLog(`Finished step ${step.name}`); } - logger.info('So done right now'); - await new Promise(resolve => setTimeout(resolve, 5000)); - await task.complete('completed'); } catch (error) { task.emitLog(String(error.stack)); From 9afa2d24c63bad5e9b1f789c06bb5a6ae3f5bcfc Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 5 Feb 2021 15:25:34 +0100 Subject: [PATCH 57/78] scaffolder: Fix legacy publish error message --- plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index 6daf1ba4f6..f151780ba9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -87,10 +87,9 @@ export function registerLegacyActions( } const owner = values.owner as unknown; if (typeof owner !== 'string') { - throw new Error( - `Invalid store path passed to publish, got ${typeof owner}`, - ); + throw new Error(`Invalid owner passed to publish, got ${typeof owner}`); } + const publisher = publishers.get(storePath); ctx.logger.info('Will now store the template'); const { remoteUrl, catalogInfoUrl } = await publisher.publish({ From e84170d6443d907806d86c2966aa17eaf2110b62 Mon Sep 17 00:00:00 2001 From: Gowind Date: Fri, 5 Feb 2021 15:37:44 +0100 Subject: [PATCH 58/78] use child logger to log single location update --- .../src/ingestion/HigherOrderOperations.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index ba97232810..467c0eb2e8 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -130,7 +130,7 @@ export class HigherOrderOperations implements HigherOrderOperation { `Locations Refresh: Refreshing location ${location.type}:${location.target}`, ); try { - await this.refreshSingleLocation(location); + await this.refreshSingleLocation(location, logger); await this.locationsCatalog.logUpdateSuccess(location.id, undefined); } catch (e) { logger.warn( @@ -148,8 +148,12 @@ export class HigherOrderOperations implements HigherOrderOperation { } // Performs a full refresh of a single location - private async refreshSingleLocation(location: Location) { + private async refreshSingleLocation( + location: Location, + optionalLogger?: Logger, + ) { let startTimestamp = process.hrtime(); + const logger = optionalLogger || this.logger; const readerOutput = await this.locationReader.read({ type: location.type, @@ -157,12 +161,12 @@ export class HigherOrderOperations implements HigherOrderOperation { }); for (const item of readerOutput.errors) { - this.logger.warn( + logger.warn( `Failed item in location ${item.location.type}:${item.location.target}, ${item.error.stack}`, ); } - this.logger.info( + logger.info( `Read ${readerOutput.entities.length} entities from location ${ location.type }:${location.target} in ${durationText(startTimestamp)}`, @@ -186,14 +190,14 @@ export class HigherOrderOperations implements HigherOrderOperation { throw e; } - this.logger.debug(`Posting update success markers`); + logger.debug(`Posting update success markers`); await this.locationsCatalog.logUpdateSuccess( location.id, readerOutput.entities.map(e => e.entity.metadata.name), ); - this.logger.info( + logger.info( `Wrote ${readerOutput.entities.length} entities from location ${ location.type }:${location.target} in ${durationText(startTimestamp)}`, From 8f3443427287abf6408d686739bc2fc84043fe27 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 5 Feb 2021 10:49:12 -0500 Subject: [PATCH 59/78] Rename green-rabbits-burn.md to techdocs-green-rabbits-burn.md --- .../{green-rabbits-burn.md => techdocs-green-rabbits-burn.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changeset/{green-rabbits-burn.md => techdocs-green-rabbits-burn.md} (100%) diff --git a/.changeset/green-rabbits-burn.md b/.changeset/techdocs-green-rabbits-burn.md similarity index 100% rename from .changeset/green-rabbits-burn.md rename to .changeset/techdocs-green-rabbits-burn.md From fd933c4239e4099034f6c0bafd6a124f5d78f85b Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 5 Feb 2021 10:50:11 -0500 Subject: [PATCH 60/78] Add comment TODO --- plugins/techdocs/src/reader/components/Reader.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index c9e76ddeeb..f1937ae8a8 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -155,6 +155,8 @@ export const Reader = ({ entityId, onReady }: Props) => { ]); if (error) { + // TODO Enhance API call to return customize error objects so we can identify which we ran into + // For now this defaults to display error code 404 return ; } From 2ab3b3f0e3614de2faed69f23387b2969de4d510 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 5 Feb 2021 17:49:41 +0100 Subject: [PATCH 61/78] core-api: fix type inference of createRouteRef --- packages/core-api/src/routing/RouteRef.ts | 23 ++++++++++++++++---- packages/core-api/src/routing/hooks.test.tsx | 3 ++- packages/core-api/src/routing/index.ts | 2 +- packages/core-api/src/routing/types.ts | 8 ------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/core-api/src/routing/RouteRef.ts b/packages/core-api/src/routing/RouteRef.ts index 4c2da34098..3f48d70827 100644 --- a/packages/core-api/src/routing/RouteRef.ts +++ b/packages/core-api/src/routing/RouteRef.ts @@ -14,7 +14,16 @@ * limitations under the License. */ -import { RouteRefConfig, RouteRef } from './types'; +import { RouteRef } from './types'; +import { IconComponent } from '../icons'; + +export type RouteRefConfig = { + params?: Array; + /** @deprecated Route refs no longer decide their own path */ + path?: string; + icon?: IconComponent; + title: string; +}; export class AbsoluteRouteRef { constructor(private readonly config: RouteRefConfig) {} @@ -38,9 +47,15 @@ export class AbsoluteRouteRef { } export function createRouteRef< - ParamKeys extends string, - Params extends { [param in string]: string } = { [name in ParamKeys]: string } ->(config: RouteRefConfig): RouteRef { + Params extends { [param in ParamKey]: string }, + ParamKey extends string = never +>(config: { + params?: ParamKey[]; + /** @deprecated Route refs no longer decide their own path */ + path?: string; + icon?: IconComponent; + title: string; +}): RouteRef { return new AbsoluteRouteRef(config); } diff --git a/packages/core-api/src/routing/hooks.test.tsx b/packages/core-api/src/routing/hooks.test.tsx index 3dbeaf8e3e..84a8314807 100644 --- a/packages/core-api/src/routing/hooks.test.tsx +++ b/packages/core-api/src/routing/hooks.test.tsx @@ -39,8 +39,9 @@ import { createRouteRef, createExternalRouteRef, ExternalRouteRef, + RouteRefConfig, } from './RouteRef'; -import { RouteRef, RouteRefConfig } from './types'; +import { RouteRef } from './types'; const mockConfig = (extra?: Partial>) => ({ path: '/unused', diff --git a/packages/core-api/src/routing/index.ts b/packages/core-api/src/routing/index.ts index 9564b3b225..af71e0c3e9 100644 --- a/packages/core-api/src/routing/index.ts +++ b/packages/core-api/src/routing/index.ts @@ -16,11 +16,11 @@ export type { RouteRef, - RouteRefConfig, AbsoluteRouteRef, ConcreteRoute, MutableRouteRef, } from './types'; export { FlatRoutes } from './FlatRoutes'; export { createRouteRef } from './RouteRef'; +export type { RouteRefConfig } from './RouteRef'; export { useRouteRef } from './hooks'; diff --git a/packages/core-api/src/routing/types.ts b/packages/core-api/src/routing/types.ts index 99a20c31e9..5f9376c1d2 100644 --- a/packages/core-api/src/routing/types.ts +++ b/packages/core-api/src/routing/types.ts @@ -45,14 +45,6 @@ export type AbsoluteRouteRef = RouteRef<{}>; */ export type MutableRouteRef = RouteRef<{}>; -export type RouteRefConfig = { - params?: Array; - /** @deprecated Route refs no longer decide their own path */ - path?: string; - icon?: IconComponent; - title: string; -}; - // A duplicate of the react-router RouteObject, but with routeRef added export interface BackstageRouteObject { caseSensitive: boolean; From b51ee6ece987552331c8673ba7452ab4bcd95575 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 5 Feb 2021 17:56:01 +0100 Subject: [PATCH 62/78] added changesets --- .changeset/poor-sheep-give.md | 5 +++++ .changeset/tricky-dancers-rush.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/poor-sheep-give.md create mode 100644 .changeset/tricky-dancers-rush.md diff --git a/.changeset/poor-sheep-give.md b/.changeset/poor-sheep-give.md new file mode 100644 index 0000000000..4c52594b35 --- /dev/null +++ b/.changeset/poor-sheep-give.md @@ -0,0 +1,5 @@ +--- +'@backstage/test-utils': patch +--- + +Added `mountedRoutes` option to `wrapInTestApp`, allowing routes to be associated to concrete paths to make `useRouteRef` usable in tested components. diff --git a/.changeset/tricky-dancers-rush.md b/.changeset/tricky-dancers-rush.md new file mode 100644 index 0000000000..301b23d98e --- /dev/null +++ b/.changeset/tricky-dancers-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/core': patch +--- + +Fixed type inference of `createRouteRef`. From fb53eb7cb4057a33ae76f09fa1f009334adf1f51 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Fri, 5 Feb 2021 19:41:39 +0100 Subject: [PATCH 63/78] Don't respond to a request twice if an entity has not been found. --- .changeset/fluffy-hats-hug.md | 5 +++++ plugins/catalog-backend/src/service/router.ts | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .changeset/fluffy-hats-hug.md diff --git a/.changeset/fluffy-hats-hug.md b/.changeset/fluffy-hats-hug.md new file mode 100644 index 0000000000..b3d06337fd --- /dev/null +++ b/.changeset/fluffy-hats-hug.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Don't respond to a request twice if an entity has not been found. diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 43c2967233..6064164e31 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -76,8 +76,9 @@ export async function createRouter( ); if (!entities.length) { res.status(404).send(`No entity with uid ${uid}`); + } else { + res.status(200).send(entities[0]); } - res.status(200).send(entities[0]); }) .delete('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; @@ -99,8 +100,9 @@ export async function createRouter( .send( `No entity with kind ${kind} namespace ${namespace} name ${name}`, ); + } else { + res.status(200).send(entities[0]); } - res.status(200).send(entities[0]); }); } From 7cdd586e579643f1213096323d11c8d4e4fe7465 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 6 Feb 2021 01:19:20 +0100 Subject: [PATCH 64/78] create-app: update changelog to put more emphasis on plugin-catalog-react migration --- packages/create-app/CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 01432ba150..5e8ff0e9b1 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -4,9 +4,9 @@ ### Patch Changes -- 019fe39a0: `@backstage/plugin-catalog` stopped exporting hooks and helpers for other - plugins. They are migrated to `@backstage/plugin-catalog-react`. - Change both your dependencies and imports to the new package. +- 019fe39a0: **BREAKING CHANGE**: The `useEntity` hooks has been moved from `@backstage/plugin-catalog` to `@backstage/plugin-catalog-react`. + To apply this change to an existing app, add `@backstage/plugin-catalog-react` to your dependencies in `packages/app/package.json`, and update + the import inside `packages/app/src/components/catalog/EntityPage.tsx` as well as any other places you were using `useEntity` or any other functions that were moved to `@backstage/plugin-catalog-react`. - 436ca3f62: Remove techdocs.requestUrl and techdocs.storageUrl from app-config.yaml - Updated dependencies [ceef4dd89] - Updated dependencies [720149854] From bee68316675ad51de20a750a980a2cc488a80bf0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 6 Feb 2021 10:13:24 +0100 Subject: [PATCH 65/78] Update packages/create-app/CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw --- packages/create-app/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 5e8ff0e9b1..13b854972a 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -4,7 +4,7 @@ ### Patch Changes -- 019fe39a0: **BREAKING CHANGE**: The `useEntity` hooks has been moved from `@backstage/plugin-catalog` to `@backstage/plugin-catalog-react`. +- 019fe39a0: **BREAKING CHANGE**: The `useEntity` hook has been moved from `@backstage/plugin-catalog` to `@backstage/plugin-catalog-react`. To apply this change to an existing app, add `@backstage/plugin-catalog-react` to your dependencies in `packages/app/package.json`, and update the import inside `packages/app/src/components/catalog/EntityPage.tsx` as well as any other places you were using `useEntity` or any other functions that were moved to `@backstage/plugin-catalog-react`. - 436ca3f62: Remove techdocs.requestUrl and techdocs.storageUrl from app-config.yaml From 43397061fdcc861d0b537c51d87171be569d5376 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 6 Feb 2021 17:26:44 +0100 Subject: [PATCH 66/78] docs(TechDocs): docs/ is configurable by mkdocs.yml --- docs/features/techdocs/creating-and-publishing.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index c15dc1fe69..b9340330a8 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -80,6 +80,10 @@ Create a `/docs` folder in the root of the project with at least an `index.md` file. _(If you add more markdown files, make sure to update the nav in the mkdocs.yml file to get a proper navigation for your documentation.)_ +> Note - Although `docs` is a popular directory name for storing documentation, +> it can be renamed to something else and can be configured by `mkdocs.yml`. See +> https://www.mkdocs.org/user-guide/configuration/#docs_dir + The `docs/index.md` can for example have the following content: ```md From 1585732a7bf9e67a0506c378d08c7698706f2c2d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 7 Feb 2021 20:10:54 +0100 Subject: [PATCH 67/78] core-api: document createRouteRef type inference logic --- packages/core-api/src/routing/RouteRef.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/core-api/src/routing/RouteRef.ts b/packages/core-api/src/routing/RouteRef.ts index 3f48d70827..0a3df49b58 100644 --- a/packages/core-api/src/routing/RouteRef.ts +++ b/packages/core-api/src/routing/RouteRef.ts @@ -47,7 +47,12 @@ export class AbsoluteRouteRef { } export function createRouteRef< + // Params is the type that we care about and the one to be embedded in the route ref. + // For example, given the params ['name', 'kind'], Params will be {name: string, kind: string} Params extends { [param in ParamKey]: string }, + // ParamKey is here to make sure the Params type properly has its keys narrowed down + // to only the elements of params. Defaulting to never makes sure we end up with + // Param = {} if the params array is empty. ParamKey extends string = never >(config: { params?: ParamKey[]; From 3eb1b1baccfc0328b796808dacbf9ea6e1223732 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 7 Feb 2021 23:40:44 +0100 Subject: [PATCH 68/78] github-actions: GitHub -> Github --- .changeset/selfish-years-pump.md | 2 +- plugins/github-actions/src/index.ts | 10 +++++----- plugins/github-actions/src/plugin.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.changeset/selfish-years-pump.md b/.changeset/selfish-years-pump.md index 48994a1d9e..e5cc5f3bc3 100644 --- a/.changeset/selfish-years-pump.md +++ b/.changeset/selfish-years-pump.md @@ -2,4 +2,4 @@ '@backstage/plugin-github-actions': patch --- -Migrate to new composability API, exporting the plugin instance as `githubActionsPlugin`, the entity content as `EntityGitHubActionsContent`, entity conditional as `isGitHubActionsAvailable`, and entity cards as `EntityLatestGitHubActionRunCard`, `EntityLatestGitHubActionsForBranchCard`, and `EntityRecentGitHubActionsRunsCard`. +Migrate to new composability API, exporting the plugin instance as `githubActionsPlugin`, the entity content as `EntityGithubActionsContent`, entity conditional as `isGithubActionsAvailable`, and entity cards as `EntityLatestGithubActionRunCard`, `EntityLatestGithubActionsForBranchCard`, and `EntityRecentGithubActionsRunsCard`. diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 6dd3b9724e..6dd4807b75 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -17,16 +17,16 @@ export { githubActionsPlugin, githubActionsPlugin as plugin, - EntityGitHubActionsContent, - EntityLatestGitHubActionRunCard, - EntityLatestGitHubActionsForBranchCard, - EntityRecentGitHubActionsRunsCard, + EntityGithubActionsContent, + EntityLatestGithubActionRunCard, + EntityLatestGithubActionsForBranchCard, + EntityRecentGithubActionsRunsCard, } from './plugin'; export * from './api'; export { Router, isPluginApplicableToEntity, - isPluginApplicableToEntity as isGitHubActionsAvailable, + isPluginApplicableToEntity as isGithubActionsAvailable, } from './components/Router'; export * from './components/Cards'; export { GITHUB_ACTIONS_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/github-actions/src/plugin.ts b/plugins/github-actions/src/plugin.ts index 71c05745d4..d296b4ad30 100644 --- a/plugins/github-actions/src/plugin.ts +++ b/plugins/github-actions/src/plugin.ts @@ -51,14 +51,14 @@ export const githubActionsPlugin = createPlugin({ }, }); -export const EntityGitHubActionsContent = githubActionsPlugin.provide( +export const EntityGithubActionsContent = githubActionsPlugin.provide( createRoutableExtension({ component: () => import('./components/Router').then(m => m.Router), mountPoint: rootRouteRef, }), ); -export const EntityLatestGitHubActionRunCard = githubActionsPlugin.provide( +export const EntityLatestGithubActionRunCard = githubActionsPlugin.provide( createComponentExtension({ component: { lazy: () => @@ -67,7 +67,7 @@ export const EntityLatestGitHubActionRunCard = githubActionsPlugin.provide( }), ); -export const EntityLatestGitHubActionsForBranchCard = githubActionsPlugin.provide( +export const EntityLatestGithubActionsForBranchCard = githubActionsPlugin.provide( createComponentExtension({ component: { lazy: () => @@ -76,7 +76,7 @@ export const EntityLatestGitHubActionsForBranchCard = githubActionsPlugin.provid }), ); -export const EntityRecentGitHubActionsRunsCard = githubActionsPlugin.provide( +export const EntityRecentGithubActionsRunsCard = githubActionsPlugin.provide( createComponentExtension({ component: { lazy: () => From 8e1789e0c312f93ed2fec60fa31f2c961216fec4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 7 Feb 2021 23:48:31 +0100 Subject: [PATCH 69/78] circleci,cloudbuild,github-actions,jenkins: rename internal entity conditional --- plugins/circleci/src/components/Router.tsx | 4 ++-- plugins/circleci/src/index.ts | 4 ++-- plugins/cloudbuild/src/components/Router.tsx | 4 ++-- plugins/cloudbuild/src/index.ts | 4 ++-- plugins/github-actions/src/components/Router.tsx | 4 ++-- plugins/github-actions/src/index.ts | 4 ++-- plugins/jenkins/src/components/Router.tsx | 4 ++-- plugins/jenkins/src/index.ts | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/plugins/circleci/src/components/Router.tsx b/plugins/circleci/src/components/Router.tsx index b3a0d2c3e2..d18b6c1e61 100644 --- a/plugins/circleci/src/components/Router.tsx +++ b/plugins/circleci/src/components/Router.tsx @@ -24,7 +24,7 @@ import { Entity } from '@backstage/catalog-model'; import { useEntity } from '@backstage/plugin-catalog-react'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isCircleCIAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[CIRCLECI_ANNOTATION]); type Props = { @@ -35,7 +35,7 @@ type Props = { export const Router = (_props: Props) => { const { entity } = useEntity(); - if (!isPluginApplicableToEntity(entity)) { + if (!isCircleCIAvailable(entity)) { return ; } diff --git a/plugins/circleci/src/index.ts b/plugins/circleci/src/index.ts index 1e8b213279..2a37cbdee0 100644 --- a/plugins/circleci/src/index.ts +++ b/plugins/circleci/src/index.ts @@ -23,7 +23,7 @@ export * from './api'; export * from './route-refs'; export { Router, - isPluginApplicableToEntity, - isPluginApplicableToEntity as isCircleCIAvailable, + isCircleCIAvailable, + isCircleCIAvailable as isPluginApplicableToEntity, } from './components/Router'; export { CIRCLECI_ANNOTATION } from './constants'; diff --git a/plugins/cloudbuild/src/components/Router.tsx b/plugins/cloudbuild/src/components/Router.tsx index c31947a531..0f71efcf86 100644 --- a/plugins/cloudbuild/src/components/Router.tsx +++ b/plugins/cloudbuild/src/components/Router.tsx @@ -23,7 +23,7 @@ import { WorkflowRunsTable } from './WorkflowRunsTable'; import { CLOUDBUILD_ANNOTATION } from './useProjectName'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isCloudbuildAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[CLOUDBUILD_ANNOTATION]); type Props = { @@ -34,7 +34,7 @@ type Props = { export const Router = (_props: Props) => { const { entity } = useEntity(); - if (!isPluginApplicableToEntity(entity)) { + if (!isCloudbuildAvailable(entity)) { // TODO(shmidt-i): move warning to a separate standardized component return ; } diff --git a/plugins/cloudbuild/src/index.ts b/plugins/cloudbuild/src/index.ts index a846792fa2..b7cd6c29d5 100644 --- a/plugins/cloudbuild/src/index.ts +++ b/plugins/cloudbuild/src/index.ts @@ -23,8 +23,8 @@ export { export * from './api'; export { Router, - isPluginApplicableToEntity, - isPluginApplicableToEntity as isCloudbuildAvailable, + isCloudbuildAvailable, + isCloudbuildAvailable as isPluginApplicableToEntity, } from './components/Router'; export * from './components/Cards'; export { CLOUDBUILD_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/github-actions/src/components/Router.tsx b/plugins/github-actions/src/components/Router.tsx index e867b05b18..834fcae43c 100644 --- a/plugins/github-actions/src/components/Router.tsx +++ b/plugins/github-actions/src/components/Router.tsx @@ -23,7 +23,7 @@ import { WorkflowRunsTable } from './WorkflowRunsTable'; import { GITHUB_ACTIONS_ANNOTATION } from './useProjectName'; import { MissingAnnotationEmptyState } from '@backstage/core'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isGithubActionsAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[GITHUB_ACTIONS_ANNOTATION]); type Props = { @@ -34,7 +34,7 @@ type Props = { export const Router = (_props: Props) => { const { entity } = useEntity(); - if (!isPluginApplicableToEntity(entity)) { + if (!isGithubActionsAvailable(entity)) { return ( ); diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 6dd4807b75..b4179a9d92 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -25,8 +25,8 @@ export { export * from './api'; export { Router, - isPluginApplicableToEntity, - isPluginApplicableToEntity as isGithubActionsAvailable, + isGithubActionsAvailable, + isGithubActionsAvailable as isPluginApplicableToEntity, } from './components/Router'; export * from './components/Cards'; export { GITHUB_ACTIONS_ANNOTATION } from './components/useProjectName'; diff --git a/plugins/jenkins/src/components/Router.tsx b/plugins/jenkins/src/components/Router.tsx index e310cd4193..85f06bb0e1 100644 --- a/plugins/jenkins/src/components/Router.tsx +++ b/plugins/jenkins/src/components/Router.tsx @@ -23,7 +23,7 @@ import { Entity } from '@backstage/catalog-model'; import { MissingAnnotationEmptyState } from '@backstage/core'; import { CITable } from './BuildsPage/lib/CITable'; -export const isPluginApplicableToEntity = (entity: Entity) => +export const isJenkinsAvailable = (entity: Entity) => Boolean(entity.metadata.annotations?.[JENKINS_ANNOTATION]); type Props = { @@ -34,7 +34,7 @@ type Props = { export const Router = (_props: Props) => { const { entity } = useEntity(); - if (!isPluginApplicableToEntity(entity)) { + if (!isJenkinsAvailable(entity)) { return ; } diff --git a/plugins/jenkins/src/index.ts b/plugins/jenkins/src/index.ts index 6d0e7837d3..e434e959e8 100644 --- a/plugins/jenkins/src/index.ts +++ b/plugins/jenkins/src/index.ts @@ -23,8 +23,8 @@ export { export { LatestRunCard } from './components/Cards'; export { Router, - isPluginApplicableToEntity, - isPluginApplicableToEntity as isJenkinsAvailable, + isJenkinsAvailable, + isJenkinsAvailable as isPluginApplicableToEntity, } from './components/Router'; export { JENKINS_ANNOTATION } from './constants'; export * from './api'; From 8761cdb3ff8c62a15bca0f30acb1bf594ef6f097 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Feb 2021 04:56:58 +0000 Subject: [PATCH 70/78] chore(deps): bump rollup-plugin-typescript2 from 0.27.3 to 0.29.0 Bumps [rollup-plugin-typescript2](https://github.com/ezolenko/rollup-plugin-typescript2) from 0.27.3 to 0.29.0. - [Release notes](https://github.com/ezolenko/rollup-plugin-typescript2/releases) - [Commits](https://github.com/ezolenko/rollup-plugin-typescript2/compare/0.27.3...0.29.0) Signed-off-by: dependabot[bot] --- packages/cli/package.json | 2 +- yarn.lock | 34 +++++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index fb33a154a7..9aaeb00459 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -93,7 +93,7 @@ "rollup-plugin-esbuild": "2.6.x", "rollup-plugin-peer-deps-external": "^2.2.2", "rollup-plugin-postcss": "^3.1.1", - "rollup-plugin-typescript2": "^0.27.3", + "rollup-plugin-typescript2": "^0.29.0", "rollup-pluginutils": "^2.8.2", "semver": "^7.3.2", "start-server-webpack-plugin": "^2.2.5", diff --git a/yarn.lock b/yarn.lock index 45f8a8b4fb..c8858afb59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2645,7 +2645,31 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/plugin-catalog@^0.2.0", "@backstage/plugin-catalog@^0.2.1": +"@backstage/plugin-catalog@^0.2.0": + version "0.3.0" + dependencies: + "@backstage/catalog-client" "^0.3.5" + "@backstage/catalog-model" "^0.7.1" + "@backstage/core" "^0.6.0" + "@backstage/plugin-catalog-react" "^0.0.2" + "@backstage/plugin-scaffolder" "^0.4.2" + "@backstage/theme" "^0.2.3" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@types/react" "^16.9" + classnames "^2.2.6" + git-url-parse "^11.4.4" + moment "^2.26.0" + react "^16.13.1" + react-dom "^16.13.1" + react-helmet "6.1.0" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-use "^15.3.3" + swr "^0.3.0" + +"@backstage/plugin-catalog@^0.2.1": version "0.3.0" dependencies: "@backstage/catalog-client" "^0.3.5" @@ -23147,10 +23171,10 @@ rollup-plugin-postcss@^3.1.1: safe-identifier "^0.4.1" style-inject "^0.3.0" -rollup-plugin-typescript2@^0.27.3: - version "0.27.3" - resolved "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.27.3.tgz#cd9455ac026d325b20c5728d2cc54a08a771b68b" - integrity sha512-gmYPIFmALj9D3Ga1ZbTZAKTXq1JKlTQBtj299DXhqYz9cL3g/AQfUvbb2UhH+Nf++cCq941W2Mv7UcrcgLzJJg== +rollup-plugin-typescript2@^0.29.0: + version "0.29.0" + resolved "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.29.0.tgz#b7ad83f5241dbc5bdf1e98d9c3fca005ffe39e1a" + integrity sha512-YytahBSZCIjn/elFugEGQR5qTsVhxhUwGZIsA9TmrSsC88qroGo65O5HZP/TTArH2dm0vUmYWhKchhwi2wL9bw== dependencies: "@rollup/pluginutils" "^3.1.0" find-cache-dir "^3.3.1" From 44da5d09c581e54f94ad989b9a411e3c35219de5 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 8 Feb 2021 09:24:48 +0100 Subject: [PATCH 71/78] scaffolder: Convert to prepared statements --- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 373a4486f1..5717f3c8c1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -158,8 +158,11 @@ export class DatabaseTaskStore implements TaskStore { 'last_heartbeat_at', '<=', this.db.client.config.client === 'sqlite3' - ? this.db.raw(`datetime('now', '-${Number(timeoutS)} seconds')`) - : this.db.raw(`dateadd('second', -${Number(timeoutS)}, now())`), + ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`]) + : this.db.raw(`dateadd('second', ?, ?)`, [ + `-${timeoutS}`, + this.db.fn.now(), + ]), ); const tasks = rawRows.map(row => ({ taskId: row.id, From 965e200c61670758d6358b4ed58c3687f6f022c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 4 Feb 2021 14:19:17 +0100 Subject: [PATCH 72/78] backend-common: slight refactoring in support of future search impl --- .changeset/old-tools-exist.md | 5 + packages/backend-common/package.json | 1 + .../src/reading/GithubUrlReader.ts | 128 ++++++++++-------- .../src/reading/tree/TarArchiveResponse.ts | 12 +- .../src/reading/tree/ZipArchiveResponse.ts | 8 +- packages/backend-common/src/reading/types.ts | 72 +++++----- .../src/github/GithubCredentialsProvider.ts | 2 +- 7 files changed, 124 insertions(+), 104 deletions(-) create mode 100644 .changeset/old-tools-exist.md diff --git a/.changeset/old-tools-exist.md b/.changeset/old-tools-exist.md new file mode 100644 index 0000000000..a9365cb784 --- /dev/null +++ b/.changeset/old-tools-exist.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Slight refactoring in support of a future search implementation in `UrlReader`. Mostly moving code around. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 3053784dc3..faec36dab4 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -33,6 +33,7 @@ "@backstage/config": "^0.1.2", "@backstage/config-loader": "^0.5.1", "@backstage/integration": "^0.3.2", + "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "archiver": "^5.0.2", diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 4f1aa7c07b..99d641f2dd 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -15,11 +15,12 @@ */ import { - GitHubIntegrationConfig, - readGitHubIntegrationConfigs, getGitHubFileFetchUrl, GithubCredentialsProvider, + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, } from '@backstage/integration'; +import { RestEndpointMethodTypes } from '@octokit/rest'; import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; import { Readable } from 'stream'; @@ -98,74 +99,26 @@ export class GithubUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { ref, filepath, full_name } = parseGitUrl(url); - // Caveat: The ref will totally be incorrect if the branch name includes a / - // Thus, readTree can not work on url containing branch name that has a / - - const { headers } = await this.deps.credentialsProvider.getCredentials({ - url, - }); - - // Get GitHub API urls for the repository - const repoGitHubResponse = await fetch( - new URL(`${this.config.apiBaseUrl}/repos/${full_name}`).toString(), - { - headers, - }, - ); - if (!repoGitHubResponse.ok) { - const message = `Failed to read tree (repository) from ${url}, ${repoGitHubResponse.status} ${repoGitHubResponse.statusText}`; - if (repoGitHubResponse.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); - } - - const repoResponseJson = await repoGitHubResponse.json(); - - // ref is an empty string if no branch is set in provided url to readTree. - // Use GitHub API to get the default branch of the repository. - const branch = ref || repoResponseJson.default_branch; - const branchesApiUrl = repoResponseJson.branches_url; - const archiveApiUrl = repoResponseJson.archive_url; - - // Fetch the latest commit in the provided or default branch to compare against - // the provided sha. - const branchGitHubResponse = await fetch( - // branchesApiUrl looks like "https://api.github.com/repos/owner/repo/branches{/branch}" - branchesApiUrl.replace('{/branch}', `/${branch}`), - { - headers, - }, - ); - if (!branchGitHubResponse.ok) { - const message = `Failed to read tree (branch) from ${url}, ${branchGitHubResponse.status} ${branchGitHubResponse.statusText}`; - if (branchGitHubResponse.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); - } - const commitSha = (await branchGitHubResponse.json()).commit.sha; + const repoDetails = await this.getRepoDetails(url); + const commitSha = repoDetails.branch.commit.sha!; if (options?.etag && options.etag === commitSha) { throw new NotModifiedError(); } - const archive = await fetch( - // archiveApiUrl looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" - archiveApiUrl + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); + + // archive_url looks like "https://api.github.com/repos/owner/repo/{archive_format}{/ref}" + const archive = await this.fetchResponse( + repoDetails.repo.archive_url .replace('{archive_format}', 'tarball') .replace('{/ref}', `/${commitSha}`), { headers }, ); - if (!archive.ok) { - const message = `Failed to read tree (archive) from ${url}, ${archive.status} ${archive.statusText}`; - if (archive.status === 404) { - throw new NotFoundError(message); - } - throw new Error(message); - } + const { filepath } = parseGitUrl(url); return await this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want // to stick to using that in exclusively backend code. @@ -180,4 +133,59 @@ export class GithubUrlReader implements UrlReader { const { host, token } = this.config; return `github{host=${host},authed=${Boolean(token)}}`; } + + private async getRepoDetails( + url: string, + ): Promise<{ + repo: RestEndpointMethodTypes['repos']['get']['response']['data']; + branch: RestEndpointMethodTypes['repos']['getBranch']['response']['data']; + }> { + const parsed = parseGitUrl(url); + const { ref, full_name } = parsed; + + // Caveat: The ref will totally be incorrect if the branch name includes a + // slash. Thus, some operations can not work on URLs containing branch + // names that have a slash in them. + + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); + + const repo: RestEndpointMethodTypes['repos']['get']['response']['data'] = await this.fetchJson( + `${this.config.apiBaseUrl}/repos/${full_name}`, + { headers }, + ); + + // branches_url looks like "https://api.github.com/repos/owner/repo/branches{/branch}" + const branch: RestEndpointMethodTypes['repos']['getBranch']['response']['data'] = await this.fetchJson( + repo.branches_url.replace('{/branch}', `/${ref || repo.default_branch}`), + { headers }, + ); + + return { repo, branch }; + } + + private async fetchResponse( + url: string | URL, + init: RequestInit, + ): Promise { + const urlAsString = url.toString(); + + const response = await fetch(urlAsString, init); + + if (!response.ok) { + const message = `Request failed for ${urlAsString}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return response; + } + + private async fetchJson(url: string | URL, init: RequestInit): Promise { + const response = await this.fetchResponse(url, init); + return await response.json(); + } } diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 24bfa3f1a8..d22529293f 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -14,16 +14,16 @@ * limitations under the License. */ -import tar, { Parse, ParseStream, ReadEntry } from 'tar'; -import platformPath from 'path'; -import fs from 'fs-extra'; -import { Readable, pipeline as pipelineCb } from 'stream'; -import { promisify } from 'util'; import concatStream from 'concat-stream'; +import fs from 'fs-extra'; +import platformPath from 'path'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import tar, { Parse, ParseStream, ReadEntry } from 'tar'; +import { promisify } from 'util'; import { ReadTreeResponse, - ReadTreeResponseFile, ReadTreeResponseDirOptions, + ReadTreeResponseFile, } from '../types'; // Tar types for `Parse` is not a proper constructor, but it should be diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 323f7b4778..2547c7b0e4 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import platformPath from 'path'; -import fs from 'fs-extra'; -import unzipper, { Entry } from 'unzipper'; import archiver from 'archiver'; +import fs from 'fs-extra'; +import platformPath from 'path'; import { Readable } from 'stream'; +import unzipper, { Entry } from 'unzipper'; import { ReadTreeResponse, - ReadTreeResponseFile, ReadTreeResponseDirOptions, + ReadTreeResponseFile, } from '../types'; // Matches a directory name + one `/` at the start of any string, diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index f9dfbe9aff..be788fc6e2 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -18,6 +18,32 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { ReadTreeResponseFactory } from './tree'; +/** + * A generic interface for fetching plain data from URLs. + */ +export type UrlReader = { + read(url: string): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; +}; + +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = (options: { + config: Config; + logger: Logger; + treeResponseFactory: ReadTreeResponseFactory; +}) => UrlReaderPredicateTuple[]; + +/** + * An options object for readTree operations. + */ export type ReadTreeOptions = { /** * A filter that can be used to select which files should be included. @@ -47,39 +73,6 @@ export type ReadTreeOptions = { etag?: string; }; -/** - * A generic interface for fetching plain data from URLs. - */ -export type UrlReader = { - read(url: string): Promise; - readTree(url: string, options?: ReadTreeOptions): Promise; -}; - -export type UrlReaderPredicateTuple = { - predicate: (url: URL) => boolean; - reader: UrlReader; -}; - -/** - * A factory function that can read config to construct zero or more - * UrlReaders along with a predicate for when it should be used. - */ -export type ReaderFactory = (options: { - config: Config; - logger: Logger; - treeResponseFactory: ReadTreeResponseFactory; -}) => UrlReaderPredicateTuple[]; - -export type ReadTreeResponseFile = { - path: string; - content(): Promise; -}; - -export type ReadTreeResponseDirOptions = { - /** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */ - targetDir?: string; -}; - export type ReadTreeResponse = { /** * files() returns an array of all the files inside the tree and corresponding functions to read their content. @@ -97,3 +90,16 @@ export type ReadTreeResponse = { */ etag: string; }; + +export type ReadTreeResponseDirOptions = { + /** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */ + targetDir?: string; +}; + +/** + * Represents a single file in a readTree response. + */ +export type ReadTreeResponseFile = { + path: string; + content(): Promise; +}; diff --git a/packages/integration/src/github/GithubCredentialsProvider.ts b/packages/integration/src/github/GithubCredentialsProvider.ts index a22c69909b..0bf217c824 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -53,7 +53,7 @@ class Cache { /** * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entierly + * It has no effect on calls to github.com and can probably be removed entirely * once GitHub Apps is out of preview. */ const HEADERS = { From 82af0855bc2d9fe8ae71d2d84af9dd467f51507a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 8 Feb 2021 12:21:47 +0100 Subject: [PATCH 73/78] backend-common: use explicit content type in error response --- packages/backend-common/src/middleware/errorHandler.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend-common/src/middleware/errorHandler.ts b/packages/backend-common/src/middleware/errorHandler.ts index a08849813d..2f3b904d38 100644 --- a/packages/backend-common/src/middleware/errorHandler.ts +++ b/packages/backend-common/src/middleware/errorHandler.ts @@ -63,10 +63,10 @@ export function errorHandler( return ( error: Error, _request: Request, - response: Response, + res: Response, next: NextFunction, ) => { - if (response.headersSent) { + if (res.headersSent) { // If the headers have already been sent, do not send the response again // as this will throw an error in the backend. next(error); @@ -80,7 +80,9 @@ export function errorHandler( logger.error(error); } - response.status(status).send(message); + res.status(status); + res.setHeader('content-type', 'text/plain'); + res.send(message); }; } From 3c4444cea2b0f65c8676d2829cf059edf7db9b46 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 8 Feb 2021 13:20:45 +0100 Subject: [PATCH 74/78] catalog-backend: use more explicit response types and let error handlers handle errors --- plugins/catalog-backend/src/service/router.ts | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 6064164e31..17cfc77d8b 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { errorHandler } from '@backstage/backend-common'; +import { errorHandler, NotFoundError } from '@backstage/backend-common'; import { locationSpecSchema, analyzeLocationSchema, @@ -57,7 +57,7 @@ export async function createRouter( const filter = EntityFilters.ofQuery(req.query); const fieldMapper = translateQueryToFieldMapper(req.query); const entities = await entitiesCatalog.entities(filter); - res.status(200).send(entities.map(fieldMapper)); + res.status(200).json(entities.map(fieldMapper)); }) .post('/entities', async (req, res) => { const body = await requireRequestBody(req); @@ -67,7 +67,7 @@ export async function createRouter( const [entity] = await entitiesCatalog.entities( EntityFilters.ofMatchers({ 'metadata.uid': result.entityId }), ); - res.status(200).send(entity); + res.status(200).json(entity); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; @@ -75,15 +75,14 @@ export async function createRouter( EntityFilters.ofMatchers({ 'metadata.uid': uid }), ); if (!entities.length) { - res.status(404).send(`No entity with uid ${uid}`); - } else { - res.status(200).send(entities[0]); + throw new NotFoundError(`No entity with uid ${uid}`); } + res.status(200).json(entities[0]); }) .delete('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; await entitiesCatalog.removeEntityByUid(uid); - res.status(204).send(); + res.status(204).end(); }) .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => { const { kind, namespace, name } = req.params; @@ -95,14 +94,11 @@ export async function createRouter( }), ); if (!entities.length) { - res - .status(404) - .send( - `No entity with kind ${kind} namespace ${namespace} name ${name}`, - ); - } else { - res.status(200).send(entities[0]); + throw new NotFoundError( + `No entity with kind ${kind} namespace ${namespace} name ${name}`, + ); } + res.status(200).json(entities[0]); }); } @@ -111,7 +107,7 @@ export async function createRouter( const input = await validateRequestBody(req, locationSpecSchema); const dryRun = yn(req.query.dryRun, { default: false }); const output = await higherOrderOperation.addLocation(input, { dryRun }); - res.status(201).send(output); + res.status(201).json(output); }); } @@ -119,22 +115,22 @@ export async function createRouter( router .get('/locations', async (_req, res) => { const output = await locationsCatalog.locations(); - res.status(200).send(output); + res.status(200).json(output); }) .get('/locations/:id/history', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.locationHistory(id); - res.status(200).send(output); + res.status(200).json(output); }) .get('/locations/:id', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.location(id); - res.status(200).send(output); + res.status(200).json(output); }) .delete('/locations/:id', async (req, res) => { const { id } = req.params; await locationsCatalog.removeLocation(id); - res.status(204).send(); + res.status(204).end(); }); } @@ -142,7 +138,7 @@ export async function createRouter( router.post('/analyze-location', async (req, res) => { const input = await validateRequestBody(req, analyzeLocationSchema); const output = await locationAnalyzer.analyzeLocation(input); - res.status(200).send(output); + res.status(200).json(output); }); } From 82b2c11b67472ec8ba2f44219e04151aadf5cc24 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 8 Feb 2021 13:26:11 +0100 Subject: [PATCH 75/78] added changesets --- .changeset/odd-buckets-compare.md | 5 +++++ .changeset/strange-cobras-unite.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/odd-buckets-compare.md create mode 100644 .changeset/strange-cobras-unite.md diff --git a/.changeset/odd-buckets-compare.md b/.changeset/odd-buckets-compare.md new file mode 100644 index 0000000000..faf68fb4a0 --- /dev/null +++ b/.changeset/odd-buckets-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Set explicit content-type in error handler responses. diff --git a/.changeset/strange-cobras-unite.md b/.changeset/strange-cobras-unite.md new file mode 100644 index 0000000000..d53a324dab --- /dev/null +++ b/.changeset/strange-cobras-unite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Refactored route response handling to use more explicit types and throw errors. From d2441aee3cd8fedc5d3962e918ba11a9634bcb79 Mon Sep 17 00:00:00 2001 From: Gowind Date: Fri, 5 Feb 2021 15:47:00 +0100 Subject: [PATCH 76/78] Add changeset --- .changeset/early-hotels-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/early-hotels-mate.md diff --git a/.changeset/early-hotels-mate.md b/.changeset/early-hotels-mate.md new file mode 100644 index 0000000000..2cda0344bf --- /dev/null +++ b/.changeset/early-hotels-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +use child logger, if provided, to log single location refresh From e3101688bc74cdc6fefbd2408eaa52f95cfab05b Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 8 Feb 2021 09:27:19 -0500 Subject: [PATCH 77/78] use P30D for snooze option --- plugins/cost-insights/src/types/Alert.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/cost-insights/src/types/Alert.ts b/plugins/cost-insights/src/types/Alert.ts index eb8919b71a..84c37a7c15 100644 --- a/plugins/cost-insights/src/types/Alert.ts +++ b/plugins/cost-insights/src/types/Alert.ts @@ -149,7 +149,7 @@ export const AlertSnoozeOptions: AlertSnoozeOption[] = [ label: '1 Month', }, { - duration: Duration.P3M, + duration: Duration.P90D, label: '1 Quarter', }, ]; From d36660721cd1a81b424a94ce7d8c40042e74ac0b Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 8 Feb 2021 09:30:36 -0500 Subject: [PATCH 78/78] changeset --- .changeset/cost-insights-tidy-geese-play.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-tidy-geese-play.md diff --git a/.changeset/cost-insights-tidy-geese-play.md b/.changeset/cost-insights-tidy-geese-play.md new file mode 100644 index 0000000000..f8aa503ca6 --- /dev/null +++ b/.changeset/cost-insights-tidy-geese-play.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +Fix snooze quarter option