From 9d9cdea1173223a6630e96d93dbc67fdf83bd119 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 30 Jan 2023 15:54:30 +0100 Subject: [PATCH 01/11] create backend-dev-utils with cli ipc client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .changeset/green-adults-retire.md | 5 + packages/backend-dev-utils/.eslintrc.js | 1 + packages/backend-dev-utils/README.md | 5 + packages/backend-dev-utils/package.json | 36 ++++++ packages/backend-dev-utils/src/index.ts | 21 ++++ packages/backend-dev-utils/src/ipcClient.ts | 120 +++++++++++++++++++ packages/backend-dev-utils/src/setupTests.ts | 16 +++ yarn.lock | 8 ++ 8 files changed, 212 insertions(+) create mode 100644 .changeset/green-adults-retire.md create mode 100644 packages/backend-dev-utils/.eslintrc.js create mode 100644 packages/backend-dev-utils/README.md create mode 100644 packages/backend-dev-utils/package.json create mode 100644 packages/backend-dev-utils/src/index.ts create mode 100644 packages/backend-dev-utils/src/ipcClient.ts create mode 100644 packages/backend-dev-utils/src/setupTests.ts diff --git a/.changeset/green-adults-retire.md b/.changeset/green-adults-retire.md new file mode 100644 index 0000000000..c13e8d664f --- /dev/null +++ b/.changeset/green-adults-retire.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-dev-utils': minor +--- + +Introduced a new package for backend development utilities. Similar to how `@backstage/dev-utils` is used in the frontend. diff --git a/packages/backend-dev-utils/.eslintrc.js b/packages/backend-dev-utils/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/packages/backend-dev-utils/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/backend-dev-utils/README.md b/packages/backend-dev-utils/README.md new file mode 100644 index 0000000000..8acce94cf0 --- /dev/null +++ b/packages/backend-dev-utils/README.md @@ -0,0 +1,5 @@ +# backend-dev-utils + +Welcome to the backend-dev-utils package! + +_This package is experimental._ diff --git a/packages/backend-dev-utils/package.json b/packages/backend-dev-utils/package.json new file mode 100644 index 0000000000..735185e37f --- /dev/null +++ b/packages/backend-dev-utils/package.json @@ -0,0 +1,36 @@ +{ + "name": "@backstage/backend-dev-utils", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "node-library" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/backend-dev-utils" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "devDependencies": { + "@backstage/cli": "workspace:^" + }, + "files": [ + "dist" + ] +} diff --git a/packages/backend-dev-utils/src/index.ts b/packages/backend-dev-utils/src/index.ts new file mode 100644 index 0000000000..bddecad73d --- /dev/null +++ b/packages/backend-dev-utils/src/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Local development helpers for backend. + * + * @packageDocumentation + */ diff --git a/packages/backend-dev-utils/src/ipcClient.ts b/packages/backend-dev-utils/src/ipcClient.ts new file mode 100644 index 0000000000..d3fa184a3f --- /dev/null +++ b/packages/backend-dev-utils/src/ipcClient.ts @@ -0,0 +1,120 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type SendMessage = Exclude; + +interface Request { + id: number; + method: string; + body: unknown; + type: string; +} + +type Response = + | { + type: string; + id: number; + body: unknown; + } + | { + type: string; + id: number; + error: Error; + }; + +const requestType = '@backstage/cli/channel/request'; +const responseType = '@backstage/cli/channel/response'; + +/** + * The client side of an IPC communication channel. + * + * @internal + */ +export class BackstageIpcClient { + #messageId = 0; + #sendMessage: SendMessage; + + /** + * Creates a new client if we're in a child process with IPC and BACKSTAGE_CLI_CHANNEL is set. + */ + static create(): BackstageIpcClient | undefined { + const sendMessage = process.send?.bind(process); + return sendMessage && process.env.BACKSTAGE_CLI_CHANNEL + ? new BackstageIpcClient(sendMessage) + : undefined; + } + + constructor(sendMessage: SendMessage) { + this.#sendMessage = sendMessage; + } + + /** + * Send a request to the parent process and wait for a response. + */ + async request( + method: string, + body: TRequestBody, + ): Promise { + return new Promise((resolve, reject) => { + const id = this.#messageId++; + + const request: Request = { + type: requestType, + id, + method, + body, + }; + + this.#sendMessage(request, (e: Error) => { + if (e) { + reject(e); + return; + } + + const timeout = setTimeout(() => { + reject(new Error('IPC request timed out')); + }, 1000); + timeout.unref(); + + const messageHandler = (response: Response) => { + if (response?.type !== responseType) { + return; + } + if (response.id !== id) { + return; + } + + if ('error' in response) { + const error = new Error(response.error.message); + if (response.error.name) { + error.name = response.error.name; + } + reject(error); + } else { + resolve(response.body as TResponseBody); + } + + clearTimeout(timeout); + process.removeListener('message', messageHandler); + }; + + process.addListener('message', messageHandler as () => void); + }); + }); + } +} + +export const ipcClient = BackstageIpcClient.create(); diff --git a/packages/backend-dev-utils/src/setupTests.ts b/packages/backend-dev-utils/src/setupTests.ts new file mode 100644 index 0000000000..4b9026cde5 --- /dev/null +++ b/packages/backend-dev-utils/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/yarn.lock b/yarn.lock index 3d809a40c8..ecd381c8f8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3519,6 +3519,14 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-dev-utils@workspace:packages/backend-dev-utils": + version: 0.0.0-use.local + resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils" + dependencies: + "@backstage/cli": "workspace:^" + languageName: unknown + linkType: soft + "@backstage/backend-plugin-api@workspace:^, @backstage/backend-plugin-api@workspace:packages/backend-plugin-api": version: 0.0.0-use.local resolution: "@backstage/backend-plugin-api@workspace:packages/backend-plugin-api" From a70806c3b09b851bc5785cc32700cb5b36f58f39 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 31 Jan 2023 12:08:55 +0100 Subject: [PATCH 02/11] backend-dev-utils: add DevDataStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Signed-off-by: Patrik Oldsberg --- packages/backend-dev-utils/api-report.md | 20 +++ .../src/DevDataStore.test.ts | 121 ++++++++++++++++++ .../backend-dev-utils/src/DevDataStore.ts | 99 ++++++++++++++ packages/backend-dev-utils/src/index.ts | 2 + 4 files changed, 242 insertions(+) create mode 100644 packages/backend-dev-utils/api-report.md create mode 100644 packages/backend-dev-utils/src/DevDataStore.test.ts create mode 100644 packages/backend-dev-utils/src/DevDataStore.ts diff --git a/packages/backend-dev-utils/api-report.md b/packages/backend-dev-utils/api-report.md new file mode 100644 index 0000000000..216311ccd8 --- /dev/null +++ b/packages/backend-dev-utils/api-report.md @@ -0,0 +1,20 @@ +## API Report File for "@backstage/backend-dev-utils" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +// @public +export class DevDataStore { + static get(): DevDataStore | undefined; + load(key: string): Promise<{ + loaded: boolean; + data: T; + }>; + save( + key: string, + data: T, + ): Promise<{ + saved: boolean; + }>; +} +``` diff --git a/packages/backend-dev-utils/src/DevDataStore.test.ts b/packages/backend-dev-utils/src/DevDataStore.test.ts new file mode 100644 index 0000000000..2bb3f221ae --- /dev/null +++ b/packages/backend-dev-utils/src/DevDataStore.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Serializable, spawn } from 'child_process'; +import { DevDataStore } from './DevDataStore'; +import { BackstageIpcClient } from './ipcClient'; + +function applyIpcTransform(value: Serializable): Promise { + const child = spawn( + 'node', + [ + '--eval', + ` + const listener = msg => { + process.send(msg); + process.removeListener('message', listener); + } + process.addListener('message', listener); + `, + ], + { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'], + serialization: 'advanced', + }, + ); + + return new Promise(resolve => { + child.addListener('message', resolve); + child.send(value); + }); +} + +class MockIpcClient implements Pick { + #data = new Map(); + + async request(method: string, body: any): Promise { + if (method === 'DevDataStore.save') { + this.#data.set(body.key, body.data); + return { saved: true }; + } else if (method === 'DevDataStore.load') { + if (!this.#data.has(body.key)) { + return { loaded: false, data: undefined }; + } + const data = this.#data.get(body.key)!; + return { loaded: true, data: await applyIpcTransform(data) }; + } + throw new Error('Unknown message'); + } +} + +describe('DevDataStore', () => { + it('should save and load data', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect(store.save('test', { foo: 'bar' })).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: { foo: 'bar' }, + }); + }); + + it('should save and load buffers', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', Buffer.from('abc', 'ascii')), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: Buffer.from('abc', 'ascii'), + }); + }); + + it('should save and load array buffers', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', new Uint8Array(Buffer.from('abc', 'ascii'))), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: new Uint8Array(Buffer.from('abc', 'ascii')), + }); + }); + + it('should save and load buffers nested in objects', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', { + x: Buffer.from('x', 'ascii'), + y: [Buffer.from('y', 'ascii')], + }), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: { x: Buffer.from('x', 'ascii'), y: [Buffer.from('y', 'ascii')] }, + }); + }); +}); diff --git a/packages/backend-dev-utils/src/DevDataStore.ts b/packages/backend-dev-utils/src/DevDataStore.ts new file mode 100644 index 0000000000..4069ab1bdb --- /dev/null +++ b/packages/backend-dev-utils/src/DevDataStore.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BackstageIpcClient, ipcClient } from './ipcClient'; + +interface SaveRequest { + key: string; + data: unknown; +} + +interface SaveResponse { + saved: boolean; +} + +interface LoadRequest { + key: string; +} + +interface LoadResponse { + loaded: boolean; + data: unknown; +} + +/** + * A data store that can be used to store temporary data during development. + * + * @public + */ +export class DevDataStore { + static #instance?: DevDataStore; + + /** + * Tries to acquire a DevDataStore instance. This will only succeed when the backend + * process is being run through the `@backstage/cli` in development mode. + * + * @returns A DevDataStore instance, or undefined if not available. + */ + static get(): DevDataStore | undefined { + if (ipcClient) { + if (!this.#instance) { + this.#instance = new DevDataStore(ipcClient); + } + return this.#instance; + } + return undefined; + } + + /** @internal */ + static forTest(client: Pick): DevDataStore { + return new DevDataStore(client as BackstageIpcClient); + } + + #client: BackstageIpcClient; + + private constructor(client: BackstageIpcClient) { + this.#client = client; + } + + /** + * Save data to the data store. + * + * @param key - The key used to identify the data. + * @param data - The data to save. The data will be serialized using advanced IPC serialization. + * @returns A promise that resolves to a result object that indicates whether the data was saved. + */ + async save(key: string, data: T): Promise<{ saved: boolean }> { + return this.#client.request( + 'DevDataStore.save', + { key, data }, + ); + } + + /** + * Loads data from the data store. + * + * @param key - The key used to identify the data. + * @returns A promise that resolves to a result object that indicates whether the data was loaded, as well as the data. + */ + async load(key: string): Promise<{ loaded: boolean; data: T }> { + const result = await this.#client.request( + 'DevDataStore.load', + { key }, + ); + return result as { loaded: boolean; data: T }; + } +} diff --git a/packages/backend-dev-utils/src/index.ts b/packages/backend-dev-utils/src/index.ts index bddecad73d..9b707db029 100644 --- a/packages/backend-dev-utils/src/index.ts +++ b/packages/backend-dev-utils/src/index.ts @@ -19,3 +19,5 @@ * * @packageDocumentation */ + +export { DevDataStore } from './DevDataStore'; From 90616df9a8784703dabc093337343ceab5d473d0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 31 Jan 2023 16:28:40 +0100 Subject: [PATCH 03/11] cli: add experimental backend start Co-authored-by: blam Signed-off-by: Patrik Oldsberg --- .changeset/weak-radios-sing.md | 5 + packages/cli/package.json | 3 + .../cli/src/commands/start/startBackend.ts | 34 +++-- .../cli/src/lib/experimental/ChannelServer.ts | 102 +++++++++++++ .../src/lib/experimental/ServerDataStore.ts | 78 ++++++++++ .../experimental/startBackendExperimental.ts | 138 ++++++++++++++++++ packages/cli/src/types.d.ts | 2 + yarn.lock | 43 +++++- 8 files changed, 392 insertions(+), 13 deletions(-) create mode 100644 .changeset/weak-radios-sing.md create mode 100644 packages/cli/src/lib/experimental/ChannelServer.ts create mode 100644 packages/cli/src/lib/experimental/ServerDataStore.ts create mode 100644 packages/cli/src/lib/experimental/startBackendExperimental.ts diff --git a/.changeset/weak-radios-sing.md b/.changeset/weak-radios-sing.md new file mode 100644 index 0000000000..dff287b565 --- /dev/null +++ b/.changeset/weak-radios-sing.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Added an experimental mode for the `package start` command for backend packages. Enabled by setting `EXPERIMENTAL_BACKEND_START`. diff --git a/packages/cli/package.json b/packages/cli/package.json index 909ac8460c..e6b9b7a97e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -36,6 +36,7 @@ "@backstage/errors": "workspace:^", "@backstage/release-manifests": "workspace:^", "@backstage/types": "workspace:^", + "@esbuild-kit/cjs-loader": "^2.4.1", "@manypkg/get-packages": "^1.1.3", "@octokit/request": "^6.0.0", "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7", @@ -65,6 +66,7 @@ "chalk": "^4.0.0", "chokidar": "^3.3.1", "commander": "^9.1.0", + "cross-spawn": "^7.0.3", "css-loader": "^6.5.1", "diff": "^5.0.0", "esbuild": "^0.17.0", @@ -136,6 +138,7 @@ "@backstage/dev-utils": "workspace:^", "@backstage/test-utils": "workspace:^", "@backstage/theme": "workspace:^", + "@types/cross-spawn": "^6.0.2", "@types/diff": "^5.0.0", "@types/express": "^4.17.6", "@types/fs-extra": "^9.0.1", diff --git a/packages/cli/src/commands/start/startBackend.ts b/packages/cli/src/commands/start/startBackend.ts index 61ada7a291..3e6a13be96 100644 --- a/packages/cli/src/commands/start/startBackend.ts +++ b/packages/cli/src/commands/start/startBackend.ts @@ -17,6 +17,7 @@ import fs from 'fs-extra'; import { paths } from '../../lib/paths'; import { serveBackend } from '../../lib/bundler'; +import { startBackendExperimental } from '../../lib/experimental/startBackendExperimental'; interface StartBackendOptions { checksEnabled: boolean; @@ -25,17 +26,28 @@ interface StartBackendOptions { } export async function startBackend(options: StartBackendOptions) { - // Cleaning dist/ before we start the dev process helps work around an issue - // where we end up with the entrypoint executing multiple times, causing - // a port bind conflict among other things. - await fs.remove(paths.resolveTarget('dist')); + if (process.env.EXPERIMENTAL_BACKEND_START) { + const waitForExit = await startBackendExperimental({ + entry: 'src/index', + checksEnabled: false, // not supported + inspectEnabled: options.inspectEnabled, + inspectBrkEnabled: options.inspectBrkEnabled, + }); - const waitForExit = await serveBackend({ - entry: 'src/index', - checksEnabled: options.checksEnabled, - inspectEnabled: options.inspectEnabled, - inspectBrkEnabled: options.inspectBrkEnabled, - }); + await waitForExit(); + } else { + // Cleaning dist/ before we start the dev process helps work around an issue + // where we end up with the entrypoint executing multiple times, causing + // a port bind conflict among other things. + await fs.remove(paths.resolveTarget('dist')); - await waitForExit(); + const waitForExit = await serveBackend({ + entry: 'src/index', + checksEnabled: options.checksEnabled, + inspectEnabled: options.inspectEnabled, + inspectBrkEnabled: options.inspectBrkEnabled, + }); + + await waitForExit(); + } } diff --git a/packages/cli/src/lib/experimental/ChannelServer.ts b/packages/cli/src/lib/experimental/ChannelServer.ts new file mode 100644 index 0000000000..db949f95b8 --- /dev/null +++ b/packages/cli/src/lib/experimental/ChannelServer.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeError } from '@backstage/errors'; +import { ChildProcess } from 'child_process'; + +interface RequestMeta { + generation: number; +} + +type MethodHandler = ( + req: TRequest, + meta: RequestMeta, +) => Promise; + +interface Request { + id: number; + method: string; + body: unknown; + type: string; +} + +const requestType = '@backstage/cli/channel/request'; +const responseType = '@backstage/cli/channel/response'; + +export class ChannelServer { + #generation = 1; + #methods = new Map>(); + + addChild(child: ChildProcess) { + const generation = this.#generation++; + const sendMessage = child.send?.bind(child); + if (!sendMessage) { + return; + } + + const messageListener = (request: Request) => { + if (request.type !== requestType) { + return; + } + + const handler = this.#methods.get(request.method); + if (!handler) { + sendMessage({ + type: responseType, + id: request.id, + error: { + name: 'NotFoundError', + message: `No handler registered for method ${request.method}`, + }, + }); + return; + } + + Promise.resolve() + .then(() => handler(request.body, { generation })) + .then(response => + sendMessage({ + type: responseType, + id: request.id, + body: response, + }), + ) + .catch(error => + sendMessage({ + type: responseType, + id: request.id, + error: serializeError(error), + }), + ); + }; + + child.addListener('message', messageListener as (req: unknown) => void); + + child.addListener('exit', () => { + child.removeListener('message', messageListener); + }); + } + + registerMethod( + method: string, + handler: MethodHandler, + ) { + if (this.#methods.has(method)) { + throw new Error(`A handler is already registered for method ${method}`); + } + this.#methods.set(method, handler); + } +} diff --git a/packages/cli/src/lib/experimental/ServerDataStore.ts b/packages/cli/src/lib/experimental/ServerDataStore.ts new file mode 100644 index 0000000000..29ac331de5 --- /dev/null +++ b/packages/cli/src/lib/experimental/ServerDataStore.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ChannelServer } from './ChannelServer'; + +interface StorageItem { + generation: number; + data: unknown; +} + +interface SaveRequest { + key: string; + data: unknown; +} + +interface SaveResponse { + saved: boolean; +} + +interface LoadRequest { + key: string; +} + +interface LoadResponse { + loaded: boolean; + data: unknown; +} + +export class ServerDataStore { + static bind(server: ChannelServer): void { + const store = new Map(); + + server.registerMethod( + 'DevDataStore.save', + async (request, { generation }) => { + const { key, data } = request; + if (!key) { + throw new Error('Key is required in DevDataStore.save'); + } + + const item = store.get(key); + + if (!item) { + store.set(key, { generation, data }); + return { saved: true }; + } + + if (item.generation > generation) { + return { saved: false }; + } + + store.set(key, { generation, data }); + return { saved: true }; + }, + ); + + server.registerMethod( + 'DevDataStore.load', + async request => { + const item = store.get(request.key); + return { loaded: Boolean(item), data: item?.data }; + }, + ); + } +} diff --git a/packages/cli/src/lib/experimental/startBackendExperimental.ts b/packages/cli/src/lib/experimental/startBackendExperimental.ts new file mode 100644 index 0000000000..94bf066925 --- /dev/null +++ b/packages/cli/src/lib/experimental/startBackendExperimental.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BackendServeOptions } from '../bundler/types'; +import type { ChildProcess } from 'child_process'; +import { fileURLToPath } from 'url'; +import { isAbsolute as isAbsolutePath } from 'path'; +import { FSWatcher, watch } from 'chokidar'; +import { ChannelServer } from './ChannelServer'; +import { ServerDataStore } from './ServerDataStore'; +import debounce from 'lodash/debounce'; +import spawn from 'cross-spawn'; +import { paths } from '../paths'; + +const loaderArgs = ['--require', require.resolve('@esbuild-kit/cjs-loader')]; + +export async function startBackendExperimental(options: BackendServeOptions) { + const envEnv = process.env as { NODE_ENV: string }; + if (!envEnv.NODE_ENV) { + envEnv.NODE_ENV = 'development'; + } + + // Set up the parent IPC server and bind the available services + const server = new ChannelServer(); + ServerDataStore.bind(server); + + let exiting = false; + let child: ChildProcess | undefined; + let watcher: FSWatcher | undefined = undefined; + let shutdownPromise: Promise | undefined = undefined; + + const restart = debounce(async () => { + // If a re-trigger happens during an existing shutdown, we just ignore it + if (shutdownPromise) { + return; + } + + if (child && !child.killed && child.exitCode === null) { + // We always wait for the existing process to exit, to make sure we don't get IPC conflicts + shutdownPromise = new Promise(resolve => child!.once('exit', resolve)); + child.kill(); + await shutdownPromise; + shutdownPromise = undefined; + } + + // We've received a shutdown signal + if (exiting) { + return; + } + + const optionArgs = new Array(); + if (options.inspectEnabled) { + optionArgs.push('--inspect'); + } else if (options.inspectBrkEnabled) { + optionArgs.push('--inspect-brk'); + } + + const userArgs = process.argv + .slice(['node', 'backstage-cli', 'package', 'start'].length) + .filter(arg => !optionArgs.includes(arg)); + + child = spawn( + process.execPath, + [...loaderArgs, ...optionArgs, options.entry, ...userArgs], + { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'], + env: { + ...process.env, + BACKSTAGE_CLI_CHANNEL: '1', + ESBK_TSCONFIG_PATH: paths.resolveTargetRoot('tsconfig.json'), + }, + serialization: 'advanced', + }, + ); + + server.addChild(child); + + // This captures messages sent by @esbuild-kit/cjs-loader + child.on('message', (data: { type?: string } | null) => { + if (typeof data === 'object' && data?.type === 'dependency') { + let path = (data as { path: string }).path; + if (path.startsWith('file:')) { + path = fileURLToPath(path); + } + + if (isAbsolutePath(path)) { + watcher?.add(path); + } + } + }); + }, 100); + + restart(); + + watcher = watch([paths.targetDir], { + cwd: process.cwd(), + ignored: ['**/.*/**', '**/node_modules/**'], + ignoreInitial: true, + ignorePermissionErrors: true, + }).on('all', restart); + + // Trigger restart on hitting enter in the terminal + process.stdin.on('data', restart); + + const exitPromise = new Promise(resolveExitPromise => { + async function handleSignal(signal: NodeJS.Signals) { + exiting = true; + + // Forward signals to child and wait for it to exit if still running + if (child && child.exitCode === null) { + await new Promise(resolve => { + child!.on('close', resolve); + child!.kill(signal); + }); + } + + resolveExitPromise(); + } + + process.once('SIGINT', handleSignal); + process.once('SIGTERM', handleSignal); + }); + + return () => exitPromise; +} diff --git a/packages/cli/src/types.d.ts b/packages/cli/src/types.d.ts index fedbe61f02..1222b2b4b8 100644 --- a/packages/cli/src/types.d.ts +++ b/packages/cli/src/types.d.ts @@ -258,3 +258,5 @@ declare module 'webpack-node-externals' { } } } + +declare module '@esbuild-kit/cjs-loader' {} diff --git a/yarn.lock b/yarn.lock index ecd381c8f8..7143443674 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3672,6 +3672,7 @@ __metadata: "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" "@backstage/types": "workspace:^" + "@esbuild-kit/cjs-loader": ^2.4.1 "@manypkg/get-packages": ^1.1.3 "@octokit/request": ^6.0.0 "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 @@ -3690,6 +3691,7 @@ __metadata: "@swc/core": ^1.3.9 "@swc/helpers": ^0.4.7 "@swc/jest": ^0.2.22 + "@types/cross-spawn": ^6.0.2 "@types/diff": ^5.0.0 "@types/express": ^4.17.6 "@types/fs-extra": ^9.0.1 @@ -3717,6 +3719,7 @@ __metadata: chalk: ^4.0.0 chokidar: ^3.3.1 commander: ^9.1.0 + cross-spawn: ^7.0.3 css-loader: ^6.5.1 del: ^6.0.0 diff: ^5.0.0 @@ -9196,6 +9199,26 @@ __metadata: languageName: node linkType: hard +"@esbuild-kit/cjs-loader@npm:^2.4.1": + version: 2.4.1 + resolution: "@esbuild-kit/cjs-loader@npm:2.4.1" + dependencies: + "@esbuild-kit/core-utils": ^3.0.0 + get-tsconfig: ^4.2.0 + checksum: a516065907be0ead76ac2199ccb08ff92659ba5e2edb4bb8772b6a63afe4faed7eb45a3b4d87266a68c7c135c3dba971cd087bc6f16c382356e835c7dd3440f5 + languageName: node + linkType: hard + +"@esbuild-kit/core-utils@npm:^3.0.0": + version: 3.0.0 + resolution: "@esbuild-kit/core-utils@npm:3.0.0" + dependencies: + esbuild: ~0.15.10 + source-map-support: ^0.5.21 + checksum: 0e89ec718e2211bf95c48a8085aaef88e8e416f42abd1c62d488d5458eecd3fbc144179a0c5570ad36fa7e2d3bbc411f8d3fb28802c37ced2154dc2c6ded9dfe + languageName: node + linkType: hard + "@esbuild/android-arm64@npm:0.17.5": version: 0.17.5 resolution: "@esbuild/android-arm64@npm:0.17.5" @@ -13912,6 +13935,15 @@ __metadata: languageName: node linkType: hard +"@types/cross-spawn@npm:^6.0.2": + version: 6.0.2 + resolution: "@types/cross-spawn@npm:6.0.2" + dependencies: + "@types/node": "*" + checksum: fa9edd32178878cab3ea8d6d0260639e0fe4860ddb3887b8de53d6e8036e154fc5f313c653f690975aa25025aea8beb83fb0870b931bf8d9202c3ac530a24c9d + languageName: node + linkType: hard + "@types/d3-color@npm:*": version: 3.0.2 resolution: "@types/d3-color@npm:3.0.2" @@ -21335,7 +21367,7 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.15.6": +"esbuild@npm:^0.15.6, esbuild@npm:~0.15.10": version: 0.15.18 resolution: "esbuild@npm:0.15.18" dependencies: @@ -23493,6 +23525,13 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.2.0": + version: 4.3.0 + resolution: "get-tsconfig@npm:4.3.0" + checksum: 2597aab99aa3a24db209e192a3e5874ac47fc5abc71703ee26346e0c5816cb346ca09fc813c739db5862d3a2905d89aeca1b0cbc46c2b272398d672309aaf414 + languageName: node + linkType: hard + "getopts@npm:2.3.0": version: 2.3.0 resolution: "getopts@npm:2.3.0" @@ -35146,7 +35185,7 @@ __metadata: languageName: node linkType: hard -"source-map-support@npm:^0.5.16, source-map-support@npm:~0.5.20": +"source-map-support@npm:^0.5.16, source-map-support@npm:^0.5.21, source-map-support@npm:~0.5.20": version: 0.5.21 resolution: "source-map-support@npm:0.5.21" dependencies: From ab2251564746e53a10759267dacdf83bb7d6769d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 31 Jan 2023 17:58:58 +0100 Subject: [PATCH 04/11] backend-app-api: move signal handlers to backend initializer + exit Signed-off-by: Patrik Oldsberg --- .changeset/red-kiwis-compare.md | 5 +++++ .../rootLifecycle/rootLifecycleFactory.ts | 5 +---- .../backend-app-api/src/wiring/BackendInitializer.ts | 12 ++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .changeset/red-kiwis-compare.md diff --git a/.changeset/red-kiwis-compare.md b/.changeset/red-kiwis-compare.md new file mode 100644 index 0000000000..6045367105 --- /dev/null +++ b/.changeset/red-kiwis-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +The shutdown signal handlers are now installed as part of the backend instance rather than the lifecycle service, and explicitly cause the process to exit. diff --git a/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts b/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts index 71ab76ae92..ebff53e9a6 100644 --- a/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts @@ -22,11 +22,8 @@ import { LoggerService, } from '@backstage/backend-plugin-api'; -const CALLBACKS = ['SIGTERM', 'SIGINT', 'beforeExit']; export class BackendLifecycleImpl implements RootLifecycleService { - constructor(private readonly logger: LoggerService) { - CALLBACKS.map(signal => process.on(signal, () => this.shutdown())); - } + constructor(private readonly logger: LoggerService) {} #isCalled = false; #shutdownTasks: Array = []; diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index ce17398a79..d1139615f1 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -87,6 +87,18 @@ export class BackendInitializer { } this.#started = true; + for (const event of ['SIGTERM', 'SIGINT', 'beforeExit']) { + process.on(event, async () => { + try { + await this.stop(); + process.exit(0); + } catch (error) { + console.error(error); + process.exit(1); + } + }); + } + // Initialize all root scoped services for (const ref of this.#serviceHolder.getServiceRefs()) { if (ref.scope === 'root') { From f60cca9da1defe5786a0f16bb17fac2502fca043 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 1 Feb 2023 10:57:42 +0100 Subject: [PATCH 05/11] backend-common: add support for restoring database state via dev store Signed-off-by: Patrik Oldsberg --- .changeset/long-chefs-burn.md | 5 ++ .changeset/old-ways-know.md | 5 ++ .../database/databaseFactory.ts | 10 +++- packages/backend-common/api-report.md | 14 ++++- packages/backend-common/package.json | 1 + .../src/database/DatabaseManager.ts | 15 +++-- .../backend-common/src/database/connection.ts | 5 +- .../src/database/connectors/sqlite3.ts | 55 +++++++++++++++---- packages/backend-common/src/database/index.ts | 5 +- packages/backend-common/src/database/types.ts | 20 ++++++- yarn.lock | 3 +- 11 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 .changeset/long-chefs-burn.md create mode 100644 .changeset/old-ways-know.md diff --git a/.changeset/long-chefs-burn.md b/.changeset/long-chefs-burn.md new file mode 100644 index 0000000000..03a7b76db9 --- /dev/null +++ b/.changeset/long-chefs-burn.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Updated database factory to pass service deps required for restoring database state during development. diff --git a/.changeset/old-ways-know.md b/.changeset/old-ways-know.md new file mode 100644 index 0000000000..60a1a923e8 --- /dev/null +++ b/.changeset/old-ways-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +The `DatabaseManager.forPlugin` method now accepts additional service dependencies. There is no need to update existing code to pass these dependencies. diff --git a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts index 8baf84cb69..4d95f863db 100644 --- a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts +++ b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts @@ -26,7 +26,8 @@ export const databaseFactory = createServiceFactory({ service: coreServices.database, deps: { config: coreServices.config, - plugin: coreServices.pluginMetadata, + lifecycle: coreServices.lifecycle, + pluginMetadata: coreServices.pluginMetadata, }, async createRootContext({ config }) { return config.getOptional('backend.database') @@ -39,7 +40,10 @@ export const databaseFactory = createServiceFactory({ }), ); }, - async factory({ plugin }, databaseManager) { - return databaseManager.forPlugin(plugin.getId()); + async factory({ pluginMetadata, lifecycle }, databaseManager) { + return databaseManager.forPlugin(pluginMetadata.getId(), { + pluginMetadata, + lifecycle, + }); }, }); diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index b03d6ec866..8b6311f8b0 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -32,6 +32,7 @@ import { IdentityService } from '@backstage/backend-plugin-api'; import { isChildPath } from '@backstage/cli-common'; import { Knex } from 'knex'; import { KubeConfig } from '@kubernetes/client-node'; +import { LifecycleService } from '@backstage/backend-plugin-api'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { Logger } from 'winston'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -40,6 +41,7 @@ import { PermissionsService } from '@backstage/backend-plugin-api'; import { CacheService as PluginCacheManager } from '@backstage/backend-plugin-api'; import { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; import { DiscoveryService as PluginEndpointDiscovery } from '@backstage/backend-plugin-api'; +import { PluginMetadataService } from '@backstage/backend-plugin-api'; import { PushResult } from 'isomorphic-git'; import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; @@ -232,6 +234,7 @@ export class Contexts { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: PluginDatabaseDependencies, ): Knex; // @public @@ -252,7 +255,10 @@ export function createStatusCheckRouter(options: { // @public export class DatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; + forPlugin( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): PluginDatabaseManager; static fromConfig( config: Config, options?: DatabaseManagerOptions, @@ -571,6 +577,12 @@ export function notFoundHandler(): RequestHandler; export { PluginCacheManager }; +// @public +export type PluginDatabaseDependencies = { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; +}; + export { PluginDatabaseManager }; export { PluginEndpointDiscovery }; diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c950950680..9f1069f322 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -35,6 +35,7 @@ }, "dependencies": { "@backstage/backend-app-api": "workspace:^", + "@backstage/backend-dev-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 5a497fff29..d8205faec1 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -27,7 +27,7 @@ import { ensureSchemaExists, normalizeConnection, } from './connection'; -import { PluginDatabaseManager } from './types'; +import { PluginDatabaseDependencies, PluginDatabaseManager } from './types'; import path from 'path'; import { LoggerService } from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; @@ -94,12 +94,15 @@ export class DatabaseManager { * should be unique as they are used to look up database config overrides under * `backend.database.plugin`. */ - forPlugin(pluginId: string): PluginDatabaseManager { + forPlugin( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): PluginDatabaseManager { const _this = this; return { getClient(): Promise { - return _this.getDatabase(pluginId); + return _this.getDatabase(pluginId, deps); }, migrations: { skip: false, @@ -307,7 +310,10 @@ export class DatabaseManager { * @returns Promise which resolves to a scoped Knex database client for a * plugin */ - private async getDatabase(pluginId: string): Promise { + private async getDatabase( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): Promise { if (this.databaseCache.has(pluginId)) { return this.databaseCache.get(pluginId)!; } @@ -351,6 +357,7 @@ export class DatabaseManager { const client = createDatabaseClient( pluginConfig, databaseClientOverrides, + deps, ); this.startKeepaliveLoop(pluginId, client); return client; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 5f487886b4..054ab7cfa8 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -19,7 +19,7 @@ import { JsonObject } from '@backstage/types'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; +import { DatabaseConnector, PluginDatabaseDependencies } from './types'; import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; @@ -55,11 +55,12 @@ const ConnectorMapping: Record = { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: PluginDatabaseDependencies, ) { const client: DatabaseClient = dbConfig.getString('client'); return ( - ConnectorMapping[client]?.createClient(dbConfig, overrides) ?? + ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ?? knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) ); } diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 6426005f0e..299e18a111 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -18,8 +18,9 @@ import { Config } from '@backstage/config'; import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; +import { DevDataStore } from '@backstage/backend-dev-utils'; import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; +import { DatabaseConnector, PluginDatabaseDependencies } from '../types'; /** * Creates a knex SQLite3 database connection @@ -30,22 +31,56 @@ import { DatabaseConnector } from '../types'; export function createSqliteDatabaseClient( dbConfig: Config, overrides?: Knex.Config, + deps?: PluginDatabaseDependencies, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); + const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig; // If storage on disk is used, ensure that the directory exists - if ( - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename && - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !== - ':memory:' - ) { - const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; - const directory = path.dirname(filename); - + if (connConfig.filename && connConfig.filename !== ':memory:') { + const directory = path.dirname(connConfig.filename); ensureDirSync(directory); } - const database = knexFactory(knexConfig); + let database: Knex; + + if (deps && connConfig.filename === ':memory:') { + // The dev store is used during watch mode to store and restore the database + // across reloads. It is only available when running the backend through + // `backstage-cli package start`. + const devStore = DevDataStore.get(); + const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`; + + if (devStore) { + database = knexFactory({ + ...knexConfig, + connection: async () => { + // If seed data is available, use it to restore the database + const { data: seedData } = await devStore?.load(dataKey); + + return { + ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig), + filename: seedData ?? ':memory:', + }; + }, + }); + } else { + database = knexFactory(knexConfig); + } + + if (devStore) { + // If the dev store is available we save the database state on shutdown + deps?.lifecycle?.addShutdownHook({ + async fn() { + const connection = await database.client.acquireConnection(); + const data = connection.serialize(); + await devStore.save(dataKey, data); + }, + }); + } + } else { + database = knexFactory(knexConfig); + } database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 0dde1ec239..483dbd9e40 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -22,5 +22,8 @@ export * from './DatabaseManager'; */ export { createDatabaseClient, ensureDatabaseExists } from './connection'; -export type { PluginDatabaseManager } from './types'; +export type { + PluginDatabaseManager, + PluginDatabaseDependencies, +} from './types'; export { isDatabaseConflictError } from './util'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index 2d59cf636e..7e96c44071 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -14,11 +14,25 @@ * limitations under the License. */ +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { Knex } from 'knex'; export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; +/** + * Service dependencies for `PluginDatabaseManager`. + * + * @public + */ +export type PluginDatabaseDependencies = { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; +}; + /** * DatabaseConnector manages an underlying Knex database driver. */ @@ -26,7 +40,11 @@ export interface DatabaseConnector { /** * createClient provides an instance of a knex database connector. */ - createClient(dbConfig: Config, overrides?: Partial): Knex; + createClient( + dbConfig: Config, + overrides?: Partial, + deps?: PluginDatabaseDependencies, + ): Knex; /** * createNameOverride provides a partial knex config sufficient to override a * database name. diff --git a/yarn.lock b/yarn.lock index 7143443674..650cb4c95d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3423,6 +3423,7 @@ __metadata: resolution: "@backstage/backend-common@workspace:packages/backend-common" dependencies: "@backstage/backend-app-api": "workspace:^" + "@backstage/backend-dev-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" @@ -3519,7 +3520,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/backend-dev-utils@workspace:packages/backend-dev-utils": +"@backstage/backend-dev-utils@workspace:^, @backstage/backend-dev-utils@workspace:packages/backend-dev-utils": version: 0.0.0-use.local resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils" dependencies: From cd4d0365ac9feddd40abe820d3c9097f92575202 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 1 Feb 2023 11:08:32 +0100 Subject: [PATCH 06/11] backend-next: use experimental start script Signed-off-by: Patrik Oldsberg --- packages/backend-next/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-next/package.json b/packages/backend-next/package.json index cc537ef514..fb821e19eb 100644 --- a/packages/backend-next/package.json +++ b/packages/backend-next/package.json @@ -18,7 +18,7 @@ "backstage" ], "scripts": { - "start": "backstage-cli package start", + "start": "EXPERIMENTAL_BACKEND_START=1 backstage-cli package start", "build": "backstage-cli package build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", From 03570c71e44b81f0da89d7cf244c79e4687e6046 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 5 Feb 2023 16:07:08 +0100 Subject: [PATCH 07/11] backend-common: inline plugin database deps type Signed-off-by: Patrik Oldsberg --- packages/backend-common/api-report.md | 16 ++++++++-------- .../src/database/DatabaseManager.ts | 18 ++++++++++++++---- .../backend-common/src/database/connection.ts | 11 +++++++++-- .../src/database/connectors/sqlite3.ts | 11 +++++++++-- packages/backend-common/src/database/index.ts | 5 +---- packages/backend-common/src/database/types.ts | 15 ++++----------- 6 files changed, 45 insertions(+), 31 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 8b6311f8b0..e77a58b34c 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -234,7 +234,10 @@ export class Contexts { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): Knex; // @public @@ -257,7 +260,10 @@ export function createStatusCheckRouter(options: { export class DatabaseManager { forPlugin( pluginId: string, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): PluginDatabaseManager; static fromConfig( config: Config, @@ -577,12 +583,6 @@ export function notFoundHandler(): RequestHandler; export { PluginCacheManager }; -// @public -export type PluginDatabaseDependencies = { - lifecycle: LifecycleService; - pluginMetadata: PluginMetadataService; -}; - export { PluginDatabaseManager }; export { PluginEndpointDiscovery }; diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index d8205faec1..08979a92f8 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -27,9 +27,13 @@ import { ensureSchemaExists, normalizeConnection, } from './connection'; -import { PluginDatabaseDependencies, PluginDatabaseManager } from './types'; +import { PluginDatabaseManager } from './types'; import path from 'path'; -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + LifecycleService, + LoggerService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; /** @@ -96,7 +100,10 @@ export class DatabaseManager { */ forPlugin( pluginId: string, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): PluginDatabaseManager { const _this = this; @@ -312,7 +319,10 @@ export class DatabaseManager { */ private async getDatabase( pluginId: string, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): Promise { if (this.databaseCache.has(pluginId)) { return this.databaseCache.get(pluginId)!; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 054ab7cfa8..ff10d349dd 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -19,9 +19,13 @@ import { JsonObject } from '@backstage/types'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector, PluginDatabaseDependencies } from './types'; +import { DatabaseConnector } from './types'; import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; type DatabaseClient = | 'pg' @@ -55,7 +59,10 @@ const ConnectorMapping: Record = { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ) { const client: DatabaseClient = dbConfig.getString('client'); diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 299e18a111..5a2dc46f18 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -20,7 +20,11 @@ import knexFactory, { Knex } from 'knex'; import path from 'path'; import { DevDataStore } from '@backstage/backend-dev-utils'; import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector, PluginDatabaseDependencies } from '../types'; +import { DatabaseConnector } from '../types'; +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; /** * Creates a knex SQLite3 database connection @@ -31,7 +35,10 @@ import { DatabaseConnector, PluginDatabaseDependencies } from '../types'; export function createSqliteDatabaseClient( dbConfig: Config, overrides?: Knex.Config, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig; diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 483dbd9e40..0dde1ec239 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -22,8 +22,5 @@ export * from './DatabaseManager'; */ export { createDatabaseClient, ensureDatabaseExists } from './connection'; -export type { - PluginDatabaseManager, - PluginDatabaseDependencies, -} from './types'; +export type { PluginDatabaseManager } from './types'; export { isDatabaseConflictError } from './util'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index 7e96c44071..621bcc676b 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -23,16 +23,6 @@ import { Knex } from 'knex'; export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; -/** - * Service dependencies for `PluginDatabaseManager`. - * - * @public - */ -export type PluginDatabaseDependencies = { - lifecycle: LifecycleService; - pluginMetadata: PluginMetadataService; -}; - /** * DatabaseConnector manages an underlying Knex database driver. */ @@ -43,7 +33,10 @@ export interface DatabaseConnector { createClient( dbConfig: Config, overrides?: Partial, - deps?: PluginDatabaseDependencies, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): Knex; /** * createNameOverride provides a partial knex config sufficient to override a From cbeba4b6f38d7db41f5b9d1f90585af2debaf614 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 5 Feb 2023 16:07:49 +0100 Subject: [PATCH 08/11] backend-common: simplify SQLite setup + work around warning Signed-off-by: Patrik Oldsberg --- .../src/database/connectors/sqlite3.ts | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 5a2dc46f18..e1ed6b18e9 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -43,47 +43,53 @@ export function createSqliteDatabaseClient( const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig; + const filename = connConfig.filename ?? ':memory:'; + // If storage on disk is used, ensure that the directory exists - if (connConfig.filename && connConfig.filename !== ':memory:') { - const directory = path.dirname(connConfig.filename); + if (filename !== ':memory:') { + const directory = path.dirname(filename); ensureDirSync(directory); } let database: Knex; - if (deps && connConfig.filename === ':memory:') { + if (deps && filename === ':memory:') { // The dev store is used during watch mode to store and restore the database // across reloads. It is only available when running the backend through // `backstage-cli package start`. const devStore = DevDataStore.get(); - const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`; if (devStore) { + const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`; + + const connectionLoader = async () => { + // If seed data is available, use it tconnectionLoader restore the database + const { data: seedData } = await devStore.load(dataKey); + + return { + ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig), + filename: seedData ?? ':memory:', + }; + }; + database = knexFactory({ ...knexConfig, - connection: async () => { - // If seed data is available, use it to restore the database - const { data: seedData } = await devStore?.load(dataKey); - - return { - ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig), - filename: seedData ?? ':memory:', - }; - }, + connection: Object.assign(connectionLoader, { + // This is a workaround for the knex SQLite driver always warning when using a config loader + filename: ':memory:', + }), }); - } else { - database = knexFactory(knexConfig); - } - if (devStore) { // If the dev store is available we save the database state on shutdown - deps?.lifecycle?.addShutdownHook({ + deps.lifecycle.addShutdownHook({ async fn() { const connection = await database.client.acquireConnection(); const data = connection.serialize(); await devStore.save(dataKey, data); }, }); + } else { + database = knexFactory(knexConfig); } } else { database = knexFactory(knexConfig); From 4f5bf2835c43699d61d6313abde4ec9a2329d025 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 5 Feb 2023 16:11:23 +0100 Subject: [PATCH 09/11] backend-app-api: tweak backend initializer to wait for startup before shutdown Signed-off-by: Patrik Oldsberg --- .../src/wiring/BackendInitializer.ts | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index d1139615f1..4bdee8b085 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -28,7 +28,7 @@ import { } from './types'; export class BackendInitializer { - #started = false; + #startPromise?: Promise; #features = new Map(); #registerInits = new Array(); #extensionPoints = new Map, unknown>(); @@ -75,30 +75,40 @@ export class BackendInitializer { } add(feature: BackendFeature, options?: TOptions) { - if (this.#started) { + if (this.#startPromise) { throw new Error('feature can not be added after the backend has started'); } this.#features.set(feature, options); } async start(): Promise { - if (this.#started) { + if (this.#startPromise) { throw new Error('Backend has already started'); } - this.#started = true; - for (const event of ['SIGTERM', 'SIGINT', 'beforeExit']) { - process.on(event, async () => { - try { - await this.stop(); - process.exit(0); - } catch (error) { - console.error(error); - process.exit(1); - } - }); - } + const exitHandler = async () => { + process.removeListener('SIGTERM', exitHandler); + process.removeListener('SIGINT', exitHandler); + process.removeListener('beforeExit', exitHandler); + try { + await this.stop(); + process.exit(0); + } catch (error) { + console.error(error); + process.exit(1); + } + }; + + process.addListener('SIGTERM', exitHandler); + process.addListener('SIGINT', exitHandler); + process.addListener('beforeExit', exitHandler); + + this.#startPromise = this.#doStart(); + await this.#startPromise; + } + + async #doStart(): Promise { // Initialize all root scoped services for (const ref of this.#serviceHolder.getServiceRefs()) { if (ref.scope === 'root') { @@ -189,9 +199,10 @@ export class BackendInitializer { } async stop(): Promise { - if (!this.#started) { + if (!this.#startPromise) { return; } + await this.#startPromise; const lifecycleService = await this.#serviceHolder.get( coreServices.rootLifecycle, From 81c06d5a4ad0e4579e40526dcb73e5f7728b21a3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 5 Feb 2023 16:11:47 +0100 Subject: [PATCH 10/11] backend-dev-utils: remove message listener when IPC client reuqest times out Signed-off-by: Patrik Oldsberg --- packages/backend-dev-utils/src/ipcClient.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/backend-dev-utils/src/ipcClient.ts b/packages/backend-dev-utils/src/ipcClient.ts index d3fa184a3f..87c1596a39 100644 --- a/packages/backend-dev-utils/src/ipcClient.ts +++ b/packages/backend-dev-utils/src/ipcClient.ts @@ -84,10 +84,7 @@ export class BackstageIpcClient { return; } - const timeout = setTimeout(() => { - reject(new Error('IPC request timed out')); - }, 1000); - timeout.unref(); + let timeout: NodeJS.Timeout | undefined = undefined; const messageHandler = (response: Response) => { if (response?.type !== responseType) { @@ -111,6 +108,12 @@ export class BackstageIpcClient { process.removeListener('message', messageHandler); }; + timeout = setTimeout(() => { + reject(new Error(`IPC request '${method}' with ID ${id} timed out`)); + process.removeListener('message', messageHandler); + }, 5000); + timeout.unref(); + process.addListener('message', messageHandler as () => void); }); }); From 1e6f5fcfede627b980d8e627b94ed8b857219213 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 5 Feb 2023 16:12:53 +0100 Subject: [PATCH 11/11] cli: rename ChannelServer to IpcServer Signed-off-by: Patrik Oldsberg --- .../src/lib/experimental/{ChannelServer.ts => IpcServer.ts} | 2 +- packages/cli/src/lib/experimental/ServerDataStore.ts | 4 ++-- packages/cli/src/lib/experimental/startBackendExperimental.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename packages/cli/src/lib/experimental/{ChannelServer.ts => IpcServer.ts} (98%) diff --git a/packages/cli/src/lib/experimental/ChannelServer.ts b/packages/cli/src/lib/experimental/IpcServer.ts similarity index 98% rename from packages/cli/src/lib/experimental/ChannelServer.ts rename to packages/cli/src/lib/experimental/IpcServer.ts index db949f95b8..8ceb72b046 100644 --- a/packages/cli/src/lib/experimental/ChannelServer.ts +++ b/packages/cli/src/lib/experimental/IpcServer.ts @@ -36,7 +36,7 @@ interface Request { const requestType = '@backstage/cli/channel/request'; const responseType = '@backstage/cli/channel/response'; -export class ChannelServer { +export class IpcServer { #generation = 1; #methods = new Map>(); diff --git a/packages/cli/src/lib/experimental/ServerDataStore.ts b/packages/cli/src/lib/experimental/ServerDataStore.ts index 29ac331de5..c0232157cb 100644 --- a/packages/cli/src/lib/experimental/ServerDataStore.ts +++ b/packages/cli/src/lib/experimental/ServerDataStore.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ChannelServer } from './ChannelServer'; +import { IpcServer } from './IpcServer'; interface StorageItem { generation: number; @@ -40,7 +40,7 @@ interface LoadResponse { } export class ServerDataStore { - static bind(server: ChannelServer): void { + static bind(server: IpcServer): void { const store = new Map(); server.registerMethod( diff --git a/packages/cli/src/lib/experimental/startBackendExperimental.ts b/packages/cli/src/lib/experimental/startBackendExperimental.ts index 94bf066925..c08fbda0da 100644 --- a/packages/cli/src/lib/experimental/startBackendExperimental.ts +++ b/packages/cli/src/lib/experimental/startBackendExperimental.ts @@ -19,7 +19,7 @@ import type { ChildProcess } from 'child_process'; import { fileURLToPath } from 'url'; import { isAbsolute as isAbsolutePath } from 'path'; import { FSWatcher, watch } from 'chokidar'; -import { ChannelServer } from './ChannelServer'; +import { IpcServer } from './IpcServer'; import { ServerDataStore } from './ServerDataStore'; import debounce from 'lodash/debounce'; import spawn from 'cross-spawn'; @@ -34,7 +34,7 @@ export async function startBackendExperimental(options: BackendServeOptions) { } // Set up the parent IPC server and bind the available services - const server = new ChannelServer(); + const server = new IpcServer(); ServerDataStore.bind(server); let exiting = false;