diff --git a/.changeset/cyan-peaches-lay.md b/.changeset/cyan-peaches-lay.md new file mode 100644 index 0000000000..a2b393caf8 --- /dev/null +++ b/.changeset/cyan-peaches-lay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Create a separate route for the template form editor so we refresh it without being redirected to scaffolder edit page. diff --git a/.changeset/fifty-trainers-watch.md b/.changeset/fifty-trainers-watch.md new file mode 100644 index 0000000000..5cef1ad2b2 --- /dev/null +++ b/.changeset/fifty-trainers-watch.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Make `emptyState` input optional on `entity-content:techdocs` extension so that +the default empty state extension works correctly. diff --git a/.changeset/slow-gorillas-thank.md b/.changeset/slow-gorillas-thank.md new file mode 100644 index 0000000000..2c52964a47 --- /dev/null +++ b/.changeset/slow-gorillas-thank.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-events-backend': patch +--- + +The events backend now has its own built-in event bus for distributing events across multiple backend instances. It exposes a new HTTP API under `/bus/v1/` for publishing and reading events from the bus, as well as its own storage and notification mechanism for events. + +The backing event store for the bus only supports scaled deployment if PostgreSQL is used as the DBMS. If SQLite or MySQL is used, the event bus will fall back to an in-memory store that does not support multiple backend instances. + +The default `EventsService` implementation from `@backstage/plugin-events-node` has also been updated to use the new events bus. diff --git a/.changeset/stale-roses-serve.md b/.changeset/stale-roses-serve.md new file mode 100644 index 0000000000..29dee56e22 --- /dev/null +++ b/.changeset/stale-roses-serve.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-events-node': patch +--- + +The default implementation of the `EventsService` now uses the new event bus for distributing events across multiple backend instances if the events backend plugin is installed. diff --git a/.changeset/sweet-chicken-smash.md b/.changeset/sweet-chicken-smash.md new file mode 100644 index 0000000000..9aa4c60803 --- /dev/null +++ b/.changeset/sweet-chicken-smash.md @@ -0,0 +1,14 @@ +--- +'@backstage/cli': minor +--- + +**BREAKING**: Removed the following deprecated commands: + +- `create`: Use `backstage-cli new` instead +- `create-plugin`: Use `backstage-cli new` instead +- `plugin:diff`: Use `backstage-cli fix` instead +- `test`: Use `backstage-cli repo test` or `backstage-cli package test` instead +- `versions:check`: Use `yarn dedupe` or `yarn-deduplicate` instead +- `clean`: Use `backstage-cli package clean` instead + +In addition, the experimental `install` and `onboard` commands have been removed since they have not received any updates since their introduction and we're expecting usage to be low. If you where relying on these commands, please let us know by opening an issue towards the main Backstage repository. diff --git a/packages/backend/package.json b/packages/backend/package.json index 64c24b9a4f..f5d7f88d1a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -44,6 +44,7 @@ "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^", "@backstage/plugin-devtools-backend": "workspace:^", + "@backstage/plugin-events-backend": "workspace:^", "@backstage/plugin-kubernetes-backend": "workspace:^", "@backstage/plugin-notifications-backend": "workspace:^", "@backstage/plugin-permission-backend": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 952f3315c4..3e8c2910b1 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -39,6 +39,7 @@ backend.add( import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'), ); backend.add(import('@backstage/plugin-catalog-backend/alpha')); +backend.add(import('@backstage/plugin-events-backend/alpha')); backend.add(import('@backstage/plugin-devtools-backend')); backend.add(import('@backstage/plugin-kubernetes-backend/alpha')); backend.add( diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index d77271c4b3..2eb9bb8dea 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -13,7 +13,6 @@ Options: Commands: new [options] - test config:docs [options] config:print [options] config:check [options] @@ -23,7 +22,6 @@ Commands: migrate [command] versions:bump [options] versions:migrate [options] - clean build-workspace [options] [packages...] create-github-app info @@ -40,15 +38,6 @@ Options: -h, --help ``` -### `backstage-cli clean` - -``` -Usage: backstage-cli clean [options] - -Options: - -h, --help -``` - ### `backstage-cli config:check` ``` @@ -481,120 +470,6 @@ Options: -h, --help ``` -### `backstage-cli test` - -``` -Usage: backstage-cli [--config=] [TestPathPattern] - -Options: - -h, --help - --version - --all - --automock - -b, --bail - --cache - --cacheDirectory - --changedFilesWithAncestor - --changedSince - --ci - --clearCache - --clearMocks - --collectCoverage - --collectCoverageFrom - --color - --colors - -c, --config - --coverage - --coverageDirectory - --coveragePathIgnorePatterns - --coverageProvider - --coverageReporters - --coverageThreshold - --debug - --detectLeaks - --detectOpenHandles - --env - --errorOnDeprecated - -e, --expand - --filter - --findRelatedTests - --forceExit - --globalSetup - --globalTeardown - --globals - --haste - --ignoreProjects - --init - --injectGlobals - --json - --lastCommit - --listTests - --logHeapUsage - --maxConcurrency - -w, --maxWorkers - --moduleDirectories - --moduleFileExtensions - --moduleNameMapper - --modulePathIgnorePatterns - --modulePaths - --noStackTrace - --notify - --notifyMode - -o, --onlyChanged - -f, --onlyFailures - --openHandlesTimeout - --outputFile - --passWithNoTests - --preset - --prettierPath - --projects - --randomize - --reporters - --resetMocks - --resetModules - --resolver - --restoreMocks - --rootDir - --roots - -i, --runInBand - --runTestsByPath - --runner - --seed - --selectProjects - --setupFiles - --setupFilesAfterEnv - --shard - --showConfig - --showSeed - --silent - --skipFilter - --snapshotSerializers - --testEnvironment - --testEnvironmentOptions - --testFailureExitCode - --testLocationInResults - --testMatch - -t, --testNamePattern - --testPathIgnorePatterns - --testPathPattern - --testRegex - --testResultsProcessor - --testRunner - --testSequencer - --testTimeout - --transform - --transformIgnorePatterns - --unmockedModulePathPatterns - -u, --updateSnapshot - --useStderr - --verbose - --watch - --watchAll - --watchPathIgnorePatterns - --watchman - --workerThreads -``` - ### `backstage-cli versions:bump` ``` diff --git a/packages/cli/src/commands/create-plugin/createPlugin.test.ts b/packages/cli/src/commands/create-plugin/createPlugin.test.ts deleted file mode 100644 index 84af6ebf69..0000000000 --- a/packages/cli/src/commands/create-plugin/createPlugin.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import path from 'path'; -import { movePlugin } from './createPlugin'; -import { createMockDirectory } from '@backstage/backend-test-utils'; - -const id = 'testPluginMock'; - -describe('createPlugin', () => { - const mockDir = createMockDirectory(); - - describe('movePlugin', () => { - it('should move the temporary plugin directory to its final place', async () => { - mockDir.setContent({ - [id]: {}, - }); - const tempDir = mockDir.resolve(id); - const pluginDir = mockDir.resolve('test-temp/plugins', id); - - await movePlugin(tempDir, pluginDir, id); - await expect(fs.pathExists(pluginDir)).resolves.toBe(true); - expect(pluginDir).toMatch(path.join('', 'plugins', id)); - }); - }); -}); diff --git a/packages/cli/src/commands/create-plugin/createPlugin.ts b/packages/cli/src/commands/create-plugin/createPlugin.ts deleted file mode 100644 index f8d7e54e97..0000000000 --- a/packages/cli/src/commands/create-plugin/createPlugin.ts +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { promisify } from 'util'; -import chalk from 'chalk'; -import inquirer, { Answers, Question } from 'inquirer'; -import { exec as execCb } from 'child_process'; -import { join as joinPath, resolve as resolvePath } from 'path'; -import camelCase from 'lodash/camelCase'; -import upperFirst from 'lodash/upperFirst'; -import os from 'os'; -import { OptionValues } from 'commander'; -import { assertError } from '@backstage/errors'; -import { - addCodeownersEntry, - getCodeownersFilePath, - parseOwnerIds, -} from '../../lib/codeowners'; -import { paths } from '../../lib/paths'; -import { Task, templatingTask } from '../../lib/tasks'; -import { Lockfile } from '../../lib/versioning'; -import { createPackageVersionProvider } from '../../lib/version'; - -const exec = promisify(execCb); - -async function checkExists(destination: string) { - await Task.forItem('checking', destination, async () => { - if (await fs.pathExists(destination)) { - const existing = chalk.cyan( - destination.replace(`${paths.targetRoot}/`, ''), - ); - throw new Error( - `A plugin with the same name already exists: ${existing}\nPlease try again with a different plugin ID`, - ); - } - }); -} - -const sortObjectByKeys = (obj: { [name in string]: string }) => { - return Object.keys(obj) - .sort() - .reduce((result, key: string) => { - result[key] = obj[key]; - return result; - }, {} as { [name in string]: string }); -}; - -export const capitalize = (str: string): string => - str.charAt(0).toUpperCase() + str.slice(1); - -export const addExportStatement = async ( - file: string, - exportStatement: string, -) => { - const newContents = fs - .readFileSync(file, 'utf8') - .split('\n') - .filter(Boolean) // get rid of empty lines - .concat([exportStatement]) - .concat(['']) // newline at end of file - .join('\n'); - - await fs.writeFile(file, newContents, 'utf8'); -}; - -export async function addPluginDependencyToApp( - rootDir: string, - pluginPackage: string, - versionStr: string, -) { - const packageFilePath = 'packages/app/package.json'; - const packageFile = resolvePath(rootDir, packageFilePath); - - await Task.forItem('processing', packageFilePath, async () => { - const packageFileContent = await fs.readFile(packageFile, 'utf-8'); - const packageFileJson = JSON.parse(packageFileContent); - const dependencies = packageFileJson.dependencies; - - if (dependencies[pluginPackage]) { - throw new Error( - `Plugin ${pluginPackage} already exists in ${packageFile}`, - ); - } - - dependencies[pluginPackage] = `^${versionStr}`; - packageFileJson.dependencies = sortObjectByKeys(dependencies); - const newContents = `${JSON.stringify(packageFileJson, null, 2)}\n`; - - await fs.writeFile(packageFile, newContents, 'utf-8').catch(error => { - throw new Error( - `Failed to add plugin as dependency to app: ${packageFile}: ${error.message}`, - ); - }); - }); -} - -export async function addPluginExtensionToApp( - pluginId: string, - extensionName: string, - pluginPackage: string, -) { - const pluginsFilePath = paths.resolveTargetRoot('packages/app/src/App.tsx'); - if (!(await fs.pathExists(pluginsFilePath))) { - return; - } - - await Task.forItem('processing', pluginsFilePath, async () => { - const content = await fs.readFile(pluginsFilePath, 'utf8'); - const revLines = content.split('\n').reverse(); - - const lastImportIndex = revLines.findIndex(line => - line.match(/ from ("|').*("|')/), - ); - const lastRouteIndex = revLines.findIndex(line => - line.match(/<\/FlatRoutes/), - ); - - if (lastImportIndex !== -1 && lastRouteIndex !== -1) { - revLines.splice( - lastImportIndex, - 0, - `import { ${extensionName} } from '${pluginPackage}';`, - ); - const [indentation] = revLines[lastRouteIndex + 1].match(/^\s*/) ?? []; - revLines.splice( - lastRouteIndex + 1, - 0, - `${indentation}}/>`, - ); - - const newContent = revLines.reverse().join('\n'); - await fs.writeFile(pluginsFilePath, newContent, 'utf8'); - } - }); -} - -async function cleanUp(tempDir: string) { - await Task.forItem('remove', 'temporary directory', async () => { - await fs.remove(tempDir); - }); -} - -async function buildPlugin(pluginFolder: string) { - const commands = [ - 'yarn install', - 'yarn lint --fix', - 'yarn tsc', - 'yarn build', - ]; - for (const command of commands) { - try { - await Task.forItem('executing', command, async () => { - process.chdir(pluginFolder); - await exec(command); - }).catch(error => { - process.stdout.write(error.stderr); - process.stdout.write(error.stdout); - throw new Error( - `Warning: Could not execute command ${chalk.cyan(command)}`, - ); - }); - } catch (error) { - assertError(error); - Task.error(error.message); - break; - } - } -} - -export async function movePlugin( - tempDir: string, - destination: string, - id: string, -) { - await Task.forItem('moving', id, async () => { - await fs.move(tempDir, destination).catch(error => { - throw new Error( - `Failed to move plugin from ${tempDir} to ${destination}: ${error.message}`, - ); - }); - }); -} - -export default async (opts: OptionValues) => { - const codeownersPath = await getCodeownersFilePath(paths.targetRoot); - - const questions: Question[] = [ - { - type: 'input', - name: 'id', - message: chalk.blue('Enter an ID for the plugin [required]'), - validate: (value: any) => { - if (!value) { - return chalk.red('Please enter an ID for the plugin'); - } else if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(value)) { - return chalk.red( - 'Plugin IDs must be lowercase and contain only letters, digits, and dashes.', - ); - } - return true; - }, - }, - ]; - - if (codeownersPath) { - questions.push({ - type: 'input', - name: 'owner', - message: chalk.blue( - 'Enter the owner(s) of the plugin. If specified, this will be added to CODEOWNERS for the plugin path. [optional]', - ), - validate: (value: any) => { - if (!value) { - return true; - } - - const ownerIds = parseOwnerIds(value); - if (!ownerIds) { - return chalk.red( - 'The owner must be a space separated list of team names (e.g. @org/team-name), usernames (e.g. @username), or the email addresses of users (e.g. user@example.com).', - ); - } - - return true; - }, - }); - } - - const answers: Answers = await inquirer.prompt(questions); - const pluginId = - opts.backend && !answers.id.endsWith('-backend') - ? `${answers.id}-backend` - : answers.id; - - const name = opts.scope - ? `@${opts.scope.replace(/^@/, '')}/plugin-${pluginId}` - : `plugin-${pluginId}`; - const pluginVar = `${camelCase(answers.id)}Plugin`; - const extensionName = `${upperFirst(camelCase(answers.id))}Page`; - const npmRegistry = opts.npmRegistry && opts.scope ? opts.npmRegistry : ''; - const privatePackage = opts.private === false ? false : true; - const isMonoRepo = await fs.pathExists(paths.resolveTargetRoot('lerna.json')); - const appPackage = paths.resolveTargetRoot('packages/app'); - const templateDir = paths.resolveOwn( - opts.backend - ? 'templates/default-backend-plugin' - : 'templates/default-plugin', - ); - const pluginDir = isMonoRepo - ? paths.resolveTargetRoot('plugins', pluginId) - : paths.resolveTargetRoot(pluginId); - const { version: pluginVersion } = isMonoRepo - ? await fs.readJson(paths.resolveTargetRoot('lerna.json')) - : { version: '0.1.0' }; - const license = opts.license ?? 'Apache-2.0'; - - let lockfile: Lockfile | undefined; - try { - lockfile = await Lockfile.load(paths.resolveTargetRoot('yarn.lock')); - } catch (error) { - console.warn(`No yarn.lock available, ${error}`); - } - - Task.log(); - Task.log('Creating the plugin...'); - - Task.section('Checking if the plugin ID is available'); - await checkExists(pluginDir); - - Task.section('Creating a temporary plugin directory'); - const tempDir = await fs.mkdtemp( - joinPath(os.tmpdir(), `backstage-plugin-${pluginId}`), - ); - - try { - Task.section('Preparing files'); - - await templatingTask( - templateDir, - tempDir, - { - ...answers, - pluginVar, - pluginVersion, - extensionName, - name, - privatePackage, - npmRegistry, - license, - }, - createPackageVersionProvider(lockfile), - isMonoRepo, - ); - - Task.section('Moving to final location'); - await movePlugin(tempDir, pluginDir, pluginId); - - Task.section('Building the plugin'); - await buildPlugin(pluginDir); - - if ((await fs.pathExists(appPackage)) && !opts.backend) { - Task.section('Adding plugin as dependency in app'); - await addPluginDependencyToApp(paths.targetRoot, name, pluginVersion); - - Task.section('Import plugin in app'); - await addPluginExtensionToApp(pluginId, extensionName, name); - } - - if (answers.owner) { - await addCodeownersEntry(`/plugins/${pluginId}`, answers.owner); - } - - Task.log(); - Task.log(`🥇 Successfully created ${chalk.cyan(`${name}`)}`); - Task.log(); - Task.exit(); - } catch (error) { - assertError(error); - Task.error(error.message); - - Task.log('It seems that something went wrong when creating the plugin 🤔'); - Task.log('We are going to clean up, and then you can try again.'); - - Task.section('Cleanup'); - await cleanUp(tempDir); - Task.error('🔥 Failed to create plugin!'); - Task.exit(1); - } -}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index b1658a5ff1..7cc23300af 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -25,13 +25,6 @@ const configOption = [ Array(), ] as const; -export function registerOnboardCommand(program: Command) { - program - .command('onboard', { hidden: true }) - .description('Get help setting up your Backstage App.') - .action(lazy(() => import('./onboard').then(m => m.command))); -} - export function registerRepoCommand(program: Command) { const command = program .command('repo [command]') @@ -265,63 +258,6 @@ export function registerCommands(program: Command) { .option('--no-private', 'Do not mark new packages as private') .action(lazy(() => import('./new/new').then(m => m.default))); - program - .command('create', { hidden: true }) - .storeOptionsAsProperties(false) - .description( - 'Open up an interactive guide to creating new things in your app [DEPRECATED]', - ) - .option( - '--select ', - 'Select the thing you want to be creating upfront', - ) - .option( - '--option =', - 'Pre-fill options for the creation process', - (opt, arr: string[]) => [...arr, opt], - [], - ) - .option('--scope ', 'The scope to use for new packages') - .option( - '--npm-registry ', - 'The package registry to use for new packages', - ) - .option('--no-private', 'Do not mark new packages as private') - .action(lazy(() => import('./new/new').then(m => m.default))); - - program - .command('create-plugin', { hidden: true }) - .option( - '--backend', - 'Create plugin with the backend dependencies as default', - ) - .description('Creates a new plugin in the current repository [DEPRECATED]') - .option('--scope ', 'npm scope') - .option('--npm-registry ', 'npm registry URL') - .option('--no-private', 'Public npm package') - .action( - lazy(() => import('./create-plugin/createPlugin').then(m => m.default)), - ); - - program - .command('plugin:diff', { hidden: true }) - .option('--check', 'Fail if changes are required') - .option('--yes', 'Apply all changes') - .description( - 'Diff an existing plugin with the creation template [DEPRECATED]', - ) - .action(lazy(() => import('./plugin/diff').then(m => m.default))); - - // TODO(Rugvip): Deprecate in favor of package variant - program - .command('test') - .allowUnknownOption(true) // Allows the command to run, but we still need to parse raw args - .helpOption(', --backstage-cli-help') // Let Jest handle help - .description( - 'Run tests, forwarding args to Jest, defaulting to watch mode [DEPRECATED]', - ) - .action(lazy(() => import('./test').then(m => m.default))); - program .command('config:docs') .option( @@ -385,7 +321,6 @@ export function registerCommands(program: Command) { registerRepoCommand(program); registerScriptCommand(program); registerMigrateCommand(program); - registerOnboardCommand(program); program .command('versions:bump') @@ -403,12 +338,6 @@ export function registerCommands(program: Command) { .description('Bump Backstage packages to the latest versions') .action(lazy(() => import('./versions/bump').then(m => m.default))); - program - .command('versions:check', { hidden: true }) - .option('--fix', 'Fix any auto-fixable versioning problems') - .description('Check Backstage package versioning') - .action(lazy(() => import('./versions/lint').then(m => m.default))); - program .command('versions:migrate') .option( @@ -424,12 +353,6 @@ export function registerCommands(program: Command) { ) .action(lazy(() => import('./versions/migrate').then(m => m.default))); - // TODO(Rugvip): Deprecate in favor of package variant - program - .command('clean') - .description('Delete cache directories [DEPRECATED]') - .action(lazy(() => import('./clean/clean').then(m => m.default))); - program .command('build-workspace [packages...]') .option( @@ -449,14 +372,48 @@ export function registerCommands(program: Command) { .description('Show helpful information for debugging and reporting bugs') .action(lazy(() => import('./info').then(m => m.default))); + // Notifications for removed commands program - .command('install [plugin-id]', { hidden: true }) - .option( - '--from ', - 'Install from a local package.json containing the installation recipe', - ) - .description('Install a Backstage plugin [EXPERIMENTAL]') - .action(lazy(() => import('./install/install').then(m => m.default))); + .command('create') + .allowUnknownOption(true) + .action(removed("use 'backstage-cli new' instead")); + program + .command('create-plugin') + .allowUnknownOption(true) + .action(removed("use 'backstage-cli new' instead")); + program + .command('plugin:diff') + .allowUnknownOption(true) + .action(removed("use 'backstage-cli fix' instead")); + program + .command('test') + .allowUnknownOption(true) + .action( + removed( + "use 'backstage-cli repo test' or 'backstage-cli package test' instead", + ), + ); + program + .command('clean') + .allowUnknownOption(true) + .action(removed("use 'backstage-cli package clean' instead")); + program + .command('versions:check') + .allowUnknownOption(true) + .action(removed("use 'yarn dedupe' or 'yarn-deduplicate' instead")); + program.command('install').allowUnknownOption(true).action(removed()); + program.command('onboard').allowUnknownOption(true).action(removed()); +} + +function removed(message?: string) { + return () => { + console.error( + message + ? `This command has been removed, ${message}` + : 'This command has been removed', + ); + process.exit(1); + }; } // Wraps an action function so that it always exits and handles errors diff --git a/packages/cli/src/commands/install/install.ts b/packages/cli/src/commands/install/install.ts deleted file mode 100644 index e395088b3e..0000000000 --- a/packages/cli/src/commands/install/install.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Step, - PackageWithInstallRecipe, - PeerPluginDependencies, -} from './types'; -import { fetchPackageInfo } from '../../lib/versioning'; -import { NotFoundError } from '../../lib/errors'; -import * as stepDefinitionMap from './steps'; -import { OptionValues } from 'commander'; -import fs from 'fs-extra'; - -const stepDefinitions = Object.values(stepDefinitionMap); - -async function fetchPluginPackage( - id: string, -): Promise { - const searchNames = [`@backstage/plugin-${id}`, `backstage-plugin-${id}`, id]; - - for (const name of searchNames) { - try { - const packageInfo = (await fetchPackageInfo( - name, - )) as PackageWithInstallRecipe; - return packageInfo; - } catch (error) { - if (error.name !== 'NotFoundError') { - throw error; - } - } - } - - throw new NotFoundError( - `No matching package found for '${id}', tried ${searchNames.join(', ')}`, - ); -} - -type Steps = Array<{ - type: string; - step: Step; -}>; - -class PluginInstaller { - static async resolveSteps( - pkg: PackageWithInstallRecipe, - versionToInstall?: string, - ) { - const steps: Steps = []; - - // collectDependencies - // TODO: Deps mean the plugin package itself, and any other backstage plugins/packages it depends on, in its installation recipe. - const dependencies = []; - dependencies.push({ - target: 'packages/app', - type: 'dependencies' as const, - name: pkg.name, - query: versionToInstall || `^${pkg.version}`, - }); - steps.push({ - type: 'dependencies', - step: stepDefinitionMap.dependencies.create({ dependencies }), - }); - - for (const step of pkg.experimentalInstallationRecipe?.steps ?? []) { - const { type } = step; - - const definition = stepDefinitions.find(d => d.type === type); - if (definition) { - steps.push({ - type, - step: definition.deserialize(step, pkg), - }); - } else { - throw new Error(`Unsupported step type: ${type}`); - } - } - - return steps; - } - - constructor(private readonly steps: Steps) {} - - async run() { - for (const { type, step } of this.steps) { - // TODO(Rugvip): Add spinners, nicer message about the step. - console.log(`Running step ${type}`); - await step.run(); - } - } -} - -async function installPluginAndPeerPlugins(pkg: PackageWithInstallRecipe) { - const pluginDeps: PeerPluginDependencies = new Map(); - pluginDeps.set(pkg.name, { pkg }); - await loadPeerPluginDeps(pkg, pluginDeps); - - console.log(`Installing ${pkg.name} AND any peer plugin dependencies.`); - for (const [_pluginDepName, pluginDep] of pluginDeps.entries()) { - const { pkg: pluginDepPkg, versionToInstall } = pluginDep; - console.log( - `Installing plugin: ${pluginDepPkg.name}: ${ - versionToInstall || pluginDepPkg.version - }`, - ); - const steps = await PluginInstaller.resolveSteps( - pluginDepPkg, - versionToInstall, - ); - const installer = new PluginInstaller(steps); - await installer.run(); - } -} - -async function loadPackageJson( - plugin: string, -): Promise { - if (plugin.endsWith('package.json')) { - // Install from local package if pluginId is a package.json file - needs to be absolute path - return await fs.readJson(plugin); - } - return await fetchPluginPackage(plugin); -} - -async function loadPeerPluginDeps( - pkg: PackageWithInstallRecipe, - pluginMap: PeerPluginDependencies, -) { - for (const [pluginId, pluginVersion] of Object.entries( - pkg.experimentalInstallationRecipe?.peerPluginDependencies ?? {}, - )) { - const depPkg = await loadPackageJson(pluginId); - if (!pluginMap.get(depPkg.name)) { - pluginMap.set(depPkg.name, { - pkg: depPkg, - versionToInstall: pluginVersion, - }); - await loadPeerPluginDeps(depPkg, pluginMap); - } - } -} - -export default async (pluginId?: string, cmd?: OptionValues) => { - const from = pluginId || cmd?.from; - // TODO(himanshu): If no plugin id is provided, it should list all plugins available. Maybe in some other command? - if (!from) { - throw new Error( - 'Missing both or a package.json file path in the --from flag.', - ); - } - const pkg = await loadPackageJson(from); - await installPluginAndPeerPlugins(pkg); -}; diff --git a/packages/cli/src/commands/install/steps/appRoute.ts b/packages/cli/src/commands/install/steps/appRoute.ts deleted file mode 100644 index f89b7eedff..0000000000 --- a/packages/cli/src/commands/install/steps/appRoute.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { paths } from '../../../lib/paths'; -import { Step, createStepDefinition } from '../types'; - -type Data = { - path: string; - element: string; - packageName: string; -}; - -class AppRouteStep implements Step { - constructor(private readonly data: Data) {} - - async run() { - const { path, element, packageName } = this.data; - - const appTsxPath = paths.resolveTargetRoot('packages/app/src/App.tsx'); - const contents = await fs.readFile(appTsxPath, 'utf-8'); - let failed = false; - - // Add a new route just above the end of the FlatRoutes block - const contentsWithRoute = contents.replace( - /(\s*)<\/FlatRoutes>/, - `$1 $1`, - ); - if (contentsWithRoute === contents) { - failed = true; - } - - // Grab the component name from the element - const componentName = element.match(/[A-Za-z0-9]+/)?.[0]; - if (!componentName) { - throw new Error(`Could not find component name in ${element}`); - } - - // Add plugin import - // TODO(Rugvip): Attempt to add this among the other plugin imports - const contentsWithImport = contentsWithRoute.replace( - /^import /m, - `import { ${componentName} } from '${packageName}';\nimport `, - ); - if (contentsWithImport === contentsWithRoute) { - failed = true; - } - - if (failed) { - console.log( - 'Failed to automatically add a route to package/app/src/App.tsx', - ); - console.log(`Action needed, add the following:`); - console.log(`1. import { ${componentName} } from '${packageName}';`); - console.log(`2. `); - } else { - await fs.writeFile(appTsxPath, contentsWithImport); - } - } -} - -export const appRoute = createStepDefinition({ - type: 'app-route', - - deserialize(obj, pkg) { - const { path, element } = obj; - if (!path || typeof path !== 'string') { - throw new Error("Invalid install step, 'path' must be a string"); - } - if (!element || typeof element !== 'string') { - throw new Error("Invalid install step, 'element' must be a string"); - } - return new AppRouteStep({ path, element, packageName: pkg.name }); - }, - - create(data: Data) { - return new AppRouteStep(data); - }, -}); diff --git a/packages/cli/src/commands/install/steps/dependencies.ts b/packages/cli/src/commands/install/steps/dependencies.ts deleted file mode 100644 index 4381066159..0000000000 --- a/packages/cli/src/commands/install/steps/dependencies.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import chalk from 'chalk'; -import sortBy from 'lodash/sortBy'; -import groupBy from 'lodash/groupBy'; -import { paths } from '../../../lib/paths'; -import { run } from '../../../lib/run'; -import { Step, createStepDefinition } from '../types'; - -type Data = { - dependencies: Array<{ - target: string; - type: 'dependencies'; - name: string; - query: string; - }>; -}; - -class DependenciesStep implements Step { - constructor(private readonly data: Data) {} - - async run() { - const { dependencies } = this.data; - // yarn --cwd packages/app add - const byTarget = groupBy(dependencies, 'target'); - - // Go through each target package and install the dependencies. - for (const [target, deps] of Object.entries(byTarget)) { - const pkgPath = paths.resolveTargetRoot(target, 'package.json'); - const pkgJson = await fs.readJson(pkgPath); - - // Populate each type of dependency object, dependencies, devDependencies, etc. - const depTypes = new Set(); - for (const dep of deps) { - depTypes.add(dep.type); - pkgJson[dep.type][dep.name] = dep.query; - } - - // Be nice and sort the dependencies alphabetically - for (const depType of depTypes) { - pkgJson[depType] = Object.fromEntries( - sortBy(Object.entries(pkgJson[depType]), ([key]) => key), - ); - } - await fs.writeJson(pkgPath, pkgJson, { spaces: 2 }); - } - - console.log(); - console.log( - `Running ${chalk.blue('yarn install')} to install new versions`, - ); - console.log(); - await run('yarn', ['install']); - } -} - -export const dependencies = createStepDefinition({ - type: 'dependencies', - - deserialize() { - throw new Error('The dependency step may not be defined in JSON'); - }, - - create(data: Data) { - return new DependenciesStep(data); - }, -}); diff --git a/packages/cli/src/commands/install/steps/message.ts b/packages/cli/src/commands/install/steps/message.ts deleted file mode 100644 index 5821c7d98b..0000000000 --- a/packages/cli/src/commands/install/steps/message.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Step, createStepDefinition } from '../types'; - -type Data = { - message: string; -}; - -class MessageStep implements Step { - constructor(private readonly data: Data) {} - - async run() { - console.log(this.data.message); - } -} - -export const message = createStepDefinition({ - type: 'message', - - deserialize(obj) { - const { message: msg } = obj; - - if (!msg || (typeof msg !== 'string' && !Array.isArray(msg))) { - throw new Error( - "Invalid install step, 'message' must be a string or array", - ); - } - return new MessageStep({ message: [msg].flat().join('') }); - }, - - create(data: Data) { - return new MessageStep(data); - }, -}); diff --git a/packages/cli/src/commands/install/types.ts b/packages/cli/src/commands/install/types.ts deleted file mode 100644 index 1fa52a5e82..0000000000 --- a/packages/cli/src/commands/install/types.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { YarnInfoInspectData } from '../../lib/versioning'; -import { JsonObject } from '@backstage/types'; - -/** - * TODO: possible types - * - * frontend-deps: Install one or many frontend packages in a Backstage app - * backend-deps: Install one or many backend packages in a Backstage app - * app-config: Update app-config.yaml (and ask for inputs). E.g. Use local or docker for techdocs.builder - * frontend-route: Add a frontend route to the plugin homepage - * backend-route: Add a backend route to the plugin - * entity-page-tab: Add a tab on Catalog’s entity page - * sidebar-item: Add a sidebar item - * frontend-api: Add a custom API - */ - -/** A serialized install step as it appears in JSON */ -export type SerializedStep = { - type: string; -} & unknown; - -export type InstallationRecipe = { - type?: 'frontend' | 'backend'; - steps: SerializedStep[]; - peerPluginDependencies: { [pluginId: string]: string }; -}; - -/** package.json data */ -export type PackageWithInstallRecipe = YarnInfoInspectData & { - version: string; - experimentalInstallationRecipe?: InstallationRecipe; -}; - -export type PeerPluginDependencies = Map< - string, - { - pkg: PackageWithInstallRecipe; - versionToInstall?: string; - } ->; - -export interface Step { - run(): Promise; -} - -export interface StepDefinition { - /** The string identifying this type of step */ - type: string; - - /** Deserializes and validate a JSON description of the step data */ - deserialize(obj: JsonObject, pkg: PackageWithInstallRecipe): Step; - - /** Creates a step using known parameters */ - create(options: Options): Step; -} - -/** Creates a new step definition. Only used as a helper for type inference */ -export function createStepDefinition( - config: StepDefinition, -): StepDefinition { - return config; -} diff --git a/packages/cli/src/commands/onboard/README.md b/packages/cli/src/commands/onboard/README.md deleted file mode 100644 index 0fb4d22666..0000000000 --- a/packages/cli/src/commands/onboard/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# onboard - -## Local dev - -# Create a new app - -```bash -cd .. -npx @backstage/create-app -cd new-backstage-app - -# Run the modded cli in the new app - -../RELATIVE_PATH_TO_PROJECT/backstage/packages/cli/bin/backstage-cli onboard -? Do you want to set up Authentication for this project? (Y/n) -... - -# Try the app with the new changes - -yarn dev - -``` diff --git a/packages/cli/src/commands/onboard/auth/index.ts b/packages/cli/src/commands/onboard/auth/index.ts deleted file mode 100644 index 0f4537b110..0000000000 --- a/packages/cli/src/commands/onboard/auth/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import inquirer from 'inquirer'; -import { Task } from '../../../lib/tasks'; -import { github, Answers as GitHubAnswers } from './providers/github'; -import { gitlab, Answers as GitLabAnswers } from './providers/gitlab'; - -export { type GitHubAnswers, type GitLabAnswers }; - -export async function auth(): Promise<{ - provider: string; - answers: GitHubAnswers | GitLabAnswers; -}> { - const answers = await inquirer.prompt<{ - provider: string; - }>([ - { - type: 'list', - name: 'provider', - message: 'Please select an authentication provider:', - choices: ['GitHub', 'GitLab'], - }, - ]); - - const { provider } = answers; - - let providerAnswers; - switch (provider) { - case 'GitHub': { - providerAnswers = await github(); - break; - } - case 'GitLab': { - providerAnswers = await gitlab(); - break; - } - default: - throw new Error(`Provider ${provider} not implemented yet.`); - } - - Task.log(); - Task.log(`Done setting up ${provider} Authentication!`); - Task.log(); - - return { - provider, - answers: providerAnswers, - }; -} diff --git a/packages/cli/src/commands/onboard/auth/patch.ts b/packages/cli/src/commands/onboard/auth/patch.ts deleted file mode 100644 index d004eeb157..0000000000 --- a/packages/cli/src/commands/onboard/auth/patch.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as fs from 'fs-extra'; -import * as path from 'path'; -import * as differ from 'diff'; -import { PATCH_FOLDER } from '../files'; -import { findPaths } from '@backstage/cli-common'; - -/* eslint-disable-next-line no-restricted-syntax */ -const { targetRoot } = findPaths(__dirname); - -export const patch = async (patchFile: string) => { - const patchContent = await fs.readFile( - path.join(PATCH_FOLDER, patchFile), - 'utf8', - ); - const targetName = patchContent.split('\n')[0].replace('--- a', ''); - const targetFile = path.join(targetRoot, targetName); - const oldContent = await fs.readFile(targetFile, 'utf8'); - const newContent = differ.applyPatch(oldContent, patchContent); - if (!newContent) { - throw new Error( - `Patch ${patchFile} was not applied correctly. - Did you change ${targetName} manually before running this command?`, - ); - } - - return await fs.writeFile(targetFile, newContent, 'utf8'); -}; diff --git a/packages/cli/src/commands/onboard/auth/patches/01-github.Add-SignInPage.patch b/packages/cli/src/commands/onboard/auth/patches/01-github.Add-SignInPage.patch deleted file mode 100644 index db9f5228e8..0000000000 --- a/packages/cli/src/commands/onboard/auth/patches/01-github.Add-SignInPage.patch +++ /dev/null @@ -1,25 +0,0 @@ ---- a/packages/app/src/App.tsx -+++ b/packages/app/src/App.tsx -@@ -30 +30,5 @@ import { Root } from './components/Root'; --import { AlertDisplay, OAuthRequestDialog } from '@backstage/core-components'; -+import { -+ AlertDisplay, -+ OAuthRequestDialog, -+ SignInPage, -+} from '@backstage/core-components'; -@@ -35,0 +40 @@ import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/ -+import { githubAuthApiRef } from '@backstage/core-plugin-api'; -@@ -38,0 +44,14 @@ const app = createApp({ -+ components: { -+ SignInPage: props => ( -+ -+ ), -+ }, \ No newline at end of file diff --git a/packages/cli/src/commands/onboard/auth/patches/02-gitlab.Add-SignInPage.patch b/packages/cli/src/commands/onboard/auth/patches/02-gitlab.Add-SignInPage.patch deleted file mode 100644 index da849220b8..0000000000 --- a/packages/cli/src/commands/onboard/auth/patches/02-gitlab.Add-SignInPage.patch +++ /dev/null @@ -1,25 +0,0 @@ ---- a/packages/app/src/App.tsx -+++ b/packages/app/src/App.tsx -@@ -30 +30,5 @@ import { Root } from './components/Root'; --import { AlertDisplay, OAuthRequestDialog } from '@backstage/core-components'; -+import { -+ AlertDisplay, -+ OAuthRequestDialog, -+ SignInPage, -+} from '@backstage/core-components'; -@@ -35,0 +40 @@ import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/ -+import { gitlabAuthApiRef } from '@backstage/core-plugin-api'; -@@ -38,0 +44,14 @@ const app = createApp({ -+ components: { -+ SignInPage: props => ( -+ -+ ), -+ }, \ No newline at end of file diff --git a/packages/cli/src/commands/onboard/auth/patches/03-gitlab.Change-signIn-resolver.patch b/packages/cli/src/commands/onboard/auth/patches/03-gitlab.Change-signIn-resolver.patch deleted file mode 100644 index 7a69669674..0000000000 --- a/packages/cli/src/commands/onboard/auth/patches/03-gitlab.Change-signIn-resolver.patch +++ /dev/null @@ -1,5 +0,0 @@ ---- a/packages/backend/src/plugins/auth.ts -+++ b/packages/backend/src/plugins/auth.ts -@@ -38 +38 @@ export default async function createPlugin( -- github: providers.github.create({ -+ gitlab: providers.gitlab.create({ \ No newline at end of file diff --git a/packages/cli/src/commands/onboard/auth/providers/github.ts b/packages/cli/src/commands/onboard/auth/providers/github.ts deleted file mode 100644 index 36c6b02038..0000000000 --- a/packages/cli/src/commands/onboard/auth/providers/github.ts +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { OAuthApp } from '@octokit/oauth-app'; -import chalk from 'chalk'; -import * as fs from 'fs-extra'; -import inquirer from 'inquirer'; -import { Task } from '../../../../lib/tasks'; -import { updateConfigFile } from '../../config'; -import { APP_CONFIG_FILE, PATCH_FOLDER } from '../../files'; -import { patch } from '../patch'; - -export type Answers = { - clientSecret: string; - clientId: string; - hasEnterprise: boolean; - enterpriseInstanceUrl?: string; -}; - -const validateCredentials = async (clientId: string, clientSecret: string) => { - try { - const app = new OAuthApp({ - clientId, - clientSecret, - }); - await app.createToken({ - code: '%NOT-VALID-CODE%', - }); - } catch (error) { - // @octokit/request returns a error.response object when a request is rejected. - // We can check it to see what kind of error we received. - - // If error.response is successful we can double-check that the error itself was due to the bad code. - // If that's the case then we can assume that the client id and secret exists as we otherwise would - // have gotten a 400/404. - if ( - error.response.status !== 200 && - error.response.data.error !== 'bad_verification_code' - ) { - throw new Error(`Validating GitHub Credentials failed.`); - } - } -}; - -const getConfig = (answers: Answers) => { - const { clientId, clientSecret, hasEnterprise, enterpriseInstanceUrl } = - answers; - - return { - auth: { - providers: { - github: { - development: { - clientId, - clientSecret, - ...(hasEnterprise && { - enterpriseInstanceUrl, - }), - }, - }, - }, - }, - }; -}; - -export const github = async (): Promise => { - Task.log(` - To add GitHub authentication, you must create an OAuth App from the GitHub developer settings: ${chalk.blue( - 'https://github.com/settings/developers', - )} - The Homepage URL should point to Backstage's frontend, while the Authorization callback URL will point to the auth backend. - - Settings for local development: - ${chalk.cyan(` - Homepage URL: http://localhost:3000 - Authorization callback URL: http://localhost:7007/api/auth/github/handler/frame`)} - - You can find the full documentation page here: ${chalk.blue( - 'https://backstage.io/docs/auth/github/provider', - )} - `); - - const answers = await inquirer.prompt([ - { - type: 'input', - name: 'clientId', - message: 'What is your Client Id?', - validate: (input: string) => (input.length ? true : false), - }, - { - type: 'input', - name: 'clientSecret', - message: 'What is your Client Secret?', - validate: (input: string) => (input.length ? true : false), - }, - { - type: 'confirm', - name: 'hasEnterprise', - message: 'Are you using GitHub Enterprise?', - }, - { - type: 'input', - name: 'enterpriseInstanceUrl', - message: 'What is your URL for GitHub Enterprise?', - when: ({ hasEnterprise }) => hasEnterprise, - validate: (input: string) => Boolean(new URL(input)), - }, - ]); - - const { clientId, clientSecret } = answers; - const config = getConfig(answers); - - Task.log('Setting up GitHub Authentication for you...'); - - await Task.forItem( - 'Validating', - 'credentials', - async () => await validateCredentials(clientId, clientSecret), - ); - await Task.forItem( - 'Updating', - APP_CONFIG_FILE, - async () => await updateConfigFile(APP_CONFIG_FILE, config), - ); - - const patches = await fs.readdir(PATCH_FOLDER); - for (const patchFile of patches.filter(p => p.includes('github'))) { - await Task.forItem('Patching', patchFile, async () => { - await patch(patchFile); - }); - } - - return answers; -}; diff --git a/packages/cli/src/commands/onboard/auth/providers/gitlab.ts b/packages/cli/src/commands/onboard/auth/providers/gitlab.ts deleted file mode 100644 index bc5491111c..0000000000 --- a/packages/cli/src/commands/onboard/auth/providers/gitlab.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import * as fs from 'fs-extra'; -import inquirer from 'inquirer'; -import { Task } from '../../../../lib/tasks'; -import { updateConfigFile } from '../../config'; -import { APP_CONFIG_FILE, PATCH_FOLDER } from '../../files'; -import { patch } from '../patch'; - -const getConfig = (answers: Answers) => { - const { clientId, clientSecret, hasAudience, audience } = answers; - - return { - auth: { - providers: { - gitlab: { - development: { - clientId, - clientSecret, - ...(hasAudience && { - audience, - }), - }, - }, - }, - }, - }; -}; - -export type Answers = { - clientSecret: string; - clientId: string; - hasAudience: boolean; - audience?: string; -}; - -export const gitlab = async (): Promise => { - Task.log(` - To add GitLab authentication, you must create an Application from the GitLab Settings: ${chalk.blue( - 'https://gitlab.com/-/profile/applications', - )} - The Redirect URI should point to your Backstage backend auth handler. - - Settings for local development: - ${chalk.cyan(` - Name: Backstage (or your custom app name) - Redirect URI: http://localhost:7007/api/auth/gitlab/handler/frame - Scopes: read_api and read_user`)} - - You can find the full documentation page here: ${chalk.blue( - 'https://backstage.io/docs/auth/gitlab/provider', - )} - `); - - const answers = await inquirer.prompt([ - { - type: 'input', - name: 'clientId', - message: 'What is your Application Id?', - validate: (input: string) => (input.length ? true : false), - }, - { - type: 'input', - name: 'clientSecret', - message: 'What is your Application Secret?', - validate: (input: string) => (input.length ? true : false), - }, - { - type: 'confirm', - name: 'hasAudience', - message: 'Do you have a self-hosted instance of GitLab?', - }, - { - type: 'input', - name: 'audience', - message: 'What is the URL for your GitLab instance?', - when: ({ hasAudience }) => hasAudience, - validate: (input: string) => Boolean(new URL(input)), - }, - ]); - - const config = getConfig(answers); - - Task.log('Setting up GitLab Authentication for you...'); - - await Task.forItem( - 'Updating', - APP_CONFIG_FILE, - async () => await updateConfigFile(APP_CONFIG_FILE, config), - ); - - const patches = await fs.readdir(PATCH_FOLDER); - for (const patchFile of patches.filter(p => p.includes('gitlab'))) { - await Task.forItem('Patching', patchFile, async () => { - await patch(patchFile); - }); - } - - return answers; -}; diff --git a/packages/cli/src/commands/onboard/command.ts b/packages/cli/src/commands/onboard/command.ts deleted file mode 100644 index f68685f3e6..0000000000 --- a/packages/cli/src/commands/onboard/command.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import inquirer from 'inquirer'; -import { Task } from '../../lib/tasks'; -import { auth } from './auth'; -import { integrations } from './integrations'; -import { discover } from './discovery'; - -export async function command(): Promise { - const answers = await inquirer.prompt<{ - shouldSetupAuth: boolean; - shouldSetupScaffolder: boolean; - shouldDiscoverEntities: boolean; - }>([ - { - type: 'confirm', - name: 'shouldSetupAuth', - message: 'Do you want to set up Authentication for this project?', - default: true, - }, - { - type: 'confirm', - name: 'shouldSetupScaffolder', - message: 'Do you want to use Software Templates in this project?', - default: true, - }, - { - type: 'confirm', - name: 'shouldDiscoverEntities', - message: - 'Do you want to discover entities and add them to the Software Catalog?', - default: true, - }, - ]); - - const { shouldSetupAuth, shouldSetupScaffolder, shouldDiscoverEntities } = - answers; - - if (!shouldSetupAuth && !shouldSetupScaffolder && !shouldDiscoverEntities) { - Task.log( - chalk.yellow( - 'If you change your mind, feel free to re-run this command.', - ), - ); - return; - } - - let providerInfo; - if (shouldSetupAuth) { - providerInfo = await auth(); - } - - if (shouldSetupScaffolder) { - await integrations(providerInfo); - } - - if (shouldDiscoverEntities) { - await discover(providerInfo); - } - - Task.log(); - Task.log( - `You can now start your app with ${chalk.inverse( - chalk.italic('yarn dev'), - )}`, - ); - Task.log(); -} diff --git a/packages/cli/src/commands/onboard/config.ts b/packages/cli/src/commands/onboard/config.ts deleted file mode 100644 index cce3119fcc..0000000000 --- a/packages/cli/src/commands/onboard/config.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as fs from 'fs-extra'; -import yaml from 'yaml'; - -const readYaml = async (file: string) => { - return yaml.parse(await fs.readFile(file, 'utf8')); -}; - -export const updateConfigFile = async (file: string, config: T) => { - const staticContent = - '# Backstage override configuration for your local development environment \n'; - - const content = fs.existsSync(file) - ? yaml.stringify( - { ...(await readYaml(file)), ...config }, - { - indent: 2, - }, - ) - : staticContent.concat( - yaml.stringify( - { ...config }, - { - indent: 2, - }, - ), - ); - - return await fs.writeFile(file, content, 'utf8'); -}; diff --git a/packages/cli/src/commands/onboard/discovery/Discovery.ts b/packages/cli/src/commands/onboard/discovery/Discovery.ts deleted file mode 100644 index b0db4255ad..0000000000 --- a/packages/cli/src/commands/onboard/discovery/Discovery.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import { Entity } from '@backstage/catalog-model'; -import { Analyzer } from './analyzers/types'; -import { Provider } from './providers/types'; -import { DefaultAnalysisOutputs } from './analyzers/DefaultAnalysisOutputs'; -import { Task } from '../../../lib/tasks'; - -export class Discovery { - readonly #providers: Provider[] = []; - readonly #analyzers: Analyzer[] = []; - - addProvider(provider: Provider) { - this.#providers.push(provider); - } - - addAnalyzer(analyzer: Analyzer) { - this.#analyzers.push(analyzer); - } - - async run(url: string): Promise<{ entities: Entity[] }> { - Task.log(`Running discovery for ${chalk.cyan(url)}`); - const result: Entity[] = []; - - for (const provider of this.#providers) { - const repositories = await provider.discover(url); - if (repositories && repositories.length) { - Task.log( - `Discovered ${chalk.cyan( - repositories.length, - )} repositories for ${chalk.cyan(provider.name())}`, - ); - - for (const repository of repositories) { - await Task.forItem('Analyzing', repository.name, async () => { - const output = new DefaultAnalysisOutputs(); - for (const analyzer of this.#analyzers) { - await analyzer.analyzeRepository({ repository, output }); - } - - output - .list() - .filter(entry => entry.type === 'entity') - .forEach(({ entity }) => result.push(entity)); - }); - } - - Task.log(`Produced ${chalk.cyan(result.length || 'no')} entities`); - } - } - - return { - entities: result, - }; - } -} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts b/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts deleted file mode 100644 index 82c8c2cdae..0000000000 --- a/packages/cli/src/commands/onboard/discovery/analyzers/BasicRepositoryAnalyzer.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ComponentEntity } from '@backstage/catalog-model'; -import { AnalysisOutputs, Analyzer } from './types'; -import { Repository } from '../providers/types'; - -/** - * Naive analyzer that produces a single entity that represents the repository - * as a whole. - */ -export class BasicRepositoryAnalyzer implements Analyzer { - name(): string { - return BasicRepositoryAnalyzer.name; - } - - async analyzeRepository(options: { - repository: Repository; - output: AnalysisOutputs; - }): Promise { - const entity: ComponentEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: options.repository.name, - ...(options.repository.description - ? { description: options.repository.description } - : {}), - }, - spec: { - type: 'service', - lifecycle: 'production', - owner: 'user:guest', - }, - }; - - options.output.produce({ - type: 'entity', - path: '/', - entity: entity, - }); - } -} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts b/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts deleted file mode 100644 index 16dd4b9301..0000000000 --- a/packages/cli/src/commands/onboard/discovery/analyzers/PackageJsonAnalyzer.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ANNOTATION_SOURCE_LOCATION, - ComponentEntity, -} from '@backstage/catalog-model'; -import z from 'zod'; -import { AnalysisOutputs, Analyzer } from './types'; -import { Repository, RepositoryFile } from '../providers/types'; - -export class PackageJsonAnalyzer implements Analyzer { - name(): string { - return PackageJsonAnalyzer.name; - } - - async analyzeRepository(options: { - repository: Repository; - output: AnalysisOutputs; - }): Promise { - const packageJson = await options.repository.file('package.json'); - if (!packageJson) { - return; - } - - const content = await readPackageJson(packageJson); - if (!content) { - return; - } - - const name = sanitizeName(content?.name) ?? options.repository.name; - - const entity: ComponentEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name, - ...(options.repository.description - ? { description: options.repository.description } - : {}), - tags: ['javascript'], - annotations: { - [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`, - }, - }, - spec: { - type: 'website', - lifecycle: 'production', - owner: 'user:guest', - }, - }; - - const decorate = options.output - .list() - .find(entry => entry.entity.metadata.name === name); - - if (decorate) { - decorate.entity.spec = { - ...decorate.entity.spec, - type: 'website', - }; - - decorate.entity.metadata.tags = [ - ...(decorate.entity.metadata.tags ?? []), - 'javascript', - ]; - - decorate.entity.metadata.annotations = { - ...decorate.entity.metadata.annotations, - [ANNOTATION_SOURCE_LOCATION]: `url:${options.repository.url}`, - }; - - return; - } - - options.output.produce({ - type: 'entity', - path: '/', - entity, - }); - } -} - -const packageSchema = z.object({ - name: z.string().optional(), -}); - -/** - * Makes sure that a name retrieved from a package.json file - * is reasonable and conforms to the catalog naming format. - * - * Read more about the naming format here: - * ADR002: Default Software Catalog File Format - * https://backstage.io/docs/architecture-decisions/adrs-adr002/ - */ -function sanitizeName(name?: string) { - return name && name !== 'root' - ? name.replace(/[^a-z0-9A-Z]/g, '_').substring(0, 62) - : undefined; -} - -async function readPackageJson( - file: RepositoryFile, -): Promise | undefined> { - try { - const text = await file.text(); - const result = packageSchema.safeParse(JSON.parse(text)); - if (!result.success) { - return undefined; - } - return { name: result.data.name }; - } catch (e) { - return undefined; - } -} diff --git a/packages/cli/src/commands/onboard/discovery/index.ts b/packages/cli/src/commands/onboard/discovery/index.ts deleted file mode 100644 index 8b2a5b694a..0000000000 --- a/packages/cli/src/commands/onboard/discovery/index.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import yaml from 'yaml'; -import inquirer from 'inquirer'; -import chalk from 'chalk'; -import { loadCliConfig } from '../../../lib/config'; -import { updateConfigFile } from '../config'; -import { APP_CONFIG_FILE, DISCOVERED_ENTITIES_FILE } from '../files'; -import { Discovery } from './Discovery'; -import { BasicRepositoryAnalyzer } from './analyzers/BasicRepositoryAnalyzer'; -import { PackageJsonAnalyzer } from './analyzers/PackageJsonAnalyzer'; -import { GithubDiscoveryProvider } from './providers/github/GithubDiscoveryProvider'; -import { GitlabDiscoveryProvider } from './providers/gitlab/GitlabDiscoveryProvider'; -import { GitHubAnswers, GitLabAnswers } from '../auth'; -import { Task } from '../../../lib/tasks'; - -export async function discover(providerInfo?: { - provider: string; - answers: GitHubAnswers | GitLabAnswers; -}) { - Task.log(` - Would you like to scan for - and create - Software Catalog entities? - - You will need to select which SCM (Source Code Management) provider you are using, - and then which repository or organization you want to scan. - - This will generate a new file in the root of your app containing discovered entities, - which will be included in the Software Catalog when you start up Backstage next time. - - Note that this command requires an access token, which can be either added through the integration config or - provided as an environment variable. - `); - - const answers = await inquirer.prompt<{ - shouldContinue: boolean; - provider: string; - url: string; - }>([ - { - type: 'confirm', - name: 'shouldContinue', - message: 'Do you want to continue?', - }, - { - type: 'list', - name: 'provider', - message: 'Please select which SCM provider you want to use:', - choices: ['GitHub', 'GitLab'], - default: providerInfo?.provider, - when: ({ shouldContinue }) => shouldContinue, - }, - { - type: 'input', - name: 'url', - message: `Which repository do you want to scan?`, - when: ({ shouldContinue }) => shouldContinue, - filter: (input, { provider }) => { - if (provider === 'GitLab') { - return `https://gitlab.com/${input}`; - } - if (provider === 'GitHub') { - return `https://github.com/${input}`; - } - return false; - }, - }, - ]); - - if (!answers.shouldContinue) { - Task.log( - chalk.yellow( - 'If you change your mind, feel free to re-run this command.', - ), - ); - return; - } - - const { fullConfig: config } = await loadCliConfig({ args: [] }); - - const discovery = new Discovery(); - - if (answers.provider === 'GitHub') { - discovery.addProvider(GithubDiscoveryProvider.fromConfig(config)); - } - if (answers.provider === 'GitLab') { - discovery.addProvider(GitlabDiscoveryProvider.fromConfig(config)); - } - - discovery.addAnalyzer(new BasicRepositoryAnalyzer()); - discovery.addAnalyzer(new PackageJsonAnalyzer()); - - const { entities } = await discovery.run(answers.url); - - if (!entities.length) { - Task.log( - chalk.yellow(` - We could not find enough information to be able to generate any Software Catalog entities for you. - Perhaps you can try again with a different repository?`), - ); - return; - } - - await Task.forItem('Creating', DISCOVERED_ENTITIES_FILE, async () => { - const payload: string[] = []; - for (const entity of entities) { - payload.push('---\n', yaml.stringify(entity)); - } - await fs.writeFile(DISCOVERED_ENTITIES_FILE, payload.join('')); - }); - - await Task.forItem( - 'Updating', - APP_CONFIG_FILE, - async () => - await updateConfigFile(APP_CONFIG_FILE, { - catalog: { - locations: [ - { - type: 'file', - target: DISCOVERED_ENTITIES_FILE, - }, - ], - }, - }), - ); -} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts b/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts deleted file mode 100644 index a24756edd7..0000000000 --- a/packages/cli/src/commands/onboard/discovery/providers/github/GithubDiscoveryProvider.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Config } from '@backstage/config'; -import { - DefaultGithubCredentialsProvider, - GithubCredentialsProvider, - ScmIntegrations, -} from '@backstage/integration'; -import { graphql } from '@octokit/graphql'; -import { - Repository as GraphqlRepository, - Query as GraphqlQuery, -} from '@octokit/graphql-schema'; -import parseGitUrl from 'git-url-parse'; -import { Provider, Repository } from '../types'; -import { GithubRepository } from './GithubRepository'; - -export class GithubDiscoveryProvider implements Provider { - readonly #envToken: string | undefined; - readonly #scmIntegrations: ScmIntegrations; - readonly #credentialsProvider: GithubCredentialsProvider; - - static fromConfig(config: Config): GithubDiscoveryProvider { - const envToken = process.env.GITHUB_TOKEN || undefined; - const scmIntegrations = ScmIntegrations.fromConfig(config); - const credentialsProvider = - DefaultGithubCredentialsProvider.fromIntegrations(scmIntegrations); - return new GithubDiscoveryProvider( - envToken, - scmIntegrations, - credentialsProvider, - ); - } - - private constructor( - envToken: string | undefined, - integrations: ScmIntegrations, - credentialsProvider: GithubCredentialsProvider, - ) { - this.#envToken = envToken; - this.#scmIntegrations = integrations; - this.#credentialsProvider = credentialsProvider; - } - - name(): string { - return 'GitHub'; - } - - async discover(url: string): Promise { - if (!url.startsWith('https://github.com/')) { - return false; - } - - const scmIntegration = this.#scmIntegrations.github.byUrl(url); - if (!scmIntegration) { - throw new Error(`No GitHub integration found for ${url}`); - } - - const parsed = parseGitUrl(url); - const { name, organization } = parsed; - const org = organization || name; // depends on if it's a repo url or an org url... - - const client = graphql.defaults({ - baseUrl: scmIntegration.config.apiBaseUrl, - headers: await this.#getRequestHeaders(url), - }); - - const { repositories } = await this.#getOrganizationRepositories( - client, - org, - ); - - return repositories - .filter(repo => repo.url.startsWith(url)) - .map(repo => new GithubRepository(client, repo, org)); - } - - async #getRequestHeaders(url: string): Promise> { - const credentials = await this.#credentialsProvider.getCredentials({ - url, - }); - - if (credentials.headers) { - return credentials.headers; - } else if (credentials.token) { - return { authorization: `token ${credentials.token}` }; - } - - if (this.#envToken) { - return { authorization: `token ${this.#envToken}` }; - } - - throw new Error( - 'No token available for GitHub, please configure your integrations or set a GITHUB_TOKEN env variable', - ); - } - - async #getOrganizationRepositories(client: typeof graphql, org: string) { - const query = `query repositories($org: String!, $cursor: String) { - repositoryOwner(login: $org) { - login - repositories(first: 50, after: $cursor) { - nodes { - name - url - description - isArchived - isFork - } - pageInfo { - hasNextPage - endCursor - } - } - } - }`; - - const result: GraphqlRepository[] = []; - - let cursor: string | undefined | null = undefined; - let hasNextPage = true; - - while (hasNextPage) { - const response: GraphqlQuery = await client(query, { - org, - cursor, - }); - - const { repositories: connection } = response.repositoryOwner ?? {}; - - if (!connection) { - throw new Error(`Found no repositories for ${org}`); - } - - for (const repository of connection.nodes ?? []) { - if (repository && !repository.isArchived && !repository.isFork) { - result.push(repository); - } - } - - cursor = connection.pageInfo.endCursor; - hasNextPage = connection.pageInfo.hasNextPage; - } - - return { - repositories: result, - }; - } -} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts b/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts deleted file mode 100644 index f3c68851a0..0000000000 --- a/packages/cli/src/commands/onboard/discovery/providers/github/GithubRepository.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { graphql } from '@octokit/graphql'; -import { - Repository as GraphqlRepository, - Blob as GraphqlBlob, -} from '@octokit/graphql-schema'; -import { Repository, RepositoryFile } from '../types'; -import { GithubFile } from './GithubFile'; - -export class GithubRepository implements Repository { - readonly #client: typeof graphql; - readonly #repo: GraphqlRepository; - readonly #org: string; - - constructor(client: typeof graphql, repo: GraphqlRepository, org: string) { - this.#client = client; - this.#repo = repo; - this.#org = org; - } - - get url(): string { - return this.#repo.url; - } - - get name(): string { - return this.#repo.name; - } - - get owner(): string { - return this.#org; - } - - get description(): string | undefined { - return this.#repo.description ?? undefined; - } - - async file(filename: string): Promise { - const content = await this.#getFileContent(filename); - if (!content || content.isBinary || !content.text) { - return undefined; - } - - return new GithubFile(filename, content.text ?? ''); - } - - async #getFileContent(filename: string) { - const query = `query RepoFiles($owner: String!, $name: String!, $expr: String!) { - repository(owner: $owner, name: $name) { - object(expression: $expr) { - ...on Blob { - text - isBinary - } - } - } - }`; - - const response = await this.#client<{ repository: GraphqlRepository }>( - query, - { - name: this.#repo.name, - owner: this.#org, - expr: `HEAD:${filename}`, - }, - ); - - return response.repository.object as GraphqlBlob; - } -} diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts deleted file mode 100644 index df9ffa4206..0000000000 --- a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabDiscoveryProvider.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Config } from '@backstage/config'; -import { - DefaultGitlabCredentialsProvider, - GitlabCredentialsProvider, - ScmIntegrations, -} from '@backstage/integration'; -import fetch from 'node-fetch'; -import { Provider } from '../types'; -import { GitlabProject, ProjectResponse } from './GitlabProject'; - -export class GitlabDiscoveryProvider implements Provider { - readonly #envToken: string | undefined; - readonly #scmIntegrations: ScmIntegrations; - readonly #credentialsProvider: GitlabCredentialsProvider; - - static fromConfig(config: Config): GitlabDiscoveryProvider { - const envToken = process.env.GITLAB_TOKEN || undefined; - const scmIntegrations = ScmIntegrations.fromConfig(config); - const credentialsProvider = - DefaultGitlabCredentialsProvider.fromIntegrations(scmIntegrations); - - return new GitlabDiscoveryProvider( - envToken, - scmIntegrations, - credentialsProvider, - ); - } - - private constructor( - envToken: string | undefined, - integrations: ScmIntegrations, - credentialsProvider: GitlabCredentialsProvider, - ) { - this.#envToken = envToken; - this.#scmIntegrations = integrations; - this.#credentialsProvider = credentialsProvider; - } - - name(): string { - return 'GitLab'; - } - - async discover(url: string): Promise { - const { origin, pathname } = new URL(url); - const [, user] = pathname.split('/'); - - const scmIntegration = this.#scmIntegrations.gitlab.byUrl(origin); - if (!scmIntegration) { - throw new Error(`No GitLab integration found for ${origin}`); - } - - const headers = await this.#getRequestHeaders(origin); - - const response = await fetch( - `${scmIntegration.config.apiBaseUrl}/users/${user}/projects`, - { headers }, - ); - - if (!response.ok) { - throw new Error(`${response.status} ${response.statusText}`); - } - - const projects: ProjectResponse[] = await response.json(); - - return projects.map( - project => - new GitlabProject(project, scmIntegration.config.apiBaseUrl, headers), - ); - } - - async #getRequestHeaders(url: string): Promise> { - const credentials = await this.#credentialsProvider.getCredentials({ - url, - }); - - if (credentials.headers) { - return credentials.headers; - } else if (credentials.token) { - return { authorization: `Bearer ${credentials.token}` }; - } - - if (this.#envToken) { - return { authorization: `Bearer ${this.#envToken}` }; - } - - throw new Error( - 'No token available for GitLab, please set a GITLAB_TOKEN env variable', - ); - } -} diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts b/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts deleted file mode 100644 index 258e8176bb..0000000000 --- a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabProject.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fetch from 'node-fetch'; -import { GitlabFile } from './GitlabFile'; -import { Repository, RepositoryFile } from '../types'; - -export type ProjectResponse = { - id: string; - name: string; - description: string; - owner: { - username: string; - }; - web_url: string; -}; - -type BranchResponse = { - default: boolean; - name: string; -}; - -type FileContentResponse = { - content: string; -}; - -export class GitlabProject implements Repository { - constructor( - private readonly project: ProjectResponse, - private readonly apiBaseUrl: string, - private readonly headers: { [name: string]: string }, - ) {} - - get url(): string { - return this.project.web_url; - } - - get name(): string { - return this.project.name; - } - - get owner(): string { - return this.project.owner.username; - } - - get description(): string { - return this.project.description; - } - - async file(filename: string): Promise { - const mainBranch = await this.#getMainBranch(); - const content = await this.#getFileContent(filename, mainBranch); - - return new GitlabFile(filename, content); - } - - async #getFileContent(path: string, mainBranch: string): Promise { - const response = await fetch( - `${this.apiBaseUrl}/projects/${this.project.id}/repository/files/${path}?ref=${mainBranch}`, - { headers: this.headers }, - ); - const { content }: FileContentResponse = await response.json(); - - return Buffer.from(content, 'base64').toString('ascii'); - } - - async #getMainBranch(): Promise { - const response = await fetch( - `${this.apiBaseUrl}/projects/${this.project.id}/repository/branches`, - { headers: this.headers }, - ); - const branches: BranchResponse[] = await response.json(); - - return branches.find(branch => branch.default)?.name ?? 'main'; - } -} diff --git a/packages/cli/src/commands/onboard/discovery/providers/types.ts b/packages/cli/src/commands/onboard/discovery/providers/types.ts deleted file mode 100644 index 2d8ce70ecf..0000000000 --- a/packages/cli/src/commands/onboard/discovery/providers/types.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Abstraction for a single repository. - */ -export interface Repository { - url: string; - - name: string; - - owner: string; - - description?: string; - - file(filename: string): Promise; -} - -/** - * Abstraction for a single repository file. - */ -export interface RepositoryFile { - /** - * The filepath of the data. - */ - path: string; - - /** - * The textual contents of the file. - */ - text(): Promise; -} - -/** - * One integration that supports discovery of repositories. - */ -export interface Provider { - name(): string; - discover(url: string): Promise; -} diff --git a/packages/cli/src/commands/onboard/files.ts b/packages/cli/src/commands/onboard/files.ts deleted file mode 100644 index febec35d4a..0000000000 --- a/packages/cli/src/commands/onboard/files.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { findPaths } from '@backstage/cli-common'; -import * as path from 'path'; - -/* eslint-disable-next-line no-restricted-syntax */ -const { targetRoot, ownDir } = findPaths(__dirname); - -export const APP_CONFIG_FILE = path.join(targetRoot, 'app-config.local.yaml'); -export const DISCOVERED_ENTITIES_FILE = path.join( - targetRoot, - 'examples', - 'discovered-entities.yaml', -); -export const PATCH_FOLDER = path.join( - ownDir, - 'src', - 'commands', - 'onboard', - 'auth', - 'patches', -); diff --git a/packages/cli/src/commands/onboard/integrations/github.ts b/packages/cli/src/commands/onboard/integrations/github.ts deleted file mode 100644 index a0538c7ace..0000000000 --- a/packages/cli/src/commands/onboard/integrations/github.ts +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import inquirer from 'inquirer'; -import { Task } from '../../../lib/tasks'; -import { updateConfigFile } from '../config'; -import { APP_CONFIG_FILE } from '../files'; -import { GitHubAnswers } from '../auth'; - -type Answers = { - hasEnterprise: boolean; - enterpriseInstanceUrl: string; - apiBaseUrl: string; -}; - -const getConfig = ({ - hasEnterprise, - apiBaseUrl, - host, - token, -}: { - hasEnterprise: boolean; - apiBaseUrl: string; - host: string; - token: string; -}) => ({ - integrations: { - github: [ - { - host, - token, - ...(hasEnterprise && { - apiBaseUrl, - }), - }, - ], - }, -}); - -export const github = async (providerAnswers?: GitHubAnswers) => { - // TODO(tudi2d): Is GitHub Enterprise a valid setup if there is no Authentication? - const answers = await inquirer.prompt([ - { - type: 'confirm', - name: 'hasEnterprise', - message: 'Are you using GitHub Enterprise?', - when: () => typeof providerAnswers === 'undefined', - }, - { - type: 'input', - name: 'enterpriseInstanceUrl', - message: 'What is your URL for GitHub Enterprise?', - when: ({ hasEnterprise }) => hasEnterprise, - validate: (input: string) => Boolean(new URL(input)), - }, - { - type: 'input', - name: 'apiBaseUrl', - message: 'What is your GitHub Enterprise API path?', - default: '/api/v3', - when: ({ hasEnterprise }) => - hasEnterprise || providerAnswers?.hasEnterprise, - // TODO(tudi2d): Fetch API using OAuth Token if Auth was set up - }, - ]); - - const host = new URL( - providerAnswers?.enterpriseInstanceUrl ?? - answers?.enterpriseInstanceUrl ?? - 'http://github.com', - ); - - Task.log(` - To create new repositories in GitHub using Software Templates you first need to create a personal access token: ${chalk.blue( - `${host.origin}/settings/tokens/new`, - )} - - Select the following scopes: - - Reading software components:${chalk.cyan(` - - "repo"`)} - - Reading organization data:${chalk.cyan(` - - "read:org" - - "read:user" - - "user:email"`)} - - Publishing software templates:${chalk.cyan(` - - "repo" - - "workflow" (if templates include GitHub workflows) - `)} - - You can find the full documentation page here: ${chalk.blue( - 'https://backstage.io/docs/integrations/github/locations', - )} - `); - - const { token } = await inquirer.prompt<{ token: string }>([ - { - type: 'input', - name: 'token', - message: - 'Please insert your personal access token to setup the GitHub Integration', - // TODO(tudi2d): validate - }, - ]); - - const config = getConfig({ - hasEnterprise: providerAnswers?.hasEnterprise ?? answers.hasEnterprise, - apiBaseUrl: host.origin + answers.apiBaseUrl, - host: host.hostname, - token, - }); - - Task.log('Setting up Software Templates using GitHub integration for you...'); - - await Task.forItem( - 'Updating', - APP_CONFIG_FILE, - async () => await updateConfigFile(APP_CONFIG_FILE, config), - ); -}; diff --git a/packages/cli/src/commands/onboard/integrations/index.ts b/packages/cli/src/commands/onboard/integrations/index.ts deleted file mode 100644 index 5cb2f95b38..0000000000 --- a/packages/cli/src/commands/onboard/integrations/index.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import inquirer from 'inquirer'; -import { Task } from '../../../lib/tasks'; -import { GitHubAnswers, GitLabAnswers } from '../auth'; -import { github } from './github'; - -enum Integration { - GITHUB = 'GitHub', -} - -const Integrations: Integration[] = [Integration.GITHUB]; - -export async function integrations(providerInfo?: { - provider: string; - answers: GitHubAnswers | GitLabAnswers; -}): Promise { - const answers = await inquirer.prompt<{ - integration?: Integration; - shouldUsePreviousProvider: boolean; - }>([ - { - type: 'confirm', - name: 'shouldUsePreviousProvider', - message: `Do you want to keep using ${providerInfo?.provider} as your provider when setting up Software Templates?`, - when: () => - providerInfo?.provider && - Object.values(Integrations).includes( - providerInfo!.provider as Integration, - ), - }, - { - // TODO(tudi2d): Let's start with one, but it should be multiple choice in the future - type: 'list', - name: 'integration', - message: 'Please select an integration provider:', - choices: Integrations, - when: ({ shouldUsePreviousProvider }) => !shouldUsePreviousProvider, - }, - ]); - - if (answers.shouldUsePreviousProvider) { - answers.integration = providerInfo!.provider as Integration; - } - - switch (answers.integration) { - case Integration.GITHUB: { - const providerAnswers = - providerInfo?.provider === 'GitHub' - ? (providerInfo!.answers as GitHubAnswers) - : undefined; - await github(providerAnswers); - break; - } - default: - } - - Task.log(); - Task.log(`Done setting up ${answers.integration} Integration!`); - Task.log(); -} diff --git a/packages/cli/src/commands/plugin/diff.ts b/packages/cli/src/commands/plugin/diff.ts deleted file mode 100644 index 07f2a94ea8..0000000000 --- a/packages/cli/src/commands/plugin/diff.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { OptionValues } from 'commander'; -import { - diffTemplateFiles, - handlers, - handleAllFiles, - inquirerPromptFunc, - makeCheckPromptFunc, - yesPromptFunc, -} from '../../lib/diff'; -import { paths } from '../../lib/paths'; - -export type PluginData = { - id: string; - name: string; - privatePackage: string; - pluginVersion: string; - npmRegistry: string; -}; - -const fileHandlers = [ - { - patterns: ['package.json'], - handler: handlers.packageJson, - }, - { - // make sure files in 1st level of src/ and dev/ exist - patterns: ['.eslintrc.js'], - handler: handlers.exists, - }, - { - patterns: ['README.md', 'tsconfig.json', /^src\//, /^dev\//], - handler: handlers.skip, - }, -]; - -export default async (opts: OptionValues) => { - let promptFunc = inquirerPromptFunc; - let finalize = () => {}; - - if (opts.check) { - [promptFunc, finalize] = makeCheckPromptFunc(); - } else if (opts.yes) { - promptFunc = yesPromptFunc; - } - - const data = await readPluginData(); - const templateFiles = await diffTemplateFiles('default-plugin', data); - await handleAllFiles(fileHandlers, templateFiles, promptFunc); - finalize(); -}; - -// Reads templating data from the existing plugin -async function readPluginData(): Promise { - let name: string; - let privatePackage: string; - let pluginVersion: string; - let npmRegistry: string; - try { - const pkg = require(paths.resolveTarget('package.json')); - name = pkg.name; - privatePackage = pkg.private; - pluginVersion = pkg.version; - const scope = name.split('/')[0]; - if (`${scope}:registry` in pkg.publishConfig) { - const registryURL = pkg.publishConfig[`${scope}:registry`]; - npmRegistry = `"${scope}:registry" : "${registryURL}"`; - } else npmRegistry = ''; - } catch (error) { - throw new Error(`Failed to read target package, ${error}`); - } - - const pluginTsContents = await fs.readFile( - paths.resolveTarget('src/plugin.ts'), - 'utf8', - ); - // TODO: replace with some proper parsing logic or plugin metadata file - const pluginIdMatch = pluginTsContents.match(/id: ['"`](.+?)['"`]/); - if (!pluginIdMatch) { - throw new Error(`Failed to parse plugin.ts, no plugin ID found`); - } - - const id = pluginIdMatch[1]; - - return { id, name, privatePackage, pluginVersion, npmRegistry }; -} diff --git a/packages/cli/src/commands/remove-plugin/file-mocks.ts b/packages/cli/src/commands/remove-plugin/file-mocks.ts deleted file mode 100644 index 1264dc8f2e..0000000000 --- a/packages/cli/src/commands/remove-plugin/file-mocks.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export const pluginsFileContent = ` -export { plugin as WelcomePlugin } from '@backstage/plugin-welcome'; -export { plugin as LighthousePlugin } from '@backstage/plugin-lighthouse';`; - -export const codeownersFileContent = ` -* @spotify/backstage-core -/docs/features/techdocs @spotify/techdocs-core -/plugins/cost-insights @spotify/silver-lining -`; - -export const packageFileContent = { - name: 'example-app', - version: '0.1.1', - dependencies: {}, - devDependencies: {}, - scripts: {}, -}; diff --git a/packages/cli/src/lib/diff/handlers.ts b/packages/cli/src/lib/diff/handlers.ts deleted file mode 100644 index 55e9cddd3d..0000000000 --- a/packages/cli/src/lib/diff/handlers.ts +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import { diffLines } from 'diff'; -import { sep, posix } from 'path'; -import { FileDiff, PromptFunc, FileHandler, WriteFileFunc } from './types'; - -function sortObjectKeys(obj: Record) { - const sortedKeys = Object.keys(obj).sort(); - for (const key of sortedKeys) { - const value = obj[key]; - delete obj[key]; - obj[key] = value; - } -} - -class PackageJsonHandler { - static async handler( - { path, write, missing, targetContents, templateContents }: FileDiff, - prompt: PromptFunc, - variant?: string, - ) { - console.log('Checking package.json'); - - if (missing) { - throw new Error(`${path} doesn't exist`); - } - - const pkg = JSON.parse(templateContents); - const targetPkg = JSON.parse(targetContents); - - const handler = new PackageJsonHandler( - write, - prompt, - pkg, - targetPkg, - variant, - ); - await handler.handle(); - } - - static async appHandler(file: FileDiff, prompt: PromptFunc) { - return PackageJsonHandler.handler(file, prompt, 'app'); - } - - constructor( - private readonly writeFunc: WriteFileFunc, - private readonly prompt: PromptFunc, - private readonly pkg: any, - private readonly targetPkg: any, - private readonly variant?: string, - ) {} - - async handle() { - await this.syncField('main'); - if (this.variant !== 'app') { - await this.syncField('main:src'); - } - await this.syncField('types'); - await this.syncFiles(); - await this.syncScripts(); - await this.syncPublishConfig(); - await this.syncDependencies('dependencies'); - await this.syncDependencies('peerDependencies', true); - await this.syncDependencies('devDependencies'); - await this.syncReactDeps(); - } - - // Make sure a field inside package.json is in sync. This mutates the targetObj and writes package.json on change. - private async syncField( - fieldName: string, - obj: any = this.pkg, - targetObj: any = this.targetPkg, - prefix?: string, - sort?: boolean, - optional?: boolean, - ) { - const fullFieldName = chalk.cyan( - prefix ? `${prefix}[${fieldName}]` : fieldName, - ); - const newValue = obj[fieldName]; - const coloredNewValue = chalk.cyan(JSON.stringify(newValue)); - - if (fieldName in targetObj) { - const oldValue = targetObj[fieldName]; - if (JSON.stringify(oldValue) === JSON.stringify(newValue)) { - return; - } - - const coloredOldValue = chalk.cyan(JSON.stringify(oldValue)); - const msg = `package.json has mismatched field, ${fullFieldName}, change from ${coloredOldValue} to ${coloredNewValue}?`; - if (await this.prompt(msg)) { - targetObj[fieldName] = newValue; - if (sort) { - sortObjectKeys(targetObj); - } - await this.write(); - } - } else if (fieldName in obj && optional !== true) { - if ( - await this.prompt( - `package.json is missing field ${fullFieldName}, set to ${coloredNewValue}?`, - ) - ) { - targetObj[fieldName] = newValue; - if (sort) { - sortObjectKeys(targetObj); - } - await this.write(); - } - } - } - - private async syncFiles() { - const { configSchema } = this.targetPkg; - const hasSchemaFile = typeof configSchema === 'string'; - - if (!this.targetPkg.files) { - const expected = hasSchemaFile ? ['dist', configSchema] : ['dist']; - if ( - await this.prompt( - `package.json is missing field "files", set to ${JSON.stringify( - expected, - )}?`, - ) - ) { - this.targetPkg.files = expected; - await this.write(); - } - } else { - const missing = []; - if (!this.targetPkg.files.includes('dist')) { - missing.push('dist'); - } - if (hasSchemaFile && !this.targetPkg.files.includes(configSchema)) { - missing.push(configSchema); - } - if (missing.length) { - if ( - await this.prompt( - `package.json is missing ${JSON.stringify( - missing, - )} in the "files" field, add?`, - ) - ) { - this.targetPkg.files.push(...missing); - await this.write(); - } - } - } - } - - private async syncScripts() { - const pkgScripts = this.pkg.scripts; - const targetScripts = (this.targetPkg.scripts = - this.targetPkg.scripts || {}); - - if (!pkgScripts) { - return; - } - - // Skip diffing package scripts that have been migrated to the new commands - const hasNewScript = Object.values(targetScripts).some(script => - String(script).includes('backstage-cli package '), - ); - if (hasNewScript) { - return; - } - - for (const key of Object.keys(pkgScripts)) { - await this.syncField(key, pkgScripts, targetScripts, 'scripts'); - } - } - - private async syncPublishConfig() { - const pkgPublishConf = this.pkg.publishConfig; - const targetPublishConf = this.targetPkg.publishConfig; - - // If template doesn't have a publish config we're done - if (!pkgPublishConf) { - return; - } - - // Publish config can be removed the the target, skip in that case - if (!targetPublishConf) { - if (await this.prompt('Missing publishConfig, do you want to add it?')) { - this.targetPkg.publishConfig = pkgPublishConf; - await this.write(); - } - return; - } - - for (const key of Object.keys(pkgPublishConf)) { - // Don't want to mess with peoples internal setup - if (!['access', 'registry'].includes(key)) { - await this.syncField( - key, - pkgPublishConf, - targetPublishConf, - 'publishConfig', - ); - } - } - } - - private async syncDependencies(fieldName: string, required: boolean = false) { - const pkgDeps = this.pkg[fieldName]; - const targetDeps = (this.targetPkg[fieldName] = - this.targetPkg[fieldName] || {}); - - if (!pkgDeps && !required) { - return; - } - - // Hardcoded removal of these during migration - await this.syncField('@backstage/core', {}, targetDeps, fieldName, true); - await this.syncField( - '@backstage/core-api', - {}, - targetDeps, - fieldName, - true, - ); - - for (const key of Object.keys(pkgDeps)) { - if (this.variant === 'app' && key.startsWith('plugin-')) { - continue; - } - - await this.syncField( - key, - pkgDeps, - targetDeps, - fieldName, - true, - !required, - ); - } - } - - private async syncReactDeps() { - const targetDeps = (this.targetPkg.dependencies = - this.targetPkg.dependencies || {}); - - // Remove these from from deps since they're now in peerDeps - await this.syncField('react', {}, targetDeps, 'dependencies'); - await this.syncField('react-dom', {}, targetDeps, 'dependencies'); - } - - private async write() { - await this.writeFunc(`${JSON.stringify(this.targetPkg, null, 2)}\n`); - } -} - -// Make sure the file is an exact match of the template -async function exactMatchHandler( - { path, write, missing, targetContents, templateContents }: FileDiff, - prompt: PromptFunc, -) { - console.log(`Checking ${path}`); - const coloredPath = chalk.cyan(path); - - if (missing) { - if (await prompt(`Missing ${coloredPath}, do you want to add it?`)) { - await write(templateContents); - } - return; - } - if (targetContents === templateContents) { - return; - } - - const diffs = diffLines(targetContents, templateContents); - for (const diff of diffs) { - if (diff.added) { - process.stdout.write(chalk.green(`+${diff.value}`)); - } else if (diff.removed) { - process.stdout.write(chalk.red(`-${diff.value}`)); - } else { - process.stdout.write(` ${diff.value}`); - } - } - - if ( - await prompt( - `Outdated ${coloredPath}, do you want to apply the above patch?`, - ) - ) { - await write(templateContents); - } -} - -// Adds the file if it is missing, but doesn't check existing files -async function existsHandler( - { path, write, missing, templateContents }: FileDiff, - prompt: PromptFunc, -) { - console.log(`Making sure ${path} exists`); - - const coloredPath = chalk.cyan(path); - - if (missing) { - if (await prompt(`Missing ${coloredPath}, do you want to add it?`)) { - await write(templateContents); - } - return; - } -} - -async function skipHandler({ path }: FileDiff) { - console.log(`Skipping ${path}`); -} - -export const handlers = { - skip: skipHandler, - exists: existsHandler, - exactMatch: exactMatchHandler, - packageJson: PackageJsonHandler.handler, - appPackageJson: PackageJsonHandler.appHandler, -}; - -export async function handleAllFiles( - fileHandlers: FileHandler[], - files: FileDiff[], - promptFunc: PromptFunc, -) { - for (const file of files) { - const path = file.path.split(sep).join(posix.sep); - const fileHandler = fileHandlers.find(handler => - handler.patterns.some(pattern => - typeof pattern === 'string' ? pattern === path : pattern.test(path), - ), - ); - if (fileHandler) { - await fileHandler.handler(file, promptFunc); - } else { - throw new Error(`No template file handler found for ${path}`); - } - } -} diff --git a/packages/cli/src/lib/diff/prompts.ts b/packages/cli/src/lib/diff/prompts.ts deleted file mode 100644 index d2ddca5fc0..0000000000 --- a/packages/cli/src/lib/diff/prompts.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import chalk from 'chalk'; -import inquirer from 'inquirer'; -import { PromptFunc } from './types'; - -export const inquirerPromptFunc: PromptFunc = async msg => { - const { result } = await inquirer.prompt({ - type: 'confirm', - name: 'result', - message: chalk.blue(msg), - }); - return result; -}; - -export const makeCheckPromptFunc = () => { - let failed = false; - - const promptFunc: PromptFunc = async msg => { - failed = true; - console.log(chalk.red(`[Check Failed] ${msg}`)); - return false; - }; - - const finalize = () => { - if (failed) { - throw new Error( - 'Check failed, the plugin is not in sync with the latest template', - ); - } - }; - - return [promptFunc, finalize] as const; -}; - -export const yesPromptFunc: PromptFunc = async msg => { - console.log(`Accepting: "${msg}"`); - return true; -}; diff --git a/packages/cli/src/lib/diff/read.ts b/packages/cli/src/lib/diff/read.ts deleted file mode 100644 index 95b0bed43d..0000000000 --- a/packages/cli/src/lib/diff/read.ts +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { - dirname, - resolve as resolvePath, - relative as relativePath, -} from 'path'; -import handlebars from 'handlebars'; -import recursiveReadDir from 'recursive-readdir'; -import { paths } from '../paths'; -import { FileDiff } from './types'; -import { createPackageVersionProvider } from '../../lib/version'; - -export type TemplatedFile = { - path: string; - contents: string; -}; - -async function readTemplateFile( - templateFile: string, - templateVars: any, -): Promise { - const contents = await fs.readFile(templateFile, 'utf8'); - - if (!templateFile.endsWith('.hbs')) { - return contents; - } - const packageVersionProvider = createPackageVersionProvider(undefined); - - return handlebars.compile(contents)(templateVars, { - helpers: { - versionQuery(name: string, hint: string | unknown) { - return packageVersionProvider( - name, - typeof hint === 'string' ? hint : undefined, - ); - }, - }, - }); -} - -async function readTemplate( - templateDir: string, - templateVars: any, -): Promise { - const templateFilePaths = await recursiveReadDir(templateDir).catch(error => { - throw new Error(`Failed to read template directory: ${error.message}`); - }); - - const templatedFiles = new Array(); - for (const templateFile of templateFilePaths) { - const path = relativePath(templateDir, templateFile).replace(/\.hbs$/, ''); - const contents = await readTemplateFile(templateFile, templateVars); - - templatedFiles.push({ path, contents }); - } - - return templatedFiles; -} - -async function diffTemplatedFiles( - targetDir: string, - templatedFiles: TemplatedFile[], -): Promise { - const fileDiffs = new Array(); - for (const { path, contents: templateContents } of templatedFiles) { - const targetPath = resolvePath(targetDir, path); - const targetExists = await fs.pathExists(targetPath); - - const write = async (contents: string) => { - await fs.ensureDir(dirname(targetPath)); - await fs.writeFile(targetPath, contents, 'utf8'); - }; - - if (targetExists) { - const targetContents = await fs.readFile(targetPath, 'utf8'); - fileDiffs.push({ - path, - write, - missing: false, - targetContents, - templateContents, - }); - } else { - fileDiffs.push({ - path, - write, - missing: true, - targetContents: '', - templateContents, - }); - } - } - - return fileDiffs; -} - -// Read all template files for a given template, along with all matching files in the target dir -export async function diffTemplateFiles(template: string, templateData: any) { - const templateDir = paths.resolveOwn('templates', template); - - const templatedFiles = await readTemplate(templateDir, templateData); - const fileDiffs = await diffTemplatedFiles(paths.targetDir, templatedFiles); - return fileDiffs; -} diff --git a/packages/cli/src/lib/diff/types.ts b/packages/cli/src/lib/diff/types.ts deleted file mode 100644 index ba649f66ff..0000000000 --- a/packages/cli/src/lib/diff/types.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type WriteFileFunc = (contents: string) => Promise; - -export type FileDiff = { - // Relative path within the target directory - path: string; - // Whether the target file exists in the target directory. - missing: boolean; - // Contents of the file in the target directory, or an empty string if the file is missing. - targetContents: string; - // Contents of the compiled template file - templateContents: string; - // Write new contents to the target file - write: WriteFileFunc; -}; - -export type PromptFunc = (msg: string) => Promise; - -export type HandlerFunc = (file: FileDiff, prompt: PromptFunc) => Promise; - -export type FileHandler = { - patterns: Array; - handler: HandlerFunc; -}; diff --git a/packages/cli/src/lib/logging.ts b/packages/cli/src/lib/logging.ts deleted file mode 100644 index 59473d7f64..0000000000 --- a/packages/cli/src/lib/logging.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type LogFunc = (data: Buffer) => void; -export type LogPipe = (dst: NodeJS.WriteStream) => LogFunc; - -export type LogOptions = { - // If set, prefix each log message with this string - prefix?: string; - - // If true, clear terminal commands will be forwarded, otherwise they are removed - forwardClearTerm?: boolean; -}; - -// Creates a log pipe that binds to a destination stream and forwards logs with optional transforms. -// Use returned logPipe e.g. as follows: child.stdout.on('data', logPipe(process.stdout)) -export function createLogPipe(options: LogOptions = {}): LogPipe { - const { prefix = '', forwardClearTerm = false } = options; - - return (dst: NodeJS.WriteStream) => (data: Buffer | string) => { - let str = typeof data === 'string' ? data : data.toString('utf8'); - - if (!forwardClearTerm) { - str = trimClearTerm(str); - } - if (prefix) { - str = `${prefix}${str}`; - } - - dst.write(Buffer.from(str, 'utf8')); - }; -} - -// Wrapper around createLogPipe to avoid awkward immediate call of returned function -export function createLogFunc( - dst: NodeJS.WriteStream, - options: LogOptions = {}, -): LogFunc { - return createLogPipe(options)(dst); -} - -// Returns the string without terminal clear command if it was prefixed with one. -export function trimClearTerm(msg: string): string { - return msg.startsWith('\x1b\x63') ? msg.slice(2) : msg; -} diff --git a/packages/cli/src/lib/run.ts b/packages/cli/src/lib/run.ts index 1ef25cae62..e8602fcb18 100644 --- a/packages/cli/src/lib/run.ts +++ b/packages/cli/src/lib/run.ts @@ -22,11 +22,12 @@ import { } from 'child_process'; import { ExitCodeError } from './errors'; import { promisify } from 'util'; -import { LogFunc } from './logging'; import { assertError, ForwardedError } from '@backstage/errors'; export const execFile = promisify(execFileCb); +type LogFunc = (data: Buffer) => void; + type SpawnOptionsPartialEnv = Omit & { env?: Partial; // Pipe stdout to this log function diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts new file mode 100644 index 0000000000..d75834884e --- /dev/null +++ b/plugins/events-backend/dev/index.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { WinstonLogger } from '@backstage/backend-defaults/rootLogger'; +import { createBackend } from '@backstage/backend-defaults'; +import { + coreServices, + createBackendPlugin, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { mockServices } from '@backstage/backend-test-utils'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; + +function makeBackend(port: number) { + const backend = createBackend(); + + backend.add( + mockServices.rootConfig.factory({ + data: { + backend: { + baseUrl: `http://localhost:${port}`, + listen: { port }, + database: { + client: 'pg', + connection: { + host: 'localhost', + port: 5432, + user: process.env.USER || 'postgres', + }, + }, + }, + }, + }), + ); + backend.add( + createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + async factory() { + return WinstonLogger.create({ + meta: { + service: 'backstage', + port, + }, + // level: 'debug', + format: WinstonLogger.colorFormat(), + }); + }, + }), + ); + backend.add(import('../src/alpha')); + + return backend; +} + +const backend7008 = makeBackend(7008); +backend7008.add( + createBackendPlugin({ + pluginId: 'producer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + rootLifecycle: coreServices.rootLifecycle, + }, + async init({ events, logger, rootLifecycle }) { + rootLifecycle.addStartupHook(async () => { + const publish = () => { + logger.info(`Publishing event to topic 'test'`); + events + .publish({ + eventPayload: { foo: 'bar' }, + topic: 'test', + }) + .catch(error => { + logger.error(`Failed to publish event from producer`, error); + }); + }; + publish(); + setInterval(publish, 10000); + }); + }, + }); + }, + }), +); +backend7008.start(); + +const backend7009 = makeBackend(7009); +backend7009.add( + createBackendPlugin({ + pluginId: 'consumer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + }, + async init({ events, logger }) { + await events.subscribe({ + id: 'test', + topics: ['test'], + async onEvent(event) { + logger.info(`Received event: ${JSON.stringify(event)}`); + }, + }); + }, + }); + }, + }), +); +backend7009.start(); + +const backend7010 = makeBackend(7010); +backend7010.add( + createBackendPlugin({ + pluginId: 'consumer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + }, + async init({ events, logger }) { + await events.subscribe({ + id: 'test', + topics: ['test'], + async onEvent(event) { + logger.info(`Received event: ${JSON.stringify(event)}`); + }, + }); + }, + }); + }, + }), +); +backend7010.start(); diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js new file mode 100644 index 0000000000..9fbef79a75 --- /dev/null +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -0,0 +1,98 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function up(knex) { + // The event bus only supports PostgresSQL + if (knex.client.config.client === 'pg') { + await knex.schema.createTable('event_bus_events', table => { + table.comment('Events published to the events bus'); + table + .bigIncrements('id') + .primary() + .comment('The unique ID of this event'); + table + .text('created_by') + .notNullable() + .comment('The principal that published the event'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the event was created'); + table.text('topic').notNullable().comment('The topic of the event'); + table + .text('data_json') + .notNullable() + .comment('The payload data of this event'); + table + .specificType('notified_subscribers', 'text ARRAY') + .comment( + 'The IDs of the subscribers that have already consumed this event', + ); + + table.index('topic', 'event_bus_events_topic_idx'); + }); + + await knex.schema.createTable('event_bus_subscriptions', table => { + table.comment('Subscriptions to the event bus'); + table + .string('id') + .primary() + .notNullable() + .comment('The unique ID of this particular subscription'); + table + .text('created_by') + .notNullable() + .comment('The principal that created the subscription'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the subscription was created'); + table + .dateTime('updated_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the subscription was last updated'); + table + .bigInteger('read_until') + .notNullable() + .comment( + 'The sequence counter until which the subscription has read events', + ); + table + .specificType('topics', 'text ARRAY') + .comment('The topics that this subscriber is interested in'); + }); + } +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + if (knex.client.config.client === 'pg') { + await knex.schema.dropTable('event_bus_subscriptions'); + await knex.schema.dropTable('event_bus_events'); + } +}; diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index 34f64d05e0..06addb0546 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -39,11 +39,13 @@ }, "files": [ "config.d.ts", - "dist" + "dist", + "migrations" ], "scripts": { "build": "backstage-cli package build", "clean": "backstage-cli package clean", + "generate": "backstage-repo-tools package schema openapi generate --server --client-package plugins/events-node", "lint": "backstage-cli package lint", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", @@ -52,18 +54,25 @@ }, "dependencies": { "@backstage/backend-common": "^0.25.0", + "@backstage/backend-openapi-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/plugin-events-node": "workspace:^", + "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "knex": "^3.0.0", "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-app-api": "workspace:^", + "@backstage/backend-defaults": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@backstage/plugin-events-backend-test-utils": "workspace:^", + "@backstage/repo-tools": "workspace:^", "supertest": "^7.0.0" }, "configSchema": "config.d.ts" diff --git a/plugins/events-backend/src/migrations.test.ts b/plugins/events-backend/src/migrations.test.ts new file mode 100644 index 0000000000..1c90260ed3 --- /dev/null +++ b/plugins/events-backend/src/migrations.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import fs from 'fs'; + +const migrationsDir = `${__dirname}/../migrations`; +const migrationsFiles = fs.readdirSync(migrationsDir).sort(); + +async function migrateUpOnce(knex: Knex): Promise { + await knex.migrate.up({ directory: migrationsDir }); +} + +async function migrateDownOnce(knex: Knex): Promise { + await knex.migrate.down({ directory: migrationsDir }); +} + +async function migrateUntilBefore(knex: Knex, target: string): Promise { + const index = migrationsFiles.indexOf(target); + if (index === -1) { + throw new Error(`Migration ${target} not found`); + } + for (let i = 0; i < index; i++) { + await migrateUpOnce(knex); + } +} + +jest.setTimeout(60_000); + +describe('migrations', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], + }); + + it.each(databases.eachSupportedId())( + '20240523100528_init.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore(knex, '20240523100528_init.js'); + await migrateUpOnce(knex); + + await knex('event_bus_events').insert({ + topic: 'test', + created_by: 'abc', + data_json: JSON.stringify({ message: 'hello' }), + notified_subscribers: ['tester'], + }); + await knex('event_bus_subscriptions').insert({ + id: 'tester', + created_by: 'abc', + read_until: '5', + topics: ['test', 'test2'], + }); + + await expect(knex('event_bus_events')).resolves.toEqual([ + { + id: '1', + created_by: 'abc', + topic: 'test', + data_json: JSON.stringify({ message: 'hello' }), + created_at: expect.anything(), + notified_subscribers: ['tester'], + }, + ]); + await expect(knex('event_bus_subscriptions')).resolves.toEqual([ + { + id: 'tester', + created_by: 'abc', + created_at: expect.anything(), + updated_at: expect.anything(), + read_until: '5', + topics: ['test', 'test2'], + }, + ]); + + await migrateDownOnce(knex); + + // This looks odd - you might expect a .toThrow at the end but that + // actually is flaky for some reason specifically on sqlite when + // performing multiple runs in sequence + await expect(knex('event_bus_events')).rejects.toEqual(expect.anything()); + + await knex.destroy(); + }, + ); +}); diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts new file mode 100644 index 0000000000..8f8d6c6a82 --- /dev/null +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -0,0 +1,291 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { createValidatedOpenApiRouter } from '@backstage/backend-openapi-utils'; + +export const spec = { + openapi: '3.0.3', + info: { + title: 'events', + version: '1', + description: 'The Backstage backend plugin that powers the events system.', + license: { + name: 'Apache-2.0', + url: 'http://www.apache.org/licenses/LICENSE-2.0.html', + }, + contact: {}, + }, + servers: [ + { + url: '/', + }, + ], + components: { + examples: {}, + headers: {}, + parameters: { + subscriptionId: { + name: 'subscriptionId', + in: 'path', + required: true, + allowReserved: true, + schema: { + type: 'string', + }, + }, + }, + requestBodies: {}, + responses: { + ErrorResponse: { + description: 'An error response from the backend.', + content: { + 'application/json; charset=utf-8': { + schema: { + $ref: '#/components/schemas/Error', + }, + }, + }, + }, + }, + schemas: { + Event: { + type: 'object', + required: ['topic', 'payload'], + properties: { + topic: { + type: 'string', + description: 'The topic that the event is published on', + }, + payload: { + description: 'The event payload', + }, + }, + }, + Error: { + type: 'object', + properties: { + error: { + type: 'object', + properties: { + name: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + required: ['name', 'message'], + }, + request: { + type: 'object', + properties: { + method: { + type: 'string', + }, + url: { + type: 'string', + }, + }, + required: ['method', 'url'], + }, + response: { + type: 'object', + properties: { + statusCode: { + type: 'number', + }, + }, + required: ['statusCode'], + }, + }, + required: ['error', 'request', 'response'], + }, + }, + securitySchemes: { + JWT: { + type: 'http', + scheme: 'bearer', + bearerFormat: 'JWT', + }, + }, + }, + paths: { + '/bus/v1/events': { + post: { + operationId: 'PostEvent', + description: 'Publish a new event', + responses: { + '201': { + description: 'The event was published successfully', + }, + '204': { + description: + 'The event did not need to be published as all subscribers have already been notified', + }, + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['event'], + properties: { + event: { + $ref: '#/components/schemas/Event', + }, + notifiedSubscribers: { + type: 'array', + description: + 'The IDs of subscriptions that have already received this event', + items: { + type: 'string', + }, + }, + }, + }, + examples: { + 'Publishing a simple Event': { + value: { + event: { + topic: 'test-topic', + payload: { + myData: 'foo', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + '/bus/v1/subscriptions/{subscriptionId}': { + put: { + operationId: 'PutSubscription', + description: + 'Ensures that the subscription exists with the provided configuration', + responses: { + '201': { + description: 'The subscription exists or was created successfully', + }, + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + parameters: [ + { + $ref: '#/components/parameters/subscriptionId', + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['topics'], + properties: { + topics: { + type: 'array', + description: 'The topics to subscribe to', + items: { + type: 'string', + }, + }, + }, + }, + examples: { + 'Subscribing to a single topic': { + value: { + topics: ['test-topic'], + }, + }, + }, + }, + }, + }, + }, + }, + '/bus/v1/subscriptions/{subscriptionId}/events': { + get: { + operationId: 'GetSubscriptionEvents', + description: 'Get new events for the provided subscription', + responses: { + '200': { + description: 'New events', + content: { + 'application/json': { + schema: { + type: 'object', + required: ['events'], + properties: { + events: { + type: 'array', + items: { + $ref: '#/components/schemas/Event', + }, + }, + }, + }, + }, + }, + }, + '202': { + description: + 'No new events are available. Response will block until the client should try again.', + }, + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + parameters: [ + { + $ref: '#/components/parameters/subscriptionId', + }, + ], + }, + }, + }, +} as const; +export const createOpenApiRouter = async ( + options?: Parameters['1'], +) => createValidatedOpenApiRouter(spec, options); diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml new file mode 100644 index 0000000000..97f7705f87 --- /dev/null +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -0,0 +1,182 @@ +openapi: 3.0.3 +info: + title: events + version: '1' + description: The Backstage backend plugin that powers the events system. + license: + name: Apache-2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + contact: {} +servers: + - url: / +components: + examples: {} + headers: {} + parameters: + subscriptionId: + name: subscriptionId + in: path + required: true + allowReserved: true + schema: + type: string + requestBodies: {} + responses: + ErrorResponse: + description: An error response from the backend. + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/Error' + schemas: + Event: + type: object + required: + - topic + - payload + properties: + topic: + type: string + description: The topic that the event is published on + payload: + description: The event payload + + Error: + type: object + properties: + error: + type: object + properties: + name: + type: string + message: + type: string + required: + - name + - message + request: + type: object + properties: + method: + type: string + url: + type: string + required: + - method + - url + response: + type: object + properties: + statusCode: + type: number + required: + - statusCode + required: + - error + - request + - response + securitySchemes: + JWT: + type: http + scheme: bearer + bearerFormat: JWT +paths: + /bus/v1/events: + post: + operationId: PostEvent + description: Publish a new event + responses: + '201': + description: The event was published successfully + '204': + description: The event did not need to be published as all subscribers have already been notified + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - event + properties: + event: + $ref: '#/components/schemas/Event' + notifiedSubscribers: + type: array + description: The IDs of subscriptions that have already received this event + items: + type: string + examples: + Publishing a simple Event: + value: + event: + topic: test-topic + payload: + myData: foo + + /bus/v1/subscriptions/{subscriptionId}: + put: + operationId: PutSubscription + description: Ensures that the subscription exists with the provided configuration + responses: + '201': + description: The subscription exists or was created successfully + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + parameters: + - $ref: '#/components/parameters/subscriptionId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - topics + properties: + topics: + type: array + description: The topics to subscribe to + items: + type: string + examples: + Subscribing to a single topic: + value: + topics: + - test-topic + + /bus/v1/subscriptions/{subscriptionId}/events: + get: + operationId: GetSubscriptionEvents + description: Get new events for the provided subscription + responses: + '200': + description: New events + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: '#/components/schemas/Event' + '202': + description: No new events are available. Response will block until the client should try again. + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + parameters: + - $ref: '#/components/parameters/subscriptionId' diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 48ce1090fb..2c0befe1fd 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -14,11 +14,20 @@ * limitations under the License. */ +/* eslint-disable jest/expect-expect */ + import { createBackendModule, createServiceFactory, } from '@backstage/backend-plugin-api'; -import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { + TestBackend, + TestDatabaseId, + TestDatabases, + mockCredentials, + mockServices, + startTestBackend, +} from '@backstage/backend-test-utils'; import { eventsServiceRef } from '@backstage/plugin-events-node'; import { eventsExtensionPoint } from '@backstage/plugin-events-node/alpha'; import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; @@ -92,4 +101,334 @@ describe('eventsPlugin', () => { test: 'fake-ext', }); }); + + describe('event bus', () => { + class ReqHelper { + constructor(private readonly backend: TestBackend) {} + + subscribe(id: string, topics: string[], options?: { auth?: string }) { + return request(this.backend.server) + .put(`/api/events/bus/v1/subscriptions/${id}`) + .set( + 'authorization', + options?.auth ?? mockCredentials.service.header(), + ) + .send({ topics }); + } + + publish( + topic: string, + payload: unknown, + options?: { notifiedSubscribers?: string[] }, + ) { + return request(this.backend.server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ + event: { topic, payload }, + notifiedSubscribers: options?.notifiedSubscribers, + }); + } + + readEvents(id: string) { + return request(this.backend.server) + .get(`/api/events/bus/v1/subscriptions/${id}/events`) + .set('authorization', mockCredentials.service.header()) + .send(); + } + } + + const databases = TestDatabases.create({ + ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], + }); + + async function mockKnexFactory(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + return mockServices.database.mock({ + getClient: async () => knex, + }).factory; + } + + let backend: TestBackend | undefined = undefined; + afterEach(async () => { + if (backend) { + await backend.stop(); + backend = undefined; + } + }); + + it.each(databases.eachSupportedId())( + 'should be possible to publish events as a service, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.none.header()) + .expect(401); + + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.user.header()) + .expect(403); + + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.service.header()) + .expect(204); // 204, since there are no subscribers + }, + ); + + it.each(databases.eachSupportedId())( + 'should be possible to subscribe as a service and receive an event, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper + .subscribe('tester', ['test']) + .set('authorization', mockCredentials.none.header()) + .expect(401); + + await helper + .subscribe('tester', ['test']) + .set('authorization', mockCredentials.user.header()) + .expect(403); + + await helper.subscribe('tester', ['test']).expect(201); + + await helper + .readEvents('tester') + .set('authorization', mockCredentials.none.header()) + .expect(401); + + await helper + .readEvents('tester') + .set('authorization', mockCredentials.user.header()) + .expect(403); + + await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber + + await helper.readEvents('tester').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'should only send an event for each subscriber once, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); + + // A single event + await helper.publish('test', { n: 1 }).expect(201); + + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + // Two clients for subscriber 2, only one gets the event + const res1 = helper.readEvents('tester-2'); + const res2 = helper.readEvents('tester-2'); + + const res = await Promise.race([res1, res2]); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + // Post another event, which triggers the other client to return + await helper.publish('test', { n: 2 }).expect(201); + + const otherRes = await Promise.all([res1, res2]).then(rs => + rs.find(r => r !== res), + ); + expect(otherRes?.status).toBe(202); + + // Reading subscriber 2 should now return the second event only + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { n: 2 } }], + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'should not notify subscribers that have already consumed the event, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); + + // A single event for each subscriber, that should not be sent to the other one + await helper + .publish( + 'test', + { for: 'tester-2' }, + { + notifiedSubscribers: ['tester-1'], + }, + ) + .expect(201); + await helper + .publish( + 'test', + { for: 'tester-1' }, + { + notifiedSubscribers: ['tester-2'], + }, + ) + .expect(201); + + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-1' } }], + }); + // Single client for subscriber 2 gets the event + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-2' } }], + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'should return multiple events in order, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester', ['test']).expect(201); + + // A sequence of events published one at a time + for (let n = 0; n < 15; ++n) { + await helper.publish('test', { n }).expect(201); + } + + // Batch size it 10 + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 0 } }, + { topic: 'test', payload: { n: 1 } }, + { topic: 'test', payload: { n: 2 } }, + { topic: 'test', payload: { n: 3 } }, + { topic: 'test', payload: { n: 4 } }, + { topic: 'test', payload: { n: 5 } }, + { topic: 'test', payload: { n: 6 } }, + { topic: 'test', payload: { n: 7 } }, + { topic: 'test', payload: { n: 8 } }, + { topic: 'test', payload: { n: 9 } }, + ], + }); + + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 10 } }, + { topic: 'test', payload: { n: 11 } }, + { topic: 'test', payload: { n: 12 } }, + { topic: 'test', payload: { n: 13 } }, + { topic: 'test', payload: { n: 14 } }, + ], + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'should skip publishing if all subscribers have already consumed the event, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper.subscribe('tester', ['test']).expect(201); + + await helper + .publish( + 'test', + { for: 'tester-2' }, + { notifiedSubscribers: ['tester'] }, + ) + .expect(204); + }, + ); + + it.each(databases.eachSupportedId())( + 'should time out when no events are available, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + await helper.subscribe('tester', ['test']).expect(201); + + jest.useFakeTimers({ + advanceTimers: true, + }); + + try { + // Can't use supertest for this one because it can't handle the partially blocking response + const res = await fetch( + `http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`, + { + headers: { + authorization: mockCredentials.service.header(), + }, + }, + ); + + expect(res.status).toBe(202); + + const reader = res.body!.getReader(); + const isClosed = () => + Promise.race([ + reader + .read() + .then(() => reader.closed) + .then(() => true), + new Promise(r => setTimeout(r, 100, false)), + ]); + + await expect(isClosed()).resolves.toBe(false); + await jest.advanceTimersByTimeAsync(30000); + await expect(isClosed()).resolves.toBe(false); + await jest.advanceTimersByTimeAsync(30000); + await expect(isClosed()).resolves.toBe(true); + } finally { + jest.useRealTimers(); + } + }, + ); + + it.each(databases.eachSupportedId())( + 'should refuse listen without a subscription, %p', + async databaseId => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper.readEvents('nonexistent').expect(404); + }, + ); + }); }); diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index d95dbf3efc..1369ee98ca 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -28,6 +28,7 @@ import { } from '@backstage/plugin-events-node'; import Router from 'express-promise-router'; import { HttpPostIngressEventPublisher } from './http'; +import { createEventBusRouter } from './hub'; class EventsExtensionPointImpl implements EventsExtensionPoint { #httpPostIngresses: HttpPostIngressOptions[] = []; @@ -74,10 +75,23 @@ export const eventsPlugin = createBackendPlugin({ deps: { config: coreServices.rootConfig, events: eventsServiceRef, + database: coreServices.database, logger: coreServices.logger, + scheduler: coreServices.scheduler, + lifecycle: coreServices.lifecycle, + httpAuth: coreServices.httpAuth, router: coreServices.httpRouter, }, - async init({ config, events, logger, router }) { + async init({ + config, + events, + database, + logger, + scheduler, + lifecycle, + httpAuth, + router, + }) { const ingresses = Object.fromEntries( extensionPoint.httpPostIngresses.map(ingress => [ ingress.topic, @@ -93,6 +107,17 @@ export const eventsPlugin = createBackendPlugin({ }); const eventsRouter = Router(); http.bind(eventsRouter); + + router.use( + await createEventBusRouter({ + database, + logger, + httpAuth, + scheduler, + lifecycle, + }), + ); + router.use(eventsRouter); router.addAuthPolicy({ allow: 'unauthenticated', diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts new file mode 100644 index 0000000000..7086d90cbf --- /dev/null +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TestDatabases, + mockCredentials, + mockServices, +} from '@backstage/backend-test-utils'; +import { DatabaseEventBusStore } from './DatabaseEventBusStore'; + +const logger = mockServices.logger.mock(); + +const databases = TestDatabases.create({ + ids: ['POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], +}); + +describe('DatabaseEventBusStore', () => { + it.each(databases.eachSupportedId())( + 'should clean up old events, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ logger, db }); + + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); + await store.upsertSubscription( + 'tester-2', + ['test'], + mockCredentials.service(), + ); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), + }); + } + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + + await store.clean(); + + await expect(store.readSubscription('tester-2')).rejects.toThrow( + "Subscription with ID 'tester-2' not found", + ); + + await store.upsertSubscription( + 'tester-3', + ['test'], + mockCredentials.service(), + ); + + // Reset read pointer to read form the beginning + await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ + read_until: 0, + }); + + const { events: events3 } = await store.readSubscription('tester-3'); + expect(events3.length).toBe(5); + }, + ); + + it.each(databases.eachSupportedId())( + 'should always clean up events outside the max age window, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + maxAge: 0, + }); + + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); + await store.upsertSubscription( + 'tester-2', + ['test'], + mockCredentials.service(), + ); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), + }); + } + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + + await store.clean(); + + await expect(store.readSubscription('tester-2')).rejects.toThrow( + "Subscription with ID 'tester-2' not found", + ); + + await store.upsertSubscription( + 'tester-3', + ['test'], + mockCredentials.service(), + ); + + // Reset read pointer to read form the beginning + await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ + read_until: 0, + }); + + const { events: events3 } = await store.readSubscription('tester-3'); + expect(events3.length).toBe(0); + }, + ); + + it.each(databases.eachSupportedId())( + 'should not clean up events within the min age window, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + minAge: 1000, + }); + + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), + }); + } + + await store.clean(); + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + }, + ); + + it.each(databases.eachSupportedId())( + 'should clean up a large number of events, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + }); + + const COUNT = '100000'; + + await db.raw(` + INSERT INTO event_bus_events (id, created_by, topic, data_json) + SELECT id, 'abc', 'test', '{}' + FROM generate_series(1, ${COUNT}) AS id + `); + + await expect(db('event_bus_events').count()).resolves.toEqual([ + { count: COUNT }, + ]); + + const start = Date.now(); + + await store.clean(); + + // Local testing shows this takes about 80ms, but if this is flaky we can + // reduce the count down to 10_000. + expect(Date.now() - start).toBeLessThan(500); + + await expect(db('event_bus_events').count()).resolves.toEqual([ + { count: '5' }, + ]); + }, + ); + + it.each(databases.eachSupportedId())( + 'should perform well when looking up events by topic, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + }); + + const COUNT = '100000'; + + // Insert 100,000 events, a lot more than we'd expect to ever have + // in a real-world scenario given our count window size is 10,000. + await db.raw(` + INSERT INTO event_bus_events (id, created_by, topic, data_json, notified_subscribers) + SELECT id, 'abc', CONCAT('test-', MOD(id, 10)), CONCAT('{"payload":{"id":"', id, '"}}'), '{"${String( + Math.random(), + ).slice(2, 6)}"}' + FROM generate_series(1, ${COUNT}) AS id + `); + await db('event_bus_subscriptions').insert({ + id: 'tester', + created_by: 'abc', + read_until: 0, + topics: ['test-5'], + }); + + const start = Date.now(); + const { events } = await store.readSubscription('tester'); + const duration = Date.now() - start; + + expect(events).toEqual([ + { topic: 'test-5', eventPayload: { id: '5' } }, + { topic: 'test-5', eventPayload: { id: '15' } }, + { topic: 'test-5', eventPayload: { id: '25' } }, + { topic: 'test-5', eventPayload: { id: '35' } }, + { topic: 'test-5', eventPayload: { id: '45' } }, + { topic: 'test-5', eventPayload: { id: '55' } }, + { topic: 'test-5', eventPayload: { id: '65' } }, + { topic: 'test-5', eventPayload: { id: '75' } }, + { topic: 'test-5', eventPayload: { id: '85' } }, + { topic: 'test-5', eventPayload: { id: '95' } }, + ]); + + expect(duration).toBeLessThan(20); + }, + ); +}); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts new file mode 100644 index 0000000000..107dabfc27 --- /dev/null +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -0,0 +1,622 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { EventBusStore } from './types'; +import { Knex } from 'knex'; +import { + BackstageCredentials, + BackstageServicePrincipal, + DatabaseService, + LifecycleService, + LoggerService, + SchedulerService, + resolvePackagePath, +} from '@backstage/backend-plugin-api'; +import { ForwardedError, NotFoundError } from '@backstage/errors'; +import { HumanDuration, durationToMilliseconds } from '@backstage/types'; + +const WINDOW_MAX_COUNT_DEFAULT = 10_000; +const WINDOW_MIN_AGE_DEFAULT = { minutes: 10 }; +const WINDOW_MAX_AGE_DEFAULT = { days: 1 }; + +const MAX_BATCH_SIZE = 10; +const LISTENER_CONNECTION_TIMEOUT_MS = 60_000; +const KEEPALIVE_INTERVAL_MS = 60_000; + +const TABLE_EVENTS = 'event_bus_events'; +const TABLE_SUBSCRIPTIONS = 'event_bus_subscriptions'; +const TOPIC_PUBLISH = 'event_bus_publish'; + +type EventsRow = { + id: string; + created_by: string; + created_at: Date; + topic: string; + data_json: string; + notified_subscribers: string[]; +}; + +type SubscriptionsRow = { + id: string; + created_by: string; + created_at: Date; + updated_at: Date; + read_until: string; + topics: string[]; +}; + +function creatorId( + credentials: BackstageCredentials, +) { + return `service=${credentials.principal.subject}`; +} + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-events-backend', + 'migrations', +); + +interface InternalDbClient { + acquireRawConnection(): Promise; + destroyRawConnection(conn: InternalDbConnection): Promise; +} + +interface InternalDbConnection { + query(sql: string): Promise; + end(): Promise; + on( + event: 'notification', + listener: (event: { channel: string; payload: string }) => void, + ): void; + on(event: 'error', listener: (error: Error) => void): void; + on(event: 'end', listener: (error?: Error) => void): void; + removeAllListeners(): void; +} + +// This internal class manages a single connection to the database that all listeners share +class DatabaseEventBusListener { + readonly #client: InternalDbClient; + readonly #logger: LoggerService; + + readonly #listeners = new Set<{ + topics: Set; + resolve: (result: { topic: string }) => void; + reject: (error: Error) => void; + }>(); + + #isShuttingDown = false; + #connPromise?: Promise; + #connTimeout?: NodeJS.Timeout; + #keepaliveInterval?: NodeJS.Timeout; + + constructor(client: InternalDbClient, logger: LoggerService) { + this.#client = client; + this.#logger = logger.child({ type: 'DatabaseEventBusListener' }); + } + + async setupListener( + topics: Set, + signal: AbortSignal, + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { + if (this.#connTimeout) { + clearTimeout(this.#connTimeout); + this.#connTimeout = undefined; + } + + await this.#ensureConnection(); + + const updatePromise = new Promise<{ topic: string }>((resolve, reject) => { + const listener = { + topics, + resolve(result: { topic: string }) { + resolve(result); + cleanup(); + }, + reject(err: Error) { + reject(err); + cleanup(); + }, + }; + this.#listeners.add(listener); + + const onAbort = () => { + this.#listeners.delete(listener); + this.#maybeTimeoutConnection(); + reject(signal.reason); + cleanup(); + }; + + function cleanup() { + signal.removeEventListener('abort', onAbort); + } + + signal.addEventListener('abort', onAbort); + }); + + // Ignore unhandled rejections + updatePromise.catch(() => {}); + + return { waitForUpdate: () => updatePromise }; + } + + async shutdown() { + if (this.#isShuttingDown) { + return; + } + this.#isShuttingDown = true; + const conn = await this.#connPromise?.catch(() => undefined); + if (conn) { + this.#destroyConnection(conn); + } + } + + #handleNotify(topic: string) { + this.#logger.debug(`Listener received notification for topic '${topic}'`); + for (const l of this.#listeners) { + if (l.topics.has(topic)) { + l.resolve({ topic }); + this.#listeners.delete(l); + } + } + this.#maybeTimeoutConnection(); + } + + // We don't try to reconnect on error, instead we notify all listeners and let + // them try to establish a new connection + #handleError(error: Error) { + this.#logger.error( + `Listener connection failed, notifying all listeners`, + error, + ); + for (const l of this.#listeners) { + l.reject(new Error('Listener connection failed')); + } + this.#listeners.clear(); + this.#maybeTimeoutConnection(); + } + + #maybeTimeoutConnection() { + // If we don't have any listeners, destroy the connection after a timeout + if (this.#listeners.size === 0 && !this.#connTimeout) { + this.#connTimeout = setTimeout(() => { + this.#connTimeout = undefined; + this.#connPromise?.then(conn => { + this.#logger.info('Listener connection timed out, destroying'); + this.#connPromise = undefined; + this.#destroyConnection(conn); + }); + }, LISTENER_CONNECTION_TIMEOUT_MS); + } + } + + #destroyConnection(conn: InternalDbConnection) { + if (this.#keepaliveInterval) { + clearInterval(this.#keepaliveInterval); + this.#keepaliveInterval = undefined; + } + this.#client.destroyRawConnection(conn).catch(error => { + this.#logger.error(`Listener failed to destroy connection`, error); + }); + conn.removeAllListeners(); + } + + async #ensureConnection() { + if (this.#isShuttingDown) { + throw new Error('Listener is shutting down'); + } + if (this.#connPromise) { + await this.#connPromise; + return; + } + this.#connPromise = Promise.resolve().then(async () => { + const conn = await this.#client.acquireRawConnection(); + + try { + await conn.query(`LISTEN ${TOPIC_PUBLISH}`); + + // Set up a keepalive interval to make sure the connection stays alive + if (this.#keepaliveInterval) { + clearInterval(this.#keepaliveInterval); + } + this.#keepaliveInterval = setInterval(() => { + conn.query('select 1').catch(error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError(new ForwardedError('Keepalive failed', error)); + }); + }, KEEPALIVE_INTERVAL_MS); + + conn.on('notification', event => { + this.#handleNotify(event.payload); + }); + conn.on('error', error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError(error); + }); + conn.on('end', error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError( + error ?? new Error('Connection ended unexpectedly'), + ); + }); + return conn; + } catch (error) { + this.#destroyConnection(conn); + throw error; + } + }); + try { + await this.#connPromise; + } catch (error) { + this.#connPromise = undefined; + throw error; + } + } +} + +export class DatabaseEventBusStore implements EventBusStore { + static async create(options: { + database: DatabaseService; + logger: LoggerService; + scheduler: SchedulerService; + lifecycle: LifecycleService; + window?: { + /** Events within this range will never be deleted */ + minAge?: HumanDuration; + /** Events outside of this age will always be deleted */ + maxAge?: HumanDuration; + /** Events outside of this count will be deleted if they are outside the minAge window */ + maxCount?: number; + }; + }): Promise { + const db = await options.database.getClient(); + + if (db.client.config.client !== 'pg') { + throw new Error( + `DatabaseEventBusStore only supports PostgreSQL, got '${db.client.config.client}'`, + ); + } + + if (!options.database.migrations?.skip) { + await db.migrate.latest({ + directory: migrationsDir, + }); + } + + const listener = new DatabaseEventBusListener(db.client, options.logger); + + const store = new DatabaseEventBusStore( + db, + options.logger, + listener, + options.window?.maxCount ?? WINDOW_MAX_COUNT_DEFAULT, + durationToMilliseconds(options.window?.minAge ?? WINDOW_MIN_AGE_DEFAULT), + durationToMilliseconds(options.window?.maxAge ?? WINDOW_MAX_AGE_DEFAULT), + ); + + await options.scheduler.scheduleTask({ + id: 'event-bus-cleanup', + frequency: { seconds: 10 }, + timeout: { minutes: 1 }, + initialDelay: { seconds: 10 }, + fn: () => store.#cleanup(), + }); + + options.lifecycle.addShutdownHook(async () => { + await listener.shutdown(); + }); + + return store; + } + + /** @internal */ + static async forTest({ + db, + logger, + minAge = 0, + maxAge = 10_000, + }: { + db: Knex; + logger: LoggerService; + minAge?: number; + maxAge?: number; + }) { + await db.migrate.latest({ directory: migrationsDir }); + + const store = new DatabaseEventBusStore( + db, + logger, + new DatabaseEventBusListener(db.client, logger), + 5, + minAge, + maxAge, + ); + + return Object.assign(store, { clean: () => store.#cleanup() }); + } + + readonly #db: Knex; + readonly #logger: LoggerService; + readonly #listener: DatabaseEventBusListener; + readonly #windowMaxCount: number; + readonly #windowMinAge: number; + readonly #windowMaxAge: number; + + private constructor( + db: Knex, + logger: LoggerService, + listener: DatabaseEventBusListener, + windowMaxCount: number, + windowMinAge: number, + windowMaxAge: number, + ) { + this.#db = db; + this.#logger = logger; + this.#listener = listener; + this.#windowMaxCount = windowMaxCount; + this.#windowMinAge = windowMinAge; + this.#windowMaxAge = windowMaxAge; + } + + async publish(options: { + event: EventParams; + notifiedSubscribers?: string[]; + credentials: BackstageCredentials; + }): Promise<{ eventId: string } | undefined> { + const topic = options.event.topic; + const notifiedSubscribers = options.notifiedSubscribers ?? []; + // This query inserts a new event into the database, but only if there are + // subscribers to the topic that have not already been notified + const result = await this.#db + // There's no clean way to create a INSERT INTO .. SELECT with knex, so we end up with quite a lot of .raw(...) + .into( + this.#db.raw('?? (??, ??, ??, ??)', [ + TABLE_EVENTS, + // These are the rows that we insert, and should match the SELECT below + 'created_by', + 'topic', + 'data_json', + 'notified_subscribers', + ]), + ) + .insert( + (q: Knex.QueryBuilder) => + q + // We're not reading data to insert from anywhere else, just raw data + .select( + this.#db.raw('?', [creatorId(options.credentials)]), + this.#db.raw('?', [topic]), + this.#db.raw('?', [ + JSON.stringify({ + payload: options.event.eventPayload, + metadata: options.event.metadata, + }), + ]), + this.#db.raw('?', [notifiedSubscribers]), + ) + // The rest of this query is to check whether there are any + // subscribers that have not been notified yet + .from(TABLE_SUBSCRIPTIONS) + .whereNotIn('id', notifiedSubscribers) // Skip notified subscribers + .andWhere(this.#db.raw('? = ANY(topics)', [topic])) // Match topic + .having(this.#db.raw('count(*)'), '>', 0), // Check if there are any results + ) + .returning<{ id: string }[]>('id'); + + if (result.length === 0) { + return undefined; + } + if (result.length > 1) { + throw new Error( + `Failed to insert event, unexpectedly updated ${result.length} rows`, + ); + } + + const [{ id }] = result; + + // Notify other event bus instances that an event is available on the topic + const notifyResult = await this.#db.select( + this.#db.raw(`pg_notify(?, ?)`, [TOPIC_PUBLISH, topic]), + ); + if (notifyResult?.length !== 1) { + this.#logger.warn( + `Failed to notify subscribers of event with ID '${id}' on topic '${topic}'`, + ); + } + + return { eventId: id }; + } + + async upsertSubscription( + id: string, + topics: string[], + credentials: BackstageCredentials, + ): Promise { + const [{ max: maxId }] = await this.#db(TABLE_EVENTS).max('id'); + const result = await this.#db(TABLE_SUBSCRIPTIONS) + .insert({ + id, + created_by: creatorId(credentials), + updated_at: this.#db.fn.now(), + topics, + read_until: maxId || 0, + }) + .onConflict('id') + .merge(['created_by', 'topics', 'updated_at']) + .returning('*'); + + if (result.length !== 1) { + throw new Error( + `Failed to upsert subscription, updated ${result.length} rows`, + ); + } + } + + async readSubscription(id: string): Promise<{ events: EventParams[] }> { + // The below query selects the subscription we're reading from, locks it for + // an update, reads events for the subscription up to the limit, and then + // updates the pointer to the last read event. + // + // This is written as a plain SQL query to spare us all the horrors of + // expressing this in knex. + + const { rows: result } = await this.#db.raw<{ + rows: [] | [{ events: EventsRow[] }]; + }>( + ` + WITH subscription AS ( + SELECT topics, read_until + FROM event_bus_subscriptions + WHERE id = :id + FOR UPDATE + ), + selected_events AS ( + SELECT event_bus_events.* + FROM event_bus_events + INNER JOIN subscription + ON event_bus_events.topic = ANY(subscription.topics) + WHERE event_bus_events.id > subscription.read_until + AND NOT :id = ANY(event_bus_events.notified_subscribers) + ORDER BY event_bus_events.id ASC LIMIT :limit + ), + last_event_id AS ( + SELECT max(id) AS last_event_id + FROM selected_events + ), + events_array AS ( + SELECT json_agg(row_to_json(selected_events)) AS events + FROM selected_events + ) + UPDATE event_bus_subscriptions + SET read_until = COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events), 0) + FROM events_array, last_event_id + WHERE event_bus_subscriptions.id = :id + RETURNING events_array.events + `, + { id, limit: MAX_BATCH_SIZE }, + ); + + if (result.length === 0) { + throw new NotFoundError(`Subscription with ID '${id}' not found`); + } else if (result.length > 1) { + throw new Error( + `Failed to read subscription, unexpectedly updated ${result.length} rows`, + ); + } + + const rows = result[0].events; + if (!rows || rows.length === 0) { + return { events: [] }; + } + + return { + events: rows.map(row => { + const { payload, metadata } = JSON.parse(row.data_json); + return { + topic: row.topic, + eventPayload: payload, + metadata, + }; + }), + }; + } + + async setupListener( + subscriptionId: string, + options: { + signal: AbortSignal; + }, + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { + const result = await this.#db(TABLE_SUBSCRIPTIONS) + .select('topics') + .where({ id: subscriptionId }) + .first(); + + if (!result) { + throw new NotFoundError( + `Subscription with ID '${subscriptionId}' not found`, + ); + } + + options.signal.throwIfAborted(); + + return this.#listener.setupListener( + new Set(result.topics ?? []), + options.signal, + ); + } + + async #cleanup() { + try { + const eventCount = await this.#db(TABLE_EVENTS) + .delete() + // Delete any events that are outside both the min age and size window + .orWhere(inner => + inner + .whereIn( + 'id', + this.#db + .select('id') + .from(TABLE_EVENTS) + .orderBy('id', 'desc') + .offset(this.#windowMaxCount), + ) + .andWhere( + 'created_at', + '<', + new Date(Date.now() - this.#windowMinAge), + ), + ) + // If events are outside the max age they will always be deleted + .orWhere('created_at', '<', new Date(Date.now() - this.#windowMaxAge)); + + if (eventCount > 0) { + this.#logger.info( + `Event cleanup resulted in ${eventCount} old events being deleted`, + ); + } + } catch (error) { + this.#logger.error('Event cleanup failed', error); + } + + try { + // Delete any subscribers that aren't keeping up with current events + const [{ min: minId }] = await this.#db(TABLE_EVENTS).min('id'); + + let subscriberCount; + if (minId === null) { + // No events left, remove all subscribers. This can happen if no events + // are published within the max age window. + subscriberCount = await this.#db(TABLE_SUBSCRIPTIONS).delete(); + } else { + subscriberCount = await this.#db(TABLE_SUBSCRIPTIONS) + .delete() + // Read pointer points to the ID that has been read, so we need an additional offset + .where('read_until', '<', minId - 1); + } + + if (subscriberCount > 0) { + this.#logger.info( + `Subscription cleanup resulted in ${subscriberCount} stale subscribers being deleted`, + ); + } + } catch (error) { + this.#logger.error('Subscription cleanup failed', error); + } + } +} diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts new file mode 100644 index 0000000000..52d075951d --- /dev/null +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { MemoryEventBusStore } from './MemoryEventBusStore'; +import { mockCredentials } from '@backstage/backend-test-utils'; + +function mkEvent(message: string): EventParams { + return { + topic: 'test', + eventPayload: { message }, + }; +} + +describe('MemoryEventBusStore', () => { + it('should publish to subscribers', async () => { + const store = new MemoryEventBusStore(); + + await expect( + store.publish({ + event: mkEvent('hello'), + notifiedSubscribers: [], + credentials: mockCredentials.service(), + }), + ).resolves.toEqual(undefined); + + await expect(store.readSubscription('test')).rejects.toThrow( + 'Subscription not found', + ); + + await store.upsertSubscription('tester', ['test']); + + await expect( + store.publish({ + event: mkEvent('hello'), + notifiedSubscribers: [], + credentials: mockCredentials.service(), + }), + ).resolves.toEqual({ eventId: '1' }); + + await expect( + store.publish({ + event: mkEvent('ignored'), + notifiedSubscribers: ['tester'], + credentials: mockCredentials.service(), + }), + ).resolves.toEqual(undefined); + + await expect(store.readSubscription('tester')).resolves.toEqual({ + events: [mkEvent('hello')], + }); + }); + + it('should clean up old events', async () => { + const store = new MemoryEventBusStore({ maxEvents: 5 }); + + await store.upsertSubscription('tester', ['test']); + + for (let i = 0; i < 20; ++i) { + await expect( + store.publish({ + event: mkEvent(`hello ${i}`), + notifiedSubscribers: [], + credentials: mockCredentials.service(), + }), + ).resolves.toEqual({ eventId: String(i + 1) }); + } + + await expect(store.readSubscription('tester')).resolves.toEqual({ + events: [ + mkEvent('hello 15'), + mkEvent('hello 16'), + mkEvent('hello 17'), + mkEvent('hello 18'), + mkEvent('hello 19'), + ], + }); + }); +}); diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts new file mode 100644 index 0000000000..6c999dc390 --- /dev/null +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -0,0 +1,164 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { EventBusStore } from './types'; +import { NotFoundError } from '@backstage/errors'; +import { + BackstageCredentials, + BackstageServicePrincipal, +} from '@backstage/backend-plugin-api'; + +const MAX_BATCH_SIZE = 10; +const MAX_EVENTS_DEFAULT = 1_000; + +export class MemoryEventBusStore implements EventBusStore { + #maxEvents: number; + #events = new Array< + EventParams & { seq: number; notifiedSubscribers: Set } + >(); + #subscribers = new Map< + string, + { id: string; seq: number; topics: Set } + >(); + #listeners = new Set<{ + topics: Set; + resolve(result: { topic: string }): void; + }>(); + + constructor(options: { maxEvents?: number } = {}) { + this.#maxEvents = options.maxEvents ?? MAX_EVENTS_DEFAULT; + } + + async publish(options: { + event: EventParams; + notifiedSubscribers: string[]; + credentials: BackstageCredentials; + }): Promise<{ eventId: string } | undefined> { + const topic = options.event.topic; + const notifiedSubscribers = new Set(options.notifiedSubscribers); + + let hasOtherSubscribers = false; + for (const sub of this.#subscribers.values()) { + if (sub.topics.has(topic) && !notifiedSubscribers.has(sub.id)) { + hasOtherSubscribers = true; + break; + } + } + if (!hasOtherSubscribers) { + return undefined; + } + + const nextSeq = this.#getMaxSeq() + 1; + this.#events.push({ ...options.event, notifiedSubscribers, seq: nextSeq }); + + for (const listener of this.#listeners) { + if (listener.topics.has(topic)) { + listener.resolve({ topic }); + this.#listeners.delete(listener); + } + } + + // Trim old events + if (this.#events.length > this.#maxEvents) { + this.#events.shift(); + } + + return { eventId: String(nextSeq) }; + } + + #getMaxSeq() { + return this.#events[this.#events.length - 1]?.seq ?? 0; + } + + async upsertSubscription(id: string, topics: string[]): Promise { + const existing = this.#subscribers.get(id); + if (existing) { + existing.topics = new Set(topics); + return; + } + const sub = { + id: id, + seq: this.#getMaxSeq(), + topics: new Set(topics), + }; + this.#subscribers.set(id, sub); + } + + async readSubscription(id: string): Promise<{ events: EventParams[] }> { + const sub = this.#subscribers.get(id); + if (!sub) { + throw new NotFoundError(`Subscription not found`); + } + const events = this.#events + .filter( + event => + event.seq > sub.seq && + sub.topics.has(event.topic) && + !event.notifiedSubscribers.has(id), + ) + .slice(0, MAX_BATCH_SIZE); + + sub.seq = events[events.length - 1]?.seq ?? sub.seq; + + return { + events: events.map(({ topic, eventPayload }) => ({ + topic, + eventPayload, + })), + }; + } + + async setupListener( + subscriptionId: string, + options: { + signal: AbortSignal; + }, + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { + return { + waitForUpdate: async () => { + options.signal.throwIfAborted(); + + const sub = this.#subscribers.get(subscriptionId); + if (!sub) { + throw new NotFoundError(`Subscription not found`); + } + + return new Promise<{ topic: string }>((resolve, reject) => { + const listener = { + topics: sub.topics, + resolve(result: { topic: string }) { + resolve(result); + cleanup(); + }, + }; + this.#listeners.add(listener); + + const onAbort = () => { + this.#listeners.delete(listener); + reject(options.signal.reason); + cleanup(); + }; + + function cleanup() { + options.signal.removeEventListener('abort', onAbort); + } + + options.signal.addEventListener('abort', onAbort); + }); + }, + }; + } +} diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts new file mode 100644 index 0000000000..4a332d5ce5 --- /dev/null +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -0,0 +1,278 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DatabaseService, + HttpAuthService, + LifecycleService, + LoggerService, + SchedulerService, +} from '@backstage/backend-plugin-api'; +import { Handler } from 'express'; +import { createOpenApiRouter } from '../../schema/openapi.generated'; +import { MemoryEventBusStore } from './MemoryEventBusStore'; +import { DatabaseEventBusStore } from './DatabaseEventBusStore'; +import { EventBusStore } from './types'; +import { EventParams } from '@backstage/plugin-events-node'; + +const DEFAULT_NOTIFY_TIMEOUT_MS = 55_000; // Just below 60s, which is a common HTTP timeout + +/* + +# Event Bus + +This comment describes the event bus that is implemented here in the events +backend, and by default used by the events service. + +## Overview + +The events bus implements a subscription mechanism where subscribers must exist +upfront for events to be stored. It uses a single inbox for all events, with +each subscriber having its own pointer for how far into the inbox it has read. + +In order to avoid busy polling, the API uses a long-polling mechanism where a +request is left hanging until the client should try to read again. + +The event bus is not implemented with any guarantees of events being consumed, +but it does aim to make it unlikely that events are dropped + +## API + +### POST /bus/v1/events + +This endpoint is used to publish new events to the event bus on a specific +topic. It can optionally include a set of subscription IDs for subscribers that +have already been notified of the event. This is to enable an optimization where +we notify subscribers locally if possible, and avoid the need for events to be +relayed through the events bus at all of possible. + +For an event to be published and stored there must already exist a subscriber +that is subscribed to the event's topic, and that hasn't already been notified +of the event. If no such subscriber is found, the event will be discarded. + +### PUT /bus/v1/subscriptions/:subscriptionId + +This endpoint is used to create or update a subscriptions. Subscriptions are +shared across the entire bus and divided by subscription ID. Multiple clients +can be reading events from the same subscription at the same time, but only one +of those clients will receive each event. This enables division of work by using +the same subscriber ID across multiple instances, as well as broadcasting by +ensuring separate subscribers IDs. + +### GET /bus/v1/subscriptions/:subscriptionId/events + +This endpoint is used to read events from a subscription. It will return a batch +of events for the subscribed topics that have not yet been read by the +subscription. If no such events are available, the endpoint will return a 202 +response and then hang end response until an event is available or a timeout is +reached. This allows clients to call this endpoint in a loop but will keep +traffic overhead fairly low. + +## Delivery guarantees + +When reading events from the event bus, clients should always implement a +graceful shutdown where they process any events that are received from the +events endpoint before shutting down. This is also the reason that the events +endpoint does not return any events when responding with a 202 blocking the +response, because there would otherwise be a race condition where the events +might be lost in transit if the client shuts down. By always sending an empty +response and requiring the client to send another request, we ensure that the +client is prepared to halt shutdown until the request had been fully processed. + +## Local processing optimization + +When possible, events will be processed locally before sent to the event bus. +The client will also inform the bus of which subscriptions have already been +notified of the event, so that the bus can completely avoid storing an event if +it has already been fully consumed by all subscribers. + +## Automated cleanup & event window + +Events are deleted once they are outside the guaranteed storage window. By +default the window 10 minutes for all events, and 24 hours for the last 10000 +events. This ensures that the event log doesn't grow indefinitely, while still +allowing subscribers with restarts or outages to catch up to past events, +ensuring that events are likely not lost. + +Subscriptions are also cleaned up if their read pointer falls outside of the +current event window. This ensures that stale subscribers don't accumulate and +cause unnecessary storage of events. + +*/ + +async function createEventBusStore(deps: { + logger: LoggerService; + database: DatabaseService; + scheduler: SchedulerService; + lifecycle: LifecycleService; + httpAuth: HttpAuthService; +}): Promise { + const db = await deps.database.getClient(); + if (db.client.config.client === 'pg') { + deps.logger.info('Database is PostgreSQL, using database store'); + return await DatabaseEventBusStore.create(deps); + } + + deps.logger.info('Database is not PostgreSQL, using memory store'); + return new MemoryEventBusStore(); +} + +/** + * Creates a new event bus router + * @internal + */ +export async function createEventBusRouter(options: { + logger: LoggerService; + database: DatabaseService; + scheduler: SchedulerService; + lifecycle: LifecycleService; + httpAuth: HttpAuthService; + notifyTimeoutMs?: number; // for testing +}): Promise { + const { httpAuth, notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS } = options; + const logger = options.logger.child({ type: 'EventBus' }); + + const store = await createEventBusStore(options); + + const apiRouter = await createOpenApiRouter(); + + apiRouter.post('/bus/v1/events', async (req, res) => { + const credentials = await httpAuth.credentials(req, { + allow: ['service'], + }); + const topic = req.body.event.topic; + const notifiedSubscribers = req.body.notifiedSubscribers; + const result = await store.publish({ + event: { + topic, + eventPayload: req.body.event.payload, + } as EventParams, + notifiedSubscribers, + credentials, + }); + if (result) { + logger.debug( + `Published event to '${topic}' with ID '${result.eventId}'`, + { + subject: credentials.principal.subject, + }, + ); + res.status(201).end(); + } else { + if (notifiedSubscribers) { + const notified = `'${notifiedSubscribers.join("', '")}'`; + logger.debug( + `Skipped publishing of event to '${topic}', subscribers have already been notified: ${notified}`, + { subject: credentials.principal.subject }, + ); + } else { + logger.debug( + `Skipped publishing of event to '${topic}', no subscribers present`, + { subject: credentials.principal.subject }, + ); + } + res.status(204).end(); + } + }); + + apiRouter.get( + '/bus/v1/subscriptions/:subscriptionId/events', + async (req, res) => { + const credentials = await httpAuth.credentials(req, { + allow: ['service'], + }); + const id = req.params.subscriptionId; + + const controller = new AbortController(); + req.on('end', () => controller.abort()); + + // By setting up the listener first we make sure we don't miss any events + // that are published while reading. If an event is published we'll receive + // a notification, which we may ignore depending on the outcome of the read + const listener = await store.setupListener(id, { + signal: controller.signal, + }); + + // By timing out requests we make sure they don't stall or that events get stuck. + // For the caller there's no difference between a timeout and a + // notification, either way they should try reading again. + const timeout = setTimeout(() => { + controller.abort(); + }, notifyTimeoutMs); + + try { + const { events } = await store.readSubscription(id); + + logger.debug( + `Reading subscription '${id}' resulted in ${events.length} events`, + { subject: credentials.principal.subject }, + ); + + if (events.length > 0) { + res.json({ + events: events.map(event => ({ + topic: event.topic, + payload: event.eventPayload, + })), + }); + } else { + res.status(202); + res.flushHeaders(); + + try { + const { topic } = await listener.waitForUpdate(); + logger.debug( + `Received notification for subscription '${id}' for topic '${topic}'`, + { subject: credentials.principal.subject }, + ); + } catch (error) { + if (error !== controller.signal.reason) { + logger.error(`Error listening for subscription '${id}'`, error); + } + } finally { + // A small extra delay ensures a more even spread of events across + // consumers in case some consumers are faster than others + await new Promise(resolve => + setTimeout(resolve, 1 + Math.random() * 9), + ); + res.end(); + } + } + } finally { + controller.abort(); + clearTimeout(timeout); + } + }, + ); + + apiRouter.put('/bus/v1/subscriptions/:subscriptionId', async (req, res) => { + const credentials = await httpAuth.credentials(req, { + allow: ['service'], + }); + const id = req.params.subscriptionId; + + await store.upsertSubscription(id, req.body.topics, credentials); + + logger.debug( + `New subscription '${id}' for topics '${req.body.topics.join("', '")}'`, + { subject: credentials.principal.subject }, + ); + + res.status(201).end(); + }); + + return apiRouter; +} diff --git a/packages/cli/src/lib/diff/index.ts b/plugins/events-backend/src/service/hub/index.ts similarity index 79% rename from packages/cli/src/lib/diff/index.ts rename to plugins/events-backend/src/service/hub/index.ts index 79a55f023e..a4f2f263f1 100644 --- a/packages/cli/src/lib/diff/index.ts +++ b/plugins/events-backend/src/service/hub/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,4 @@ * limitations under the License. */ -export * from './handlers'; -export * from './prompts'; -export * from './read'; -export * from './types'; +export { createEventBusRouter } from './createEventBusRouter'; diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts new file mode 100644 index 0000000000..316d9e561f --- /dev/null +++ b/plugins/events-backend/src/service/hub/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + BackstageCredentials, + BackstageServicePrincipal, +} from '@backstage/backend-plugin-api'; +import { EventParams } from '@backstage/plugin-events-node'; + +export type EventBusStore = { + publish(options: { + event: EventParams; + notifiedSubscribers?: string[]; + credentials: BackstageCredentials; + }): Promise<{ eventId: string } | undefined>; + + upsertSubscription( + subscriptionId: string, + topics: string[], + credentials: BackstageCredentials, + ): Promise; + + readSubscription(subscriptionId: string): Promise<{ events: EventParams[] }>; + + setupListener( + subscriptionId: string, + options: { + signal: AbortSignal; + }, + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }>; +}; diff --git a/plugins/events-node/package.json b/plugins/events-node/package.json index 6795c13441..2ac1a6922b 100644 --- a/plugins/events-node/package.json +++ b/plugins/events-node/package.json @@ -51,7 +51,11 @@ "test": "backstage-cli package test" }, "dependencies": { - "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/types": "workspace:^", + "cross-fetch": "^4.0.0", + "uri-template": "^2.0.0" }, "devDependencies": { "@backstage/backend-common": "^0.25.0", diff --git a/plugins/events-node/report.api.md b/plugins/events-node/report.api.md index c885c7ec10..ceeb21de53 100644 --- a/plugins/events-node/report.api.md +++ b/plugins/events-node/report.api.md @@ -3,6 +3,9 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AuthService } from '@backstage/backend-plugin-api'; +import { DiscoveryService } from '@backstage/backend-plugin-api'; +import { LifecycleService } from '@backstage/backend-plugin-api'; import { LoggerService } from '@backstage/backend-plugin-api'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; @@ -11,7 +14,15 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; export class DefaultEventsService implements EventsService { // (undocumented) static create(options: { logger: LoggerService }): DefaultEventsService; - forPlugin(pluginId: string): EventsService; + forPlugin( + pluginId: string, + options?: { + discovery: DiscoveryService; + logger: LoggerService; + auth: AuthService; + lifecycle: LifecycleService; + }, + ): EventsService; // (undocumented) publish(params: EventParams): Promise; // (undocumented) diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index bb5c2a0ca8..aa77d6e696 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -14,9 +14,320 @@ * limitations under the License. */ -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + AuthService, + DiscoveryService, + LifecycleService, + LoggerService, +} from '@backstage/backend-plugin-api'; import { EventParams } from './EventParams'; import { EventsService, EventsServiceSubscribeOptions } from './EventsService'; +import { DefaultApiClient } from '../generated'; +import { ResponseError } from '@backstage/errors'; + +const POLL_BACKOFF_START_MS = 1_000; +const POLL_BACKOFF_MAX_MS = 60_000; +const POLL_BACKOFF_FACTOR = 2; + +/** + * Local event bus for subscribers within the same process. + * + * When publishing events we'll keep track of which subscribers we managed to + * reach locally, and forward those subscriber IDs to the events backend if it + * is in use. The events backend will then both avoid forwarding the same events + * to those subscribers again, but also avoid storing the event altogether if + * there are no other subscribers. + * @internal + */ +export class LocalEventBus { + readonly #logger: LoggerService; + + readonly #subscribers = new Map< + string, + Omit[] + >(); + + constructor(logger: LoggerService) { + this.#logger = logger; + } + + async publish( + params: EventParams, + ): Promise<{ notifiedSubscribers: string[] }> { + this.#logger.debug( + `Event received: topic=${params.topic}, metadata=${JSON.stringify( + params.metadata, + )}, payload=${JSON.stringify(params.eventPayload)}`, + ); + + if (!this.#subscribers.has(params.topic)) { + return { notifiedSubscribers: [] }; + } + + const onEventPromises: Promise[] = []; + this.#subscribers.get(params.topic)?.forEach(subscription => { + onEventPromises.push( + (async () => { + try { + await subscription.onEvent(params); + } catch (error) { + this.#logger.warn( + `Subscriber "${subscription.id}" failed to process event for topic "${params.topic}"`, + error, + ); + } + return subscription.id; + })(), + ); + }); + + return { notifiedSubscribers: await Promise.all(onEventPromises) }; + } + + async subscribe(options: EventsServiceSubscribeOptions): Promise { + options.topics.forEach(topic => { + if (!this.#subscribers.has(topic)) { + this.#subscribers.set(topic, []); + } + + this.#subscribers.get(topic)!.push({ + id: options.id, + onEvent: options.onEvent, + }); + }); + } +} + +/** + * Plugin specific events bus that delegates to the local bus, as well as the + * events backend if it is available. + */ +class PluginEventsService implements EventsService { + constructor( + private readonly pluginId: string, + private readonly localBus: LocalEventBus, + private readonly logger: LoggerService, + private client?: DefaultApiClient, + private readonly auth?: AuthService, + ) {} + + async publish(params: EventParams): Promise { + const lock = this.#getShutdownLock(); + try { + const { notifiedSubscribers } = await this.localBus.publish(params); + + if (!this.client) { + return; + } + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.postEvent( + { + body: { + event: { payload: params.eventPayload, topic: params.topic }, + notifiedSubscribers, + }, + }, + { token }, + ); + + if (!res.ok) { + if (res.status === 404) { + this.logger.warn( + `Event publish request failed with status 404, events backend not found. Future events will not be persisted.`, + ); + delete this.client; + return; + } + throw await ResponseError.fromResponse(res); + } + } finally { + lock.release(); + } + } + + async subscribe(options: EventsServiceSubscribeOptions): Promise { + const subscriptionId = `${this.pluginId}.${options.id}`; + + await this.localBus.subscribe({ + id: subscriptionId, + topics: options.topics, + onEvent: options.onEvent, + }); + + if (!this.client) { + return; + } + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.putSubscription( + { + path: { subscriptionId }, + body: { topics: options.topics }, + }, + { token }, + ); + if (!res.ok) { + if (res.status === 404) { + this.logger.warn( + `Event subscribe request failed with status 404, events backend not found. Will only receive events that were sent locally on this process.`, + ); + delete this.client; + return; + } + throw await ResponseError.fromResponse(res); + } + + this.#startPolling(subscriptionId, options.topics, options.onEvent); + } + + #startPolling( + subscriptionId: string, + topics: string[], + onEvent: EventsServiceSubscribeOptions['onEvent'], + ) { + let backoffMs = POLL_BACKOFF_START_MS; + const poll = async () => { + if (!this.client) { + return; + } + const lock = this.#getShutdownLock(); + try { + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.getSubscriptionEvents( + { + path: { subscriptionId }, + }, + { token }, + ); + + if (!res.ok) { + if (res.status === 404) { + this.logger.info( + `Polling event subscription resulted in a 404, recreating subscription`, + ); + const putRes = await this.client.putSubscription( + { + path: { subscriptionId }, + body: { topics }, + }, + { token }, + ); + if (!putRes.ok) { + throw await ResponseError.fromResponse(res); + } + } + throw await ResponseError.fromResponse(res); + } + backoffMs = POLL_BACKOFF_START_MS; + + // 202 means there were no immediately available events, but the + // response will block until either new events are available or the + // request times out. In both cases we should should try to read events + // immediately again + if (res.status === 202) { + lock.release(); + await res.body?.getReader()?.closed; + process.nextTick(poll); + } else if (res.status === 200) { + const data = await res.json(); + if (data) { + for (const event of data.events ?? []) { + try { + await onEvent({ + topic: event.topic, + eventPayload: event.payload, + }); + } catch (error) { + this.logger.warn( + `Subscriber "${subscriptionId}" failed to process event for topic "${event.topic}"`, + error, + ); + } + } + } + process.nextTick(poll); + } else { + this.logger.warn( + `Unexpected response status ${res.status} from events backend for subscription "${subscriptionId}"`, + ); + } + } catch (error) { + this.logger.warn( + `Poll failed for subscription "${subscriptionId}", retrying in ${backoffMs.toFixed( + 0, + )}ms`, + error, + ); + setTimeout(poll, backoffMs); + backoffMs = Math.min( + backoffMs * POLL_BACKOFF_FACTOR, + POLL_BACKOFF_MAX_MS, + ); + } finally { + lock.release(); + } + }; + poll(); + } + + async #getToken() { + if (!this.auth) { + throw new Error('Auth service not available'); + } + + try { + const { token } = await this.auth.getPluginRequestToken({ + onBehalfOf: await this.auth.getOwnServiceCredentials(), + targetPluginId: 'events', + }); + return token; + } catch (error) { + // This is a bit hacky, but handles the case where new auth is used + // without legacy auth fallback, and the events backend is not installed + if (String(error).includes('Unable to generate legacy token')) { + this.logger.warn( + `The events backend is not available and neither is legacy auth. Future events will not be persisted.`, + ); + delete this.client; + return undefined; + } + throw error; + } + } + + async shutdown() { + this.#isShuttingDown = true; + await Promise.all(this.#shutdownLocks); + } + + #isShuttingDown = false; + #shutdownLocks: Promise[] = []; + + // This locking mechanism helps ensure that we are either idle or waiting for + // a blocked events call before shutting down. It increases out changes of + // never dropping any events on shutdown. + #getShutdownLock(): { release(): void } { + if (this.#isShuttingDown) { + throw new Error('Service is shutting down'); + } + + let release: () => void; + this.#shutdownLocks.push( + new Promise(resolve => { + release = resolve; + }), + ); + return { release: release! }; + } +} /** * In-process event broker which will pass the event to all registered subscribers @@ -28,15 +339,16 @@ import { EventsService, EventsServiceSubscribeOptions } from './EventsService'; */ // TODO(pjungermann): add opentelemetry? (see plugins/catalog-backend/src/util/opentelemetry.ts, etc.) export class DefaultEventsService implements EventsService { - private readonly subscribers = new Map< - string, - Omit[] - >(); - - private constructor(private readonly logger: LoggerService) {} + private constructor( + private readonly logger: LoggerService, + private readonly localBus: LocalEventBus, + ) {} static create(options: { logger: LoggerService }): DefaultEventsService { - return new DefaultEventsService(options.logger); + return new DefaultEventsService( + options.logger, + new LocalEventBus(options.logger), + ); } /** @@ -45,60 +357,40 @@ export class DefaultEventsService implements EventsService { * * @param pluginId - The plugin that the `EventService` should be created for. */ - forPlugin(pluginId: string): EventsService { - return { - publish: (params: EventParams): Promise => { - return this.publish(params); - }, - subscribe: (options: EventsServiceSubscribeOptions): Promise => { - return this.subscribe({ - ...options, - id: `${pluginId}.${options.id}`, - }); - }, - }; + forPlugin( + pluginId: string, + options?: { + discovery: DiscoveryService; + logger: LoggerService; + auth: AuthService; + lifecycle: LifecycleService; + }, + ): EventsService { + const client = + options && + new DefaultApiClient({ + discoveryApi: options.discovery, + fetchApi: { fetch }, // use native node fetch + }); + const logger = options?.logger ?? this.logger; + const service = new PluginEventsService( + pluginId, + this.localBus, + logger, + client, + options?.auth, + ); + options?.lifecycle.addShutdownHook(async () => { + await service.shutdown(); + }); + return service; } async publish(params: EventParams): Promise { - this.logger.debug( - `Event received: topic=${params.topic}, metadata=${JSON.stringify( - params.metadata, - )}, payload=${JSON.stringify(params.eventPayload)}`, - ); - - if (!this.subscribers.has(params.topic)) { - return; - } - - const onEventPromises: Promise[] = []; - this.subscribers.get(params.topic)?.forEach(subscription => { - onEventPromises.push( - (async () => { - try { - await subscription.onEvent(params); - } catch (error) { - this.logger.warn( - `Subscriber "${subscription.id}" failed to process event for topic "${params.topic}"`, - error, - ); - } - })(), - ); - }); - - await Promise.all(onEventPromises); + await this.localBus.publish(params); } async subscribe(options: EventsServiceSubscribeOptions): Promise { - options.topics.forEach(topic => { - if (!this.subscribers.has(topic)) { - this.subscribers.set(topic, []); - } - - this.subscribers.get(topic)!.push({ - id: options.id, - onEvent: options.onEvent, - }); - }); + this.localBus.subscribe(options); } } diff --git a/plugins/events-node/src/generated/apis/DefaultApi.client.ts b/plugins/events-node/src/generated/apis/DefaultApi.client.ts new file mode 100644 index 0000000000..4bdeed0c33 --- /dev/null +++ b/plugins/events-node/src/generated/apis/DefaultApi.client.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { DiscoveryApi } from '../types/discovery'; +import { FetchApi } from '../types/fetch'; +import crossFetch from 'cross-fetch'; +import { pluginId } from '../pluginId'; +import * as parser from 'uri-template'; + +import { GetSubscriptionEvents200Response } from '../models/GetSubscriptionEvents200Response.model'; +import { PostEventRequest } from '../models/PostEventRequest.model'; +import { PutSubscriptionRequest } from '../models/PutSubscriptionRequest.model'; + +/** + * Wraps the Response type to convey a type on the json call. + * + * @public + */ +export type TypedResponse = Omit & { + json: () => Promise; +}; + +/** + * Options you can pass into a request for additional information. + * + * @public + */ +export interface RequestOptions { + token?: string; +} + +/** + * no description + */ +export class DefaultApiClient { + private readonly discoveryApi: DiscoveryApi; + private readonly fetchApi: FetchApi; + + constructor(options: { + discoveryApi: { getBaseUrl(pluginId: string): Promise }; + fetchApi?: { fetch: typeof fetch }; + }) { + this.discoveryApi = options.discoveryApi; + this.fetchApi = options.fetchApi || { fetch: crossFetch }; + } + + /** + * Get new events for the provided subscription + * @param subscriptionId + */ + public async getSubscriptionEvents( + // @ts-ignore + request: { + path: { + subscriptionId: string; + }; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/bus/v1/subscriptions/{subscriptionId}/events`; + + const uri = parser.parse(uriTemplate).expand({ + subscriptionId: request.path.subscriptionId, + }); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'GET', + }); + } + + /** + * Publish a new event + * @param postEventRequest + */ + public async postEvent( + // @ts-ignore + request: { + body: PostEventRequest; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/bus/v1/events`; + + const uri = parser.parse(uriTemplate).expand({}); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'POST', + body: JSON.stringify(request.body), + }); + } + + /** + * Ensures that the subscription exists with the provided configuration + * @param subscriptionId + * @param putSubscriptionRequest + */ + public async putSubscription( + // @ts-ignore + request: { + path: { + subscriptionId: string; + }; + body: PutSubscriptionRequest; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/bus/v1/subscriptions/{subscriptionId}`; + + const uri = parser.parse(uriTemplate).expand({ + subscriptionId: request.path.subscriptionId, + }); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'PUT', + body: JSON.stringify(request.body), + }); + } +} diff --git a/packages/cli/src/commands/onboard/index.ts b/plugins/events-node/src/generated/apis/index.ts similarity index 88% rename from packages/cli/src/commands/onboard/index.ts rename to plugins/events-node/src/generated/apis/index.ts index 4c23ea6e48..51dcca33fe 100644 --- a/packages/cli/src/commands/onboard/index.ts +++ b/plugins/events-node/src/generated/apis/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export { command } from './command'; +export * from './DefaultApi.client'; diff --git a/packages/cli/src/commands/install/steps/index.ts b/plugins/events-node/src/generated/index.ts similarity index 77% rename from packages/cli/src/commands/install/steps/index.ts rename to plugins/events-node/src/generated/index.ts index 590fba120f..bb399e97a0 100644 --- a/packages/cli/src/commands/install/steps/index.ts +++ b/plugins/events-node/src/generated/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,5 @@ * limitations under the License. */ -export { appRoute } from './appRoute'; -export { dependencies } from './dependencies'; -export { message } from './message'; +export * from './apis'; +export * from './models'; diff --git a/plugins/events-node/src/generated/models/ErrorError.model.ts b/plugins/events-node/src/generated/models/ErrorError.model.ts new file mode 100644 index 0000000000..4bcc60b660 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorError.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorError { + name: string; + message: string; +} diff --git a/plugins/events-node/src/generated/models/ErrorRequest.model.ts b/plugins/events-node/src/generated/models/ErrorRequest.model.ts new file mode 100644 index 0000000000..be62627814 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorRequest.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorRequest { + method: string; + url: string; +} diff --git a/plugins/events-node/src/generated/models/ErrorResponse.model.ts b/plugins/events-node/src/generated/models/ErrorResponse.model.ts new file mode 100644 index 0000000000..008f8b7348 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorResponse.model.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorResponse { + statusCode: number; +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts b/plugins/events-node/src/generated/models/Event.model.ts similarity index 58% rename from packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts rename to plugins/events-node/src/generated/models/Event.model.ts index a370b8f166..a196b42db7 100644 --- a/packages/cli/src/commands/onboard/discovery/analyzers/DefaultAnalysisOutputs.ts +++ b/plugins/events-node/src/generated/models/Event.model.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,17 @@ * limitations under the License. */ -import { AnalysisOutput, AnalysisOutputs } from './types'; +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** -export class DefaultAnalysisOutputs implements AnalysisOutputs { - readonly #outputs = new Map(); - - produce(output: AnalysisOutput) { - this.#outputs.set(output.entity.metadata.name, output); - } - - list() { - return Array.from(this.#outputs).map(([_, output]) => output); - } +export interface Event { + /** + * The topic that the event is published on + */ + topic: string; + /** + * The event payload + */ + payload: any | null; } diff --git a/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts new file mode 100644 index 0000000000..c9fc37f5bb --- /dev/null +++ b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { Event } from '../models/Event.model'; + +export interface GetSubscriptionEvents200Response { + events: Array; +} diff --git a/packages/cli/src/commands/onboard/discovery/analyzers/types.ts b/plugins/events-node/src/generated/models/ModelError.model.ts similarity index 51% rename from packages/cli/src/commands/onboard/discovery/analyzers/types.ts rename to plugins/events-node/src/generated/models/ModelError.model.ts index 57b6f2152b..88b8e76762 100644 --- a/packages/cli/src/commands/onboard/discovery/analyzers/types.ts +++ b/plugins/events-node/src/generated/models/ModelError.model.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,24 +14,15 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; -import { Repository } from '../providers/types'; +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { ErrorError } from '../models/ErrorError.model'; +import { ErrorRequest } from '../models/ErrorRequest.model'; +import { ErrorResponse } from '../models/ErrorResponse.model'; -export type AnalysisOutput = { - type: 'entity'; - path: string; - entity: Entity; -}; - -export interface AnalysisOutputs { - produce(output: AnalysisOutput): void; - list(): AnalysisOutput[]; -} - -export interface Analyzer { - name(): string; - analyzeRepository(options: { - repository: Repository; - output: AnalysisOutputs; - }): Promise; +export interface ModelError { + error: ErrorError; + request: ErrorRequest; + response: ErrorResponse; } diff --git a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts b/plugins/events-node/src/generated/models/PostEventRequest.model.ts similarity index 54% rename from packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts rename to plugins/events-node/src/generated/models/PostEventRequest.model.ts index 06f3189ba6..75289ed25b 100644 --- a/packages/cli/src/commands/onboard/discovery/providers/gitlab/GitlabFile.ts +++ b/plugins/events-node/src/generated/models/PostEventRequest.model.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,25 +14,15 @@ * limitations under the License. */ -import { RepositoryFile } from '../types'; +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { Event } from '../models/Event.model'; -/** - * A single file in a GitLab repository. - */ -export class GitlabFile implements RepositoryFile { - readonly #path: string; - readonly #content: string; - - constructor(path: string, content: string) { - this.#path = path; - this.#content = content; - } - - get path(): string { - return this.#path; - } - - async text(): Promise { - return this.#content; - } +export interface PostEventRequest { + event: Event; + /** + * The IDs of subscriptions that have already received this event + */ + notifiedSubscribers?: Array; } diff --git a/packages/cli/src/commands/plugin/test.ts b/plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts similarity index 60% rename from packages/cli/src/commands/plugin/test.ts rename to plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts index ca076ae6fc..3de35e0f81 100644 --- a/packages/cli/src/commands/plugin/test.ts +++ b/plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,13 @@ * limitations under the License. */ -import { OptionValues } from 'commander'; -import { run } from '../../lib/run'; +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** -export default async (opts: OptionValues) => { - const args = ['test']; - - if (opts.watch) { - args.push('--watch'); - } - if (opts.coverage) { - args.push('--coverage'); - } - - await run('web-scripts', args); -}; +export interface PutSubscriptionRequest { + /** + * The topics to subscribe to + */ + topics: Array; +} diff --git a/packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts b/plugins/events-node/src/generated/models/index.ts similarity index 56% rename from packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts rename to plugins/events-node/src/generated/models/index.ts index 25279b0949..5044f65fff 100644 --- a/packages/cli/src/commands/onboard/discovery/providers/github/GithubFile.ts +++ b/plugins/events-node/src/generated/models/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,11 @@ * limitations under the License. */ -import { RepositoryFile } from '../types'; - -export class GithubFile implements RepositoryFile { - readonly #path: string; - readonly #content: string; - - constructor(path: string, content: string) { - this.#path = path; - this.#content = content; - } - - get path(): string { - return this.#path; - } - - async text(): Promise { - return this.#content; - } -} +export * from '../models/ErrorError.model'; +export * from '../models/ErrorRequest.model'; +export * from '../models/ErrorResponse.model'; +export * from '../models/Event.model'; +export * from '../models/GetSubscriptionEvents200Response.model'; +export * from '../models/ModelError.model'; +export * from '../models/PostEventRequest.model'; +export * from '../models/PutSubscriptionRequest.model'; diff --git a/plugins/events-node/src/generated/pluginId.ts b/plugins/events-node/src/generated/pluginId.ts new file mode 100644 index 0000000000..36b5433cc3 --- /dev/null +++ b/plugins/events-node/src/generated/pluginId.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const pluginId = 'events'; diff --git a/plugins/events-node/src/generated/types/discovery.ts b/plugins/events-node/src/generated/types/discovery.ts new file mode 100644 index 0000000000..a7f87d3780 --- /dev/null +++ b/plugins/events-node/src/generated/types/discovery.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is a copy of the DiscoveryApi, to avoid importing core-plugin-api. + */ +export type DiscoveryApi = { + getBaseUrl(pluginId: string): Promise; +}; diff --git a/packages/cli/src/commands/versions/lint.ts b/plugins/events-node/src/generated/types/fetch.ts similarity index 74% rename from packages/cli/src/commands/versions/lint.ts rename to plugins/events-node/src/generated/types/fetch.ts index 66b5fc04c4..3de56c028e 100644 --- a/packages/cli/src/commands/versions/lint.ts +++ b/plugins/events-node/src/generated/types/fetch.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2023 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ * limitations under the License. */ -export default async () => { - throw new Error( - 'This command has been removed, please consider alternatives such as `yarn dedupe` instead.', - ); +/** + * This is a copy of FetchApi, to avoid importing core-plugin-api. + */ +export type FetchApi = { + fetch: typeof fetch; }; diff --git a/plugins/events-node/src/service.ts b/plugins/events-node/src/service.ts index af0e562c80..4f758f44e6 100644 --- a/plugins/events-node/src/service.ts +++ b/plugins/events-node/src/service.ts @@ -38,11 +38,23 @@ export const eventsServiceFactory = createServiceFactory({ deps: { pluginMetadata: coreServices.pluginMetadata, rootLogger: coreServices.rootLogger, + discovery: coreServices.discovery, + logger: coreServices.logger, + lifecycle: coreServices.lifecycle, + auth: coreServices.auth, }, async createRootContext({ rootLogger }) { return DefaultEventsService.create({ logger: rootLogger }); }, - async factory({ pluginMetadata }, eventsService) { - return eventsService.forPlugin(pluginMetadata.getId()); + async factory( + { pluginMetadata, discovery, logger, lifecycle, auth }, + eventsService, + ) { + return eventsService.forPlugin(pluginMetadata.getId(), { + discovery, + logger, + lifecycle, + auth, + }); }, }); diff --git a/plugins/scaffolder/report-alpha.api.md b/plugins/scaffolder/report-alpha.api.md index c8dc7c00e6..b71bbb1395 100644 --- a/plugins/scaffolder/report-alpha.api.md +++ b/plugins/scaffolder/report-alpha.api.md @@ -229,6 +229,8 @@ export const scaffolderTranslationRef: TranslationRef< readonly 'ongoingTask.showLogsButtonTitle': 'Show Logs'; readonly 'templateEditorForm.stepper.emptyText': 'There are no spec parameters in the template to preview.'; readonly 'templateTypePicker.title': 'Categories'; + readonly 'templateFormPage.title': 'Template Form Playground'; + readonly 'templateFormPage.subtitle': 'Edit, preview, and try out templates and template forms'; readonly 'templateEditorPage.title': 'Template Editor'; readonly 'templateEditorPage.subtitle': 'Edit, preview, and try out templates and template forms'; readonly 'templateEditorPage.dryRunResults.title': 'Dry-run results'; diff --git a/plugins/scaffolder/report.api.md b/plugins/scaffolder/report.api.md index ae8a440161..8c60eda7f5 100644 --- a/plugins/scaffolder/report.api.md +++ b/plugins/scaffolder/report.api.md @@ -614,6 +614,7 @@ export const scaffolderPlugin: BackstagePlugin< edit: SubRouteRef; editor: SubRouteRef; customFields: SubRouteRef; + templateForm: SubRouteRef; }, { registerComponent: ExternalRouteRef; diff --git a/plugins/scaffolder/src/components/Router/Router.tsx b/plugins/scaffolder/src/components/Router/Router.tsx index 27c2eede70..385a5f9fad 100644 --- a/plugins/scaffolder/src/components/Router/Router.tsx +++ b/plugins/scaffolder/src/components/Router/Router.tsx @@ -40,6 +40,7 @@ import { scaffolderListTaskRouteRef, scaffolderTaskRouteRef, selectedTemplateRouteRef, + templateFormRouteRef, } from '../../routes'; import { ErrorPage } from '@backstage/core-components'; @@ -54,6 +55,7 @@ import { TemplateListPage, TemplateWizardPage } from '../../next'; import { OngoingTask } from '../OngoingTask'; import { TemplatePage, + TemplateFormPage, TemplateEditorPage, CustomFieldsPage, } from '../../next/TemplateEditorPage'; @@ -169,11 +171,7 @@ export const Router = (props: PropsWithChildren) => { path={editRouteRef.path} element={ - + } /> @@ -185,6 +183,18 @@ export const Router = (props: PropsWithChildren) => { } /> + + + + } + /> } /> []; - layouts?: LayoutOptions[]; - formProps?: FormProps; -} - -export function TemplateEditorPage(props: TemplateEditorPageProps) { - const [selection, setSelection] = useState(); +export function TemplateEditorPage() { const navigate = useNavigate(); const actionsLink = useRouteRef(actionsRouteRef); const tasksLink = useRouteRef(scaffolderListTaskRouteRef); const createLink = useRouteRef(rootRouteRef); const editorLink = useRouteRef(editorRouteRef); const customFieldsLink = useRouteRef(customFieldsRouteRef); + const templateFormLink = useRouteRef(templateFormRouteRef); const { t } = useTranslationRef(scaffolderTranslationRef); const scaffolderPageContextMenuProps = { @@ -80,19 +50,14 @@ export function TemplateEditorPage(props: TemplateEditorPageProps) { onCreateClicked: () => navigate(createLink()), }; - let content: JSX.Element | null = null; - if (selection?.type === 'form') { - content = ( - setSelection(undefined)} - layouts={props.layouts} - formProps={props.formProps} - /> - ); - } else { - content = ( + return ( + +
+ +
{ @@ -111,25 +76,13 @@ export function TemplateEditorPage(props: TemplateEditorPageProps) { }) .catch(() => {}); } else if (option === 'form') { - setSelection({ type: 'form' }); + navigate(templateFormLink()); } else if (option === 'field-explorer') { navigate(customFieldsLink()); } }} /> - ); - } - - return ( - -
- -
- {content} ); } diff --git a/plugins/scaffolder/src/next/TemplateEditorPage/TemplateEditorToolbar.tsx b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateEditorToolbar.tsx index ef4ac95c3c..1d89616964 100644 --- a/plugins/scaffolder/src/next/TemplateEditorPage/TemplateEditorToolbar.tsx +++ b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateEditorToolbar.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { useState } from 'react'; +import React, { ReactNode, useState } from 'react'; import { makeStyles } from '@material-ui/core/styles'; import AppBar from '@material-ui/core/AppBar'; @@ -49,18 +49,29 @@ const useStyles = makeStyles( }, toolbar: { display: 'grid', - justifyItems: 'flex-end', + gridTemplateColumns: 'auto 1fr', + gridGap: theme.spacing(1), padding: theme.spacing(0, 1), backgroundColor: theme.palette.background.paper, }, + toolbarCustomActions: { + display: 'grid', + alignItems: 'center', + gridAutoFlow: 'Column', + gridGap: theme.spacing(1), + }, + toolbarDefaultActions: { + justifySelf: 'end', + }, }), { name: 'ScaffolderTemplateEditorToolbar' }, ); export function TemplateEditorToolbar(props: { + children?: ReactNode; fieldExtensions?: FieldExtensionOptions[]; }) { - const { fieldExtensions } = props; + const { children, fieldExtensions } = props; const classes = useStyles(); const [showFieldsDrawer, setShowFieldsDrawer] = useState(false); const [showActionsDrawer, setShowActionsDrawer] = useState(false); @@ -69,14 +80,19 @@ export function TemplateEditorToolbar(props: { return ( - +
{children}
+ - - diff --git a/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPage.tsx b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPage.tsx new file mode 100644 index 0000000000..c79688c8b7 --- /dev/null +++ b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPage.tsx @@ -0,0 +1,76 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useCallback } from 'react'; +import { useNavigate } from 'react-router-dom'; + +import { makeStyles } from '@material-ui/core/styles'; + +import { Page, Header, Content } from '@backstage/core-components'; +import { useRouteRef } from '@backstage/core-plugin-api'; +import { useTranslationRef } from '@backstage/core-plugin-api/alpha'; +import { + FormProps, + LayoutOptions, + FieldExtensionOptions, +} from '@backstage/plugin-scaffolder-react'; + +import { editRouteRef } from '../../routes'; +import { scaffolderTranslationRef } from '../../translation'; + +import { TemplateFormPreviewer } from './TemplateFormPreviewer'; + +const useStyles = makeStyles({ + root: { + padding: 0, + }, +}); + +interface TemplateFormPageProps { + layouts?: LayoutOptions[]; + formProps?: FormProps; + fieldExtensions?: FieldExtensionOptions[]; + defaultPreviewTemplate?: string; +} + +export function TemplateFormPage(props: TemplateFormPageProps) { + const classes = useStyles(); + const navigate = useNavigate(); + const editLink = useRouteRef(editRouteRef); + const { t } = useTranslationRef(scaffolderTranslationRef); + + const handleClose = useCallback(() => { + navigate(editLink()); + }, [navigate, editLink]); + + return ( + +
+ + + + + ); +} diff --git a/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPreviewer.tsx b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPreviewer.tsx index 4f79b7f5c2..e0c40a98be 100644 --- a/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPreviewer.tsx +++ b/plugins/scaffolder/src/next/TemplateEditorPage/TemplateFormPreviewer.tsx @@ -13,18 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity } from '@backstage/catalog-model'; import { alertApiRef, useApi } from '@backstage/core-plugin-api'; import { catalogApiRef, humanizeEntityRef, } from '@backstage/plugin-catalog-react'; -import FormControl from '@material-ui/core/FormControl'; -import IconButton from '@material-ui/core/IconButton'; -import InputLabel from '@material-ui/core/InputLabel'; import LinearProgress from '@material-ui/core/LinearProgress'; -import MenuItem from '@material-ui/core/MenuItem'; +import Paper from '@material-ui/core/Paper'; +import FormControl from '@material-ui/core/FormControl'; +import Input from '@material-ui/core/Input'; import Select from '@material-ui/core/Select'; +import Tooltip from '@material-ui/core/Tooltip'; +import IconButton from '@material-ui/core/IconButton'; +import MenuItem from '@material-ui/core/MenuItem'; import { makeStyles } from '@material-ui/core/styles'; import CloseIcon from '@material-ui/icons/Close'; import React, { useCallback, useState } from 'react'; @@ -35,6 +38,7 @@ import { FieldExtensionOptions, FormProps, } from '@backstage/plugin-scaffolder-react'; +import { TemplateEditorToolbar } from './TemplateEditorToolbar'; import { TemplateEditorForm } from './TemplateEditorForm'; import { TemplateEditorTextArea } from './TemplateEditorTextArea'; import { useTranslationRef } from '@backstage/core-plugin-api/alpha'; @@ -94,27 +98,40 @@ export type ScaffolderTemplateFormPreviewerClassKey = const useStyles = makeStyles( theme => ({ root: { + height: '100%', gridArea: 'pageContent', display: 'grid', gridTemplateAreas: ` - "controls controls" + "toolbar toolbar" "textArea preview" `, gridTemplateRows: 'auto 1fr', gridTemplateColumns: '1fr 1fr', }, - controls: { - gridArea: 'controls', - display: 'flex', - flexFlow: 'row nowrap', - alignItems: 'center', - margin: theme.spacing(1), + toolbar: { + gridArea: 'toolbar', }, textArea: { gridArea: 'textArea', + height: '100%', }, preview: { gridArea: 'preview', + position: 'relative', + borderLeft: `1px solid ${theme.palette.divider}`, + backgroundColor: theme.palette.background.default, + }, + scroll: { + position: 'absolute', + top: 0, + left: 0, + right: 0, + bottom: 0, + padding: theme.spacing(1), + }, + formControl: { + minWidth: 120, + maxWidth: 300, }, }), { name: 'ScaffolderTemplateFormPreviewer' }, @@ -137,8 +154,9 @@ export const TemplateFormPreviewer = ({ const { t } = useTranslationRef(scaffolderTranslationRef); const alertApi = useApi(alertApiRef); const catalogApi = useApi(catalogApiRef); - const [selectedTemplate, setSelectedTemplate] = useState(''); const [errorText, setErrorText] = useState(); + const [selectedTemplate, setSelectedTemplate] = + useState(null); const [templateOptions, setTemplateOptions] = useState([]); const [templateYaml, setTemplateYaml] = useState(defaultPreviewTemplate); @@ -188,29 +206,45 @@ export const TemplateFormPreviewer = ({ return ( <> {loading && } -
-
- - - {t('templateEditorPage.templateFormPreviewer.title')} - - - - - - - + +
+ + + + + + + + } + renderValue={selected => { + if (!selected) { + return t('templateEditorPage.templateFormPreviewer.title'); + } + return (selected as Entity).metadata.title; + }} + inputProps={{ + 'aria-label': t( + 'templateEditorPage.templateFormPreviewer.title', + ), + }} + > + {templateOptions.map((option, index) => ( + + {option.label} + + ))} + + +
- +
+ +
-
+ ); }; diff --git a/plugins/scaffolder/src/next/TemplateEditorPage/index.ts b/plugins/scaffolder/src/next/TemplateEditorPage/index.ts index 45e85f8a83..84ab74da98 100644 --- a/plugins/scaffolder/src/next/TemplateEditorPage/index.ts +++ b/plugins/scaffolder/src/next/TemplateEditorPage/index.ts @@ -15,6 +15,7 @@ */ export { TemplatePage } from './TemplatePage'; +export { TemplateFormPage } from './TemplateFormPage'; export { TemplateEditorPage } from './TemplateEditorPage'; export { CustomFieldsPage } from './CustomFieldsPage'; export type { ScaffolderCustomFieldExplorerClassKey } from './CustomFieldExplorer'; diff --git a/plugins/scaffolder/src/plugin.tsx b/plugins/scaffolder/src/plugin.tsx index bf4d3a41ac..8b24e86069 100644 --- a/plugins/scaffolder/src/plugin.tsx +++ b/plugins/scaffolder/src/plugin.tsx @@ -70,6 +70,7 @@ import { editRouteRef, editorRouteRef, customFieldsRouteRef, + templateFormRouteRef, } from './routes'; import { MyGroupsPicker, @@ -111,6 +112,7 @@ export const scaffolderPlugin = createPlugin({ edit: editRouteRef, editor: editorRouteRef, customFields: customFieldsRouteRef, + templateForm: templateFormRouteRef, }, externalRoutes: { registerComponent: registerComponentRouteRef, diff --git a/plugins/scaffolder/src/routes.ts b/plugins/scaffolder/src/routes.ts index 3bd4435078..55344a8b0a 100644 --- a/plugins/scaffolder/src/routes.ts +++ b/plugins/scaffolder/src/routes.ts @@ -86,7 +86,13 @@ export const editorRouteRef = createSubRouteRef({ }); export const customFieldsRouteRef = createSubRouteRef({ - id: 'scaffolder/edit', + id: 'scaffolder/customFields', parent: rootRouteRef, path: '/custom-fields', }); + +export const templateFormRouteRef = createSubRouteRef({ + id: 'scaffolder/editorForm', + parent: rootRouteRef, + path: '/template-form', +}); diff --git a/plugins/scaffolder/src/translation.ts b/plugins/scaffolder/src/translation.ts index bb0dc3ebad..99b9a69afa 100644 --- a/plugins/scaffolder/src/translation.ts +++ b/plugins/scaffolder/src/translation.ts @@ -192,6 +192,10 @@ export const scaffolderTranslationRef = createTranslationRef({ templateTypePicker: { title: 'Categories', }, + templateFormPage: { + title: 'Template Form Playground', + subtitle: 'Edit, preview, and try out templates and template forms', + }, templateEditorPage: { title: 'Template Editor', subtitle: 'Edit, preview, and try out templates and template forms', diff --git a/plugins/techdocs/report-alpha.api.md b/plugins/techdocs/report-alpha.api.md index 075cb34bfa..faaf6d6a83 100644 --- a/plugins/techdocs/report-alpha.api.md +++ b/plugins/techdocs/report-alpha.api.md @@ -229,7 +229,9 @@ const _default: FrontendPlugin< ConfigurableExtensionDataRef< React_2.JSX.Element, 'core.reactElement', - {} + { + optional: true; + } >, { singleton: true; diff --git a/plugins/techdocs/src/alpha.tsx b/plugins/techdocs/src/alpha.tsx index 2299b56be9..e0e1c1761e 100644 --- a/plugins/techdocs/src/alpha.tsx +++ b/plugins/techdocs/src/alpha.tsx @@ -157,10 +157,13 @@ const techDocsReaderPage = PageBlueprint.make({ */ const techDocsEntityContent = EntityContentBlueprint.makeWithOverrides({ inputs: { - emptyState: createExtensionInput([coreExtensionData.reactElement], { - singleton: true, - optional: true, - }), + emptyState: createExtensionInput( + [coreExtensionData.reactElement.optional()], + { + singleton: true, + optional: true, + }, + ), }, factory(originalFactory, context) { return originalFactory( diff --git a/yarn.lock b/yarn.lock index e3f5be5bd2..1aab23e1bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6141,16 +6141,23 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-events-backend@workspace:plugins/events-backend" dependencies: + "@backstage/backend-app-api": "workspace:^" "@backstage/backend-common": ^0.25.0 + "@backstage/backend-defaults": "workspace:^" + "@backstage/backend-openapi-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/plugin-events-backend-test-utils": "workspace:^" "@backstage/plugin-events-node": "workspace:^" + "@backstage/repo-tools": "workspace:^" + "@backstage/types": "workspace:^" "@types/express": ^4.17.6 express: ^4.17.1 express-promise-router: ^4.1.0 + knex: ^3.0.0 supertest: ^7.0.0 winston: ^3.2.1 languageName: unknown @@ -6164,6 +6171,10 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/types": "workspace:^" + cross-fetch: ^4.0.0 + uri-template: ^2.0.0 languageName: unknown linkType: soft @@ -26436,6 +26447,7 @@ __metadata: "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" "@backstage/plugin-devtools-backend": "workspace:^" + "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" "@backstage/plugin-notifications-backend": "workspace:^" "@backstage/plugin-permission-backend": "workspace:^"