From e8a1c1afe21b78a8b299e1135af99cc766248479 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Wed, 13 Oct 2021 16:24:07 +0200 Subject: [PATCH 01/43] Don't require a validation pattern for the jenkins base URL Signed-off-by: Dominik Henneke --- .changeset/flat-dodos-bake.md | 5 +++++ plugins/jenkins-backend/config.d.ts | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 .changeset/flat-dodos-bake.md diff --git a/.changeset/flat-dodos-bake.md b/.changeset/flat-dodos-bake.md new file mode 100644 index 0000000000..f6b52aac1a --- /dev/null +++ b/.changeset/flat-dodos-bake.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins-backend': patch +--- + +Don't require a validation pattern for the Jenkins base URL. diff --git a/plugins/jenkins-backend/config.d.ts b/plugins/jenkins-backend/config.d.ts index bfb519bb58..1fc41c1592 100644 --- a/plugins/jenkins-backend/config.d.ts +++ b/plugins/jenkins-backend/config.d.ts @@ -17,7 +17,6 @@ export interface Config { jenkins?: { /** * Default instance baseUrl, can be specified on a named instance called "default" - * @pattern "^https?://" */ baseUrl?: string; /** @@ -39,9 +38,6 @@ export interface Config { */ name: string; - /** - * @pattern "^https?://" - */ baseUrl: string; username: string; /** @visibility secret */ From 0611f3b3e2d5e4e742abbf17d6e17c1bd0c95541 Mon Sep 17 00:00:00 2001 From: Matto Date: Mon, 27 Sep 2021 23:41:24 +1000 Subject: [PATCH 02/43] Read config from remote config server Signed-off-by: Matto --- .changeset/giant-years-help.md | 8 + packages/backend-common/src/config.ts | 10 +- packages/cli/package.json | 3 +- packages/cli/src/lib/config.ts | 16 +- packages/config-loader/api-report.md | 34 +++- packages/config-loader/package.json | 10 +- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/loader.test.ts | 125 +++++++++++- packages/config-loader/src/loader.ts | 232 +++++++++++++++++++--- packages/integration/api-report.md | 5 + packages/integration/src/index.ts | 1 + yarn.lock | 69 ++++++- 12 files changed, 461 insertions(+), 54 deletions(-) create mode 100644 .changeset/giant-years-help.md diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md new file mode 100644 index 0000000000..f0ca12ef19 --- /dev/null +++ b/.changeset/giant-years-help.md @@ -0,0 +1,8 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/integration': patch +--- + +Reading app config from a remote server diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 96907870cb..afdb4fd0f7 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -19,7 +19,8 @@ import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader, JsonValue } from '@backstage/config'; -import { loadConfig } from '@backstage/config-loader'; +import { ConfigTarget, loadConfig } from '@backstage/config-loader'; +import { isValidUrl } from '@backstage/integration'; class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -117,7 +118,10 @@ export async function loadBackendConfig(options: { argv: string[]; }): Promise { const args = parseArgs(options.argv); - const configPaths: string[] = [args.config ?? []].flat(); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); const config = new ObservableConfigProxy(options.logger); @@ -126,7 +130,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configPaths.map(opt => resolvePath(opt)), + configTargets: configTargets, watch: { onChange(newConfigs) { options.logger.info( diff --git a/packages/cli/package.json b/packages/cli/package.json index 0dc4dc3cf7..47475780a6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,8 +31,10 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.3", + "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", + "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -117,7 +119,6 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.9.4", "@backstage/config": "^0.1.10", "@backstage/core-components": "^0.5.0", "@backstage/core-plugin-api": "^0.1.8", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index a93f040122..218771beb6 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; +import { + ConfigTarget, + loadConfig, + loadConfigSchema, +} from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; +import { isValidUrl } from '@backstage/integration'; type Options = { args: string[]; @@ -26,7 +31,12 @@ type Options = { }; export async function loadCliConfig(options: Options) { - const configPaths = options.args.map(arg => paths.resolveTarget(arg)); + const configTargets: ConfigTarget[] = []; + options.args.forEach(arg => { + if (!isValidUrl(arg)) { + configTargets.push({ path: paths.resolveTarget(arg) }); + } + }); // Consider all packages in the monorepo when loading in config const { Project } = require('@lerna/project'); @@ -46,7 +56,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, - configPaths, + configTargets: configTargets, }); // printing to stderr to not clobber stdout in case the cli command diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index bb3ae50d23..bb36bbe750 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = { withFilteredKeys?: boolean; }; +// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ConfigTarget = + | { + path: string; + } + | { + url: string; + }; + // @public export type ConfigVisibility = 'frontend' | 'backend' | 'secret'; @@ -35,13 +46,11 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public (undocumented) export type LoadConfigOptions = { configRoot: string; - configPaths: string[]; + configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; - watch?: { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; - }; + remote?: Remote; + watch?: Watch; }; // @public @@ -66,6 +75,13 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; +// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Remote = { + reloadIntervalSeconds: number; +}; + // @public export type TransformFunc = ( value: T, @@ -73,4 +89,12 @@ export type TransformFunc = ( visibility: ConfigVisibility; }, ) => T | undefined; + +// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Watch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; ``` diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 2f2878799e..d20c77bedc 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,8 +30,10 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", + "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", @@ -39,8 +41,10 @@ "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", "typescript-json-schema": "^0.50.1", + "uuid": "^8.3.2", "yaml": "^1.9.2", - "yup": "^0.29.3" + "yup": "^0.29.3", + "node-fetch": "2.6.5" }, "devDependencies": { "@types/jest": "^26.0.7", @@ -48,7 +52,9 @@ "@types/mock-fs": "^4.10.0", "@types/node": "^14.14.32", "@types/yup": "^0.29.8", - "mock-fs": "^5.1.0" + "mock-fs": "^5.1.0", + "fetch-mock-jest": "1.5.1", + "fetch-mock": "^9.11.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 0e66c59ec4..e0eafc00bc 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -30,4 +30,4 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './loader'; +export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3f51e76936..190904a870 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,12 +18,33 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; +import { v4 as uuidv4 } from 'uuid'; + +const fetchMock = require('fetch-mock').sandbox(); +const nodeFetch = require('node-fetch'); + +nodeFetch.default = fetchMock; describe('loadConfig', () => { beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + method: 'GET', + }, + { + body: `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + ); + mockFs({ '/root/app-config.yaml': ` app: @@ -66,6 +87,7 @@ describe('loadConfig', () => { }); afterEach(() => { + fetchMock.restore(); mockFs.restore(); }); @@ -73,7 +95,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], env: 'production', }), ).resolves.toEqual([ @@ -90,11 +112,37 @@ describe('loadConfig', () => { ]); }); + it('load config from remote path', async () => { + const configUrl = 'https://some.domain.io/app-config.yaml'; + + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + env: 'production', + remote: { + reloadIntervalSeconds: 30, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), ).resolves.toEqual([ @@ -115,9 +163,9 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', + configTargets: [ + { path: '/root/app-config.yaml' }, + { path: '/root/app-config.development.yaml' }, ], env: 'development', }), @@ -155,7 +203,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.substitute.yaml'], + configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), ).resolves.toEqual([ @@ -180,7 +228,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: onChange.resolve, stopSignal: stopSignal.promise, @@ -218,12 +266,73 @@ describe('loadConfig', () => { stopSignal.resolve(); }); + it('watches remote config urls', async () => { + const onChange = defer(); + const stopSignal = defer(); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + remote: { + reloadIntervalSeconds: 1, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + }, + { + body: `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + { overwriteRoutes: true }, + ); + + await expect(onChange.promise).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'NEW ReMOTe ExaMPLe App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + it('stops watching config files', async () => { const stopSignal = defer(); await loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: () => { expect('not').toBe('called'); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 315b44c4ee..fb8d0aab33 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,23 +17,67 @@ import fs from 'fs-extra'; import yaml from 'yaml'; import chokidar from 'chokidar'; -import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; import { AppConfig } from '@backstage/config'; import { applyConfigTransforms, - readEnvConfig, createIncludeTransform, createSubstitutionTransform, + EnvFunc, + readEnvConfig, } from './lib'; -import { EnvFunc } from './lib/transform/types'; +import fetch from 'node-fetch'; +import { isValidUrl } from '@backstage/integration'; +export type ConfigTarget = { path: string } | { url: string }; + +export type Watch = { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; +}; + +export type Remote = { + /** + * An optional remote config reloading period, in seconds + */ + reloadIntervalSeconds: number; +}; + +export type RemoteConfigProp = { + /** + * URL of the remote config + */ + url: string; + + /** + * Contents of the remote config + */ + content: string | null; + + /** + * An optional new ETag header value. Used when checking for updated config. + */ + newETag?: string; + + /** + * An optional old ETag header value. Used when checking for updated config + */ + oldETag?: string; +}; /** @public */ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; - // Absolute paths to load config files from. Configs from earlier paths have lower priority. - configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. + configTargets: ConfigTarget[]; /** @deprecated This option has been removed */ env?: string; @@ -45,22 +89,19 @@ export type LoadConfigOptions = { */ experimentalEnvFunc?: EnvFunc; + /** + * An optional remote config + */ + remote?: Remote; + /** * An optional configuration that enables watching of config files. */ - watch?: { - /** - * A listener that is called when a config file is changed. - */ - onChange: (configs: AppConfig[]) => void; - - /** - * An optional signal that stops the watcher once the promise resolves. - */ - stopSignal?: Promise; - }; + watch?: Watch; }; +const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; + /** * Load configuration data. * @@ -69,12 +110,30 @@ export type LoadConfigOptions = { export async function loadConfig( options: LoadConfigOptions, ): Promise { - const { configRoot, experimentalEnvFunc: envFunc, watch } = options; - const configPaths = options.configPaths.slice(); + const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options; + + const configPaths: string[] = options.configTargets + .slice() + .filter((e): e is { path: string } => e.hasOwnProperty('path')) + .map(configTarget => configTarget.path); + + let configUrls: string[] = options.configTargets + .slice() + .filter((e): e is { url: string } => e.hasOwnProperty('url')) + .map(configTarget => configTarget.url); + + const remoteConfigProps: RemoteConfigProp[] = []; + + if (remote === undefined && configUrls.length > 0) { + console.warn( + `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, + ); + configUrls = []; + } // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` - if (configPaths.length === 0) { + if (configPaths.length === 0 && configUrls.length === 0) { configPaths.push(resolvePath(configRoot, 'app-config.yaml')); const localConfig = resolvePath(configRoot, 'app-config.local.yaml'); @@ -99,6 +158,7 @@ export async function loadConfig( const input = yaml.parse(await readFile(configPath)); const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ createIncludeTransform(env, readFile, substitutionTransform), substitutionTransform, @@ -110,7 +170,56 @@ export async function loadConfig( return configs; }; - let fileConfigs; + const loadRemoteConfigFiles = async () => { + const configs: AppConfig[] = []; + + const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { + const response = await fetch(remoteConfigProp.url); + if (!response.ok) { + throw new Error( + `Could not read config file at ${remoteConfigProp.url}`, + ); + } + + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + remoteConfigProp.content = await response.text(); + + return remoteConfigProp; + }; + + for (let i = 0; i < configUrls.length; i++) { + const remoteConfigProp = await readConfigFromUrl({ + url: configUrls[i], + content: null, + }); + + if (!isValidUrl(remoteConfigProp.url)) { + throw new Error( + `Config load path is not valid: '${remoteConfigProp.url}'`, + ); + } + + const dir = configRoot; + if (!remoteConfigProp.content) { + throw new Error(`Config is not valid`); + } + const input = yaml.parse(remoteConfigProp.content); + const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ + substitutionTransform, + ]); + + configs.push({ data, context: remoteConfigProp.url }); + + remoteConfigProps.push(remoteConfigProp); + } + + return configs; + }; + + let fileConfigs: AppConfig[]; try { fileConfigs = await loadConfigFiles(); } catch (error) { @@ -119,15 +228,23 @@ export async function loadConfig( ); } + let remoteConfigs: AppConfig[] = []; + if (remote) { + try { + remoteConfigs = await loadRemoteConfigFiles(); + } catch (error) { + throw new Error(`Failed to read remote configuration file, ${error}`); + } + } + const envConfigs = await readEnvConfig(process.env); - // Set up config file watching if requested by the caller - if (watch) { - let currentSerializedConfig = JSON.stringify(fileConfigs); - + const watchConfigFile = (watchProp: Watch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); + + let currentSerializedConfig = JSON.stringify(fileConfigs); watcher.on('change', async () => { try { const newConfigs = await loadConfigFiles(); @@ -138,18 +255,77 @@ export async function loadConfig( } currentSerializedConfig = newSerializedConfig; - watch.onChange([...newConfigs, ...envConfigs]); + watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]); } catch (error) { console.error(`Failed to reload configuration files, ${error}`); } }); - if (watch.stopSignal) { - watch.stopSignal.then(() => { + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { watcher.close(); }); } + }; + + const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { + const requestProps = { method: 'HEAD' }; + const { headers } = await fetch(remoteConfigProp.url, requestProps); + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + + return ( + remoteConfigProp.oldETag !== undefined && + remoteConfigProp.newETag !== undefined && + remoteConfigProp.oldETag !== remoteConfigProp.newETag + ); + }; + + let handle: NodeJS.Timeout | undefined; + try { + handle = setInterval(async () => { + console.info(`Checking for config update`); + for (const remoteConfigProp of remoteConfigProps) { + if (await hasConfigChanged(remoteConfigProp)) { + console.info(`Remote config change, reloading config ...`); + const newRemoteConfigs = await loadRemoteConfigFiles(); + watchProp.onChange([ + ...newRemoteConfigs, + ...fileConfigs, + ...envConfigs, + ]); + console.info(`Remote config reloaded`); + break; + } + } + }, remoteProp.reloadIntervalSeconds * 1000); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { + if (handle !== undefined) { + console.info(`Stopping remote config watch`); + clearInterval(handle); + handle = undefined; + } + }); + } + }; + + // Set up config file watching if requested by the caller + if (watch) { + watchConfigFile(watch); } - return [...fileConfigs, ...envConfigs]; + if (watch && remote) { + watchRemoteConfig(watch, remote); + } + + return remote + ? [...remoteConfigs, ...fileConfigs, ...envConfigs] + : [...fileConfigs, ...envConfigs]; } diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 01f84383fe..e07f61eea6 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,6 +347,11 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function isValidUrl(url: string): boolean; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 3d5f336e8d..b8992f52d2 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,6 +27,7 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; +export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; diff --git a/yarn.lock b/yarn.lock index 7657498ff7..87991e1bb5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11336,6 +11336,11 @@ core-js@^2.5.0: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== +core-js@^3.0.0: + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== + core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14037,6 +14042,29 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== +fetch-mock-jest@1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" + integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== + dependencies: + fetch-mock "^9.11.0" + +fetch-mock@^9.11.0: + version "9.11.0" + resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" + integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== + dependencies: + "@babel/core" "^7.0.0" + "@babel/runtime" "^7.0.0" + core-js "^3.0.0" + debug "^4.1.1" + glob-to-regexp "^0.4.0" + is-subset "^0.1.1" + lodash.isequal "^4.5.0" + path-to-regexp "^2.2.1" + querystring "^0.2.0" + whatwg-url "^6.5.0" + fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -14867,7 +14895,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -16851,6 +16879,11 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" +is-subset@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" + integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= + is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18719,7 +18752,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0: +lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -20192,6 +20225,13 @@ node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -21526,6 +21566,11 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" +path-to-regexp@^2.2.1: + version "2.4.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -26576,6 +26621,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + "traverse@>=0.3.0 <0.4": version "0.3.9" resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" @@ -27702,6 +27752,11 @@ web-namespaces@^1.0.0: resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz#bc98a3de60dadd7faefc403d1076d529f5e030ec" integrity sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -27954,7 +28009,15 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^6.4.1: +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +whatwg-url@^6.4.1, whatwg-url@^6.5.0: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From befe7d122f5095122740d0878058ad684219c182 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 19 Oct 2021 23:18:19 +1100 Subject: [PATCH 03/43] Added documentation Signed-off-by: Matto --- docs/conf/writing.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4288d23b17..86b844484b 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,7 +1,6 @@ --- -id: writing -title: Writing Backstage Configuration Files -description: Documentation on Writing Backstage Configuration Files +id: writing title: Writing Backstage Configuration Files description: Documentation on Writing +Backstage Configuration Files --- ## File Format @@ -67,13 +66,15 @@ production build. ## Configuration Files -It is possible to have multiple configuration files, both to support different -environments, but also to define configuration that is local to specific -packages. The configuration files to load are selected using a `--config ` -flag, and it is possible to load any number of files. Paths are relative to the -working directory of the executed process, for example `package/backend`. This -means that to select a config file in the repo root when running the backend, -you would use `--config ../../my-config.yaml`. +It is possible to have multiple configuration files (bundled and/or remote), +both to support different environments, but also to define configuration that is +local to specific packages. The configuration files to load are selected using a +`--config ` flag, and it is possible to load any number of +files. Paths are relative to the working directory of the executed process, for +example `package/backend`. This means that to select a config file in the repo +root when running the backend, you would use `--config ../../my-config.yaml`, +and for config file on a config server you would use +`--config https://some.domain.io/app-config.yaml` If no `config` flags are specified, the default behavior is to load `app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root. @@ -85,7 +86,7 @@ are NOT loaded. To include them you need to explicitly include them with a flag, for example: ```shell -yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml +yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml --config https://some.domain.io/app-config.yaml ``` All loaded configuration files are merged together using the following rules: From 74ceaa64175d5eecd33577a735ce447081b7f0bc Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:20:36 +1100 Subject: [PATCH 04/43] Remove the dependency on Etag headers for reloading config Signed-off-by: Matto --- docs/conf/writing.md | 5 +- packages/cli/package.json | 2 +- packages/config-loader/package.json | 4 +- packages/config-loader/src/loader.test.ts | 79 +++++++++-------- packages/config-loader/src/loader.ts | 102 ++++++---------------- yarn.lock | 7 -- 6 files changed, 73 insertions(+), 126 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 86b844484b..a0e29820ed 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,6 +1,7 @@ --- -id: writing title: Writing Backstage Configuration Files description: Documentation on Writing -Backstage Configuration Files +id: writing +title: Writing Backstage Configuration Files +description: Documentation on Writing Backstage Configuration Files --- ## File Format diff --git a/packages/cli/package.json b/packages/cli/package.json index 47475780a6..519b45a5ad 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -34,7 +34,6 @@ "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", - "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -119,6 +118,7 @@ "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-common": "^0.9.4", "@backstage/config": "^0.1.10", "@backstage/core-components": "^0.5.0", "@backstage/core-plugin-api": "^0.1.8", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index d20c77bedc..80e4121971 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -41,7 +41,6 @@ "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", "typescript-json-schema": "^0.50.1", - "uuid": "^8.3.2", "yaml": "^1.9.2", "yup": "^0.29.3", "node-fetch": "2.6.5" @@ -53,8 +52,7 @@ "@types/node": "^14.14.32", "@types/yup": "^0.29.8", "mock-fs": "^5.1.0", - "fetch-mock-jest": "1.5.1", - "fetch-mock": "^9.11.0" + "msw": "^0.29.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 190904a870..e8b5a8828b 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,33 +18,47 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; -import { v4 as uuidv4 } from 'uuid'; - -const fetchMock = require('fetch-mock').sandbox(); -const nodeFetch = require('node-fetch'); - -nodeFetch.default = fetchMock; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; describe('loadConfig', () => { + const server = setupServer(); + const initialLoaderHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + const reloadHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + beforeAll(() => server.listen()); + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - method: 'GET', - }, - { - body: `app: - title: Remote Example App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - ); - mockFs({ '/root/app-config.yaml': ` app: @@ -87,10 +101,12 @@ describe('loadConfig', () => { }); afterEach(() => { - fetchMock.restore(); mockFs.restore(); + server.resetHandlers(); }); + afterAll(() => server.close()); + it('load config from default path', async () => { await expect( loadConfig({ @@ -113,6 +129,8 @@ describe('loadConfig', () => { }); it('load config from remote path', async () => { + server.use(initialLoaderHandler); + const configUrl = 'https://some.domain.io/app-config.yaml'; await expect( @@ -267,6 +285,8 @@ describe('loadConfig', () => { }); it('watches remote config urls', async () => { + server.use(initialLoaderHandler); + const onChange = defer(); const stopSignal = defer(); @@ -296,20 +316,7 @@ describe('loadConfig', () => { }, ]); - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - }, - { - body: `app: - title: NEW ReMOTe ExaMPLe App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - { overwriteRoutes: true }, - ); + server.use(reloadHandler); await expect(onChange.promise).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index fb8d0aab33..b853d30d8a 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -50,27 +50,6 @@ export type Remote = { reloadIntervalSeconds: number; }; -export type RemoteConfigProp = { - /** - * URL of the remote config - */ - url: string; - - /** - * Contents of the remote config - */ - content: string | null; - - /** - * An optional new ETag header value. Used when checking for updated config. - */ - newETag?: string; - - /** - * An optional old ETag header value. Used when checking for updated config - */ - oldETag?: string; -}; /** @public */ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -100,8 +79,6 @@ export type LoadConfigOptions = { watch?: Watch; }; -const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; - /** * Load configuration data. * @@ -117,18 +94,15 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); - let configUrls: string[] = options.configTargets + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) .map(configTarget => configTarget.url); - const remoteConfigProps: RemoteConfigProp[] = []; - if (remote === undefined && configUrls.length > 0) { - console.warn( + throw new Error( `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, ); - configUrls = []; } // If no paths are provided, we default to reading @@ -173,47 +147,32 @@ export async function loadConfig( const loadRemoteConfigFiles = async () => { const configs: AppConfig[] = []; - const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { - const response = await fetch(remoteConfigProp.url); + const readConfigFromUrl = async (url: string) => { + const response = await fetch(url); if (!response.ok) { - throw new Error( - `Could not read config file at ${remoteConfigProp.url}`, - ); + throw new Error(`Could not read config file at ${url}`); } - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - remoteConfigProp.content = await response.text(); - - return remoteConfigProp; + return await response.text(); }; for (let i = 0; i < configUrls.length; i++) { - const remoteConfigProp = await readConfigFromUrl({ - url: configUrls[i], - content: null, - }); - - if (!isValidUrl(remoteConfigProp.url)) { - throw new Error( - `Config load path is not valid: '${remoteConfigProp.url}'`, - ); + const configUrl = configUrls[i]; + if (!isValidUrl(configUrl)) { + throw new Error(`Config load path is not valid: '${configUrl}'`); } - const dir = configRoot; - if (!remoteConfigProp.content) { + const remoteConfigContent = await readConfigFromUrl(configUrl); + if (!remoteConfigContent) { throw new Error(`Config is not valid`); } - const input = yaml.parse(remoteConfigProp.content); + const configYaml = yaml.parse(remoteConfigContent); const substitutionTransform = createSubstitutionTransform(env); - const data = await applyConfigTransforms(dir, input, [ + const data = await applyConfigTransforms(configRoot, configYaml, [ substitutionTransform, ]); - configs.push({ data, context: remoteConfigProp.url }); - - remoteConfigProps.push(remoteConfigProp); + configs.push({ data, context: configUrl }); } return configs; @@ -269,17 +228,12 @@ export async function loadConfig( }; const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { - const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { - const requestProps = { method: 'HEAD' }; - const { headers } = await fetch(remoteConfigProp.url, requestProps); - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - + const hasConfigChanged = async ( + oldRemoteConfigs: AppConfig[], + newRemoteConfigs: AppConfig[], + ) => { return ( - remoteConfigProp.oldETag !== undefined && - remoteConfigProp.newETag !== undefined && - remoteConfigProp.oldETag !== remoteConfigProp.newETag + JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs) ); }; @@ -287,18 +241,12 @@ export async function loadConfig( try { handle = setInterval(async () => { console.info(`Checking for config update`); - for (const remoteConfigProp of remoteConfigProps) { - if (await hasConfigChanged(remoteConfigProp)) { - console.info(`Remote config change, reloading config ...`); - const newRemoteConfigs = await loadRemoteConfigFiles(); - watchProp.onChange([ - ...newRemoteConfigs, - ...fileConfigs, - ...envConfigs, - ]); - console.info(`Remote config reloaded`); - break; - } + const newRemoteConfigs = await loadRemoteConfigFiles(); + if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) { + remoteConfigs = newRemoteConfigs; + console.info(`Remote config change, reloading config ...`); + watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]); + console.info(`Remote config reloaded`); } }, remoteProp.reloadIntervalSeconds * 1000); } catch (error) { diff --git a/yarn.lock b/yarn.lock index 87991e1bb5..00c91f7783 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14042,13 +14042,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock-jest@1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" - integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== - dependencies: - fetch-mock "^9.11.0" - fetch-mock@^9.11.0: version "9.11.0" resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" From 988c5b8421a82a733cebafc7afcaf01683149a5c Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:26:08 +1100 Subject: [PATCH 05/43] Removed unused dependency Signed-off-by: Matto --- packages/config-loader/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 80e4121971..4b5afb3d29 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -33,7 +33,6 @@ "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", - "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", From 855d460611216a54d045df5b269c65f8641e94f9 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 26 Oct 2021 21:50:49 +1100 Subject: [PATCH 06/43] Change variable name Signed-off-by: Matto --- packages/config-loader/api-report.md | 12 ++++++------ packages/config-loader/src/index.ts | 7 ++++++- packages/config-loader/src/loader.ts | 15 +++++++++------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index bb36bbe750..1dad108c38 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -49,8 +49,8 @@ export type LoadConfigOptions = { configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; - remote?: Remote; - watch?: Watch; + remote?: LoadConfigOptionsRemote; + watch?: LoadConfigOptionsWatch; }; // @public @@ -75,10 +75,10 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Remote = { +export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; @@ -90,10 +90,10 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Watch = { +export type LoadConfigOptionsWatch = { onChange: (configs: AppConfig[]) => void; stopSignal?: Promise; }; diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index e0eafc00bc..c6b8ef8cef 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -30,4 +30,9 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; +export type { + ConfigTarget, + LoadConfigOptions, + LoadConfigOptionsWatch, + LoadConfigOptionsRemote, +} from './loader'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index b853d30d8a..2e7a0df801 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -31,7 +31,7 @@ import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; -export type Watch = { +export type LoadConfigOptionsWatch = { /** * A listener that is called when a config file is changed. */ @@ -43,7 +43,7 @@ export type Watch = { stopSignal?: Promise; }; -export type Remote = { +export type LoadConfigOptionsRemote = { /** * An optional remote config reloading period, in seconds */ @@ -71,12 +71,12 @@ export type LoadConfigOptions = { /** * An optional remote config */ - remote?: Remote; + remote?: LoadConfigOptionsRemote; /** * An optional configuration that enables watching of config files. */ - watch?: Watch; + watch?: LoadConfigOptionsWatch; }; /** @@ -198,7 +198,7 @@ export async function loadConfig( const envConfigs = await readEnvConfig(process.env); - const watchConfigFile = (watchProp: Watch) => { + const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); @@ -227,7 +227,10 @@ export async function loadConfig( } }; - const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const watchRemoteConfig = ( + watchProp: LoadConfigOptionsWatch, + remoteProp: LoadConfigOptionsRemote, + ) => { const hasConfigChanged = async ( oldRemoteConfigs: AppConfig[], newRemoteConfigs: AppConfig[], From 33d7bd0baaac3fa2d3b0b34d27b270f109953578 Mon Sep 17 00:00:00 2001 From: rodion Date: Tue, 26 Oct 2021 21:57:04 +0300 Subject: [PATCH 07/43] fix: sentry plugin can pass id token Signed-off-by: rodion --- .changeset/nasty-actors-push.md | 5 +++++ plugins/sentry/api-report.md | 18 +++++++++++++++++- plugins/sentry/src/api/production-api.ts | 21 ++++++++++++++++++++- plugins/sentry/src/plugin.ts | 10 ++++++++-- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 .changeset/nasty-actors-push.md diff --git a/.changeset/nasty-actors-push.md b/.changeset/nasty-actors-push.md new file mode 100644 index 0000000000..8f4d0d92a8 --- /dev/null +++ b/.changeset/nasty-actors-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-sentry': patch +--- + +fix: sentry-plugin can forward identity token to backend (for case when it requires authorization) diff --git a/plugins/sentry/api-report.md b/plugins/sentry/api-report.md index a23fa112c1..1b7a248e48 100644 --- a/plugins/sentry/api-report.md +++ b/plugins/sentry/api-report.md @@ -9,6 +9,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; +import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -34,7 +35,22 @@ export class MockSentryApi implements SentryApi { // // @public (undocumented) export class ProductionSentryApi implements SentryApi { - constructor(discoveryApi: DiscoveryApi, organization: string); + constructor( + discoveryApi: DiscoveryApi, + organization: string, + identityApi?: IdentityApi | undefined, + ); + // (undocumented) + authOptions(): Promise< + | { + headers?: undefined; + } + | { + headers: { + authorization: string; + }; + } + >; // (undocumented) fetchIssues( project: string, diff --git a/plugins/sentry/src/api/production-api.ts b/plugins/sentry/src/api/production-api.ts index 5968d20011..0632da711e 100644 --- a/plugins/sentry/src/api/production-api.ts +++ b/plugins/sentry/src/api/production-api.ts @@ -16,12 +16,13 @@ import { SentryIssue } from './sentry-issue'; import { SentryApi } from './sentry-api'; -import { DiscoveryApi } from '@backstage/core-plugin-api'; +import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; export class ProductionSentryApi implements SentryApi { constructor( private readonly discoveryApi: DiscoveryApi, private readonly organization: string, + private readonly identityApi?: IdentityApi, ) {} async fetchIssues( @@ -34,11 +35,13 @@ export class ProductionSentryApi implements SentryApi { } const apiUrl = `${await this.discoveryApi.getBaseUrl('proxy')}/sentry/api`; + const options = await this.authOptions(); const queryPart = query ? `&query=${query}` : ''; const response = await fetch( `${apiUrl}/0/projects/${this.organization}/${project}/issues/?statsPeriod=${statsFor}${queryPart}`, + options, ); if (response.status >= 400 && response.status < 600) { @@ -47,4 +50,20 @@ export class ProductionSentryApi implements SentryApi { return (await response.json()) as SentryIssue[]; } + + async authOptions() { + if (!this.identityApi) { + return {}; + } + try { + const token = await this.identityApi.getIdToken(); + return { + headers: { + authorization: `Bearer ${token}`, + }, + }; + } catch (e) { + return {}; + } + } } diff --git a/plugins/sentry/src/plugin.ts b/plugins/sentry/src/plugin.ts index 75de6ba6dc..454c1f02a6 100644 --- a/plugins/sentry/src/plugin.ts +++ b/plugins/sentry/src/plugin.ts @@ -21,6 +21,7 @@ import { createPlugin, createRouteRef, discoveryApiRef, + identityApiRef, } from '@backstage/core-plugin-api'; export const rootRouteRef = createRouteRef({ @@ -33,11 +34,16 @@ export const sentryPlugin = createPlugin({ apis: [ createApiFactory({ api: sentryApiRef, - deps: { configApi: configApiRef, discoveryApi: discoveryApiRef }, - factory: ({ configApi, discoveryApi }) => + deps: { + configApi: configApiRef, + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + }, + factory: ({ configApi, discoveryApi, identityApi }) => new ProductionSentryApi( discoveryApi, configApi.getString('sentry.organization'), + identityApi, ), }), ], From 197601f21c2f307132f4f0038821e50301ce10de Mon Sep 17 00:00:00 2001 From: rodion Date: Wed, 27 Oct 2021 11:28:58 +0300 Subject: [PATCH 08/43] fix: sentry-plugin passing token - no need for catch Signed-off-by: rodion --- plugins/sentry/src/api/production-api.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/plugins/sentry/src/api/production-api.ts b/plugins/sentry/src/api/production-api.ts index 0632da711e..37a735df14 100644 --- a/plugins/sentry/src/api/production-api.ts +++ b/plugins/sentry/src/api/production-api.ts @@ -55,15 +55,11 @@ export class ProductionSentryApi implements SentryApi { if (!this.identityApi) { return {}; } - try { - const token = await this.identityApi.getIdToken(); - return { - headers: { - authorization: `Bearer ${token}`, - }, - }; - } catch (e) { - return {}; - } + const token = await this.identityApi.getIdToken(); + return { + headers: { + authorization: `Bearer ${token}`, + }, + }; } } From a57927f5d5a41b70b2d90f4f889c65a6f22003a1 Mon Sep 17 00:00:00 2001 From: Matto Date: Wed, 27 Oct 2021 20:33:33 +1100 Subject: [PATCH 09/43] Removed dependency, introduced isValidUrl, and reinstated `configPaths` Signed-off-by: Matto --- packages/backend-common/package.json | 1 - packages/backend-common/src/config.ts | 3 +- packages/backend-common/src/urls.test.ts | 34 ++++++++++++++ packages/backend-common/src/urls.ts | 25 +++++++++++ packages/cli/package.json | 1 - packages/cli/src/lib/config.ts | 3 +- packages/cli/src/lib/urls.test.ts | 34 ++++++++++++++ packages/cli/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/api-report.md | 1 + packages/config-loader/package.json | 1 - packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/urls.test.ts | 34 ++++++++++++++ packages/config-loader/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/src/loader.test.ts | 49 ++++++++++++++++++++- packages/config-loader/src/loader.ts | 14 +++++- yarn.lock | 37 ++-------------- 16 files changed, 247 insertions(+), 41 deletions(-) create mode 100644 packages/backend-common/src/urls.test.ts create mode 100644 packages/backend-common/src/urls.ts create mode 100644 packages/cli/src/lib/urls.test.ts create mode 100644 packages/cli/src/lib/urls.ts create mode 100644 packages/config-loader/src/lib/urls.test.ts create mode 100644 packages/config-loader/src/lib/urls.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 67a2c1a853..0c368d7805 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -33,7 +33,6 @@ "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", "@backstage/errors": "^0.1.2", - "@backstage/integration": "^0.6.5", "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index afdb4fd0f7..c93f2cedf9 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,7 @@ import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader, JsonValue } from '@backstage/config'; import { ConfigTarget, loadConfig } from '@backstage/config-loader'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -130,6 +130,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, watch: { onChange(newConfigs) { diff --git a/packages/backend-common/src/urls.test.ts b/packages/backend-common/src/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-common/src/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-common/src/urls.ts b/packages/backend-common/src/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-common/src/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/cli/package.json b/packages/cli/package.json index 519b45a5ad..0dc4dc3cf7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,7 +31,6 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.3", - "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", "@hot-loader/react-dom": "^16.13.0", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 218771beb6..013328d9f3 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -21,7 +21,7 @@ import { } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; type Options = { args: string[]; @@ -56,6 +56,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, }); diff --git a/packages/cli/src/lib/urls.test.ts b/packages/cli/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/cli/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/cli/src/lib/urls.ts b/packages/cli/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/cli/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 1dad108c38..c8e01f27b0 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -46,6 +46,7 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public (undocumented) export type LoadConfigOptions = { configRoot: string; + configPaths: string[]; configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 4b5afb3d29..9475058b24 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,7 +30,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", "@types/json-schema": "^7.0.6", diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 32a0191cae..ca88b771ba 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readEnvConfig } from './env'; export * from './transform'; export * from './schema'; +export { isValidUrl } from './urls'; diff --git a/packages/config-loader/src/lib/urls.test.ts b/packages/config-loader/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/config-loader/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/config-loader/src/lib/urls.ts b/packages/config-loader/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/config-loader/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index e8b5a8828b..804254afa0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -67,6 +67,13 @@ describe('loadConfig', () => { $file: secrets/session-key.txt escaped: \$\${Escaped} `, + '/root/app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, '/root/app-config.development.yaml': ` app: sessionKey: development-key @@ -111,6 +118,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], env: 'production', }), @@ -136,6 +144,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], env: 'production', remote: { @@ -156,10 +165,43 @@ describe('loadConfig', () => { ]); }); - it('loads config with secrets', async () => { + it('loads config with secrets from two different files', async () => { await expect( loadConfig({ configRoot: '/root', + configPaths: ['/root/app-config2.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], + env: 'production', + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + { + context: 'app-config2.yaml', + data: { + app: { + title: 'Example App 2', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from single file', async () => { + await expect( + loadConfig({ + configRoot: '/root', + configPaths: ['/root/app-config.yaml'], configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), @@ -181,6 +223,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [ { path: '/root/app-config.yaml' }, { path: '/root/app-config.development.yaml' }, @@ -221,6 +264,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), @@ -246,6 +290,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: onChange.resolve, @@ -294,6 +339,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], watch: { onChange: onChange.resolve, @@ -339,6 +385,7 @@ describe('loadConfig', () => { await loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: () => { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2e7a0df801..1f20c4e437 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -24,10 +24,10 @@ import { createIncludeTransform, createSubstitutionTransform, EnvFunc, + isValidUrl, readEnvConfig, } from './lib'; import fetch from 'node-fetch'; -import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; @@ -55,6 +55,11 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; + /** Absolute paths to load config files from. Configs from earlier paths have lower priority. + * @deprecated Use {@link configTargets} instead. + */ + configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. configTargets: ConfigTarget[]; @@ -94,6 +99,13 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); + // Append deprecated configPaths to the absolute config paths received via configTargets. + options.configPaths.forEach(cp => { + if (!configPaths.includes(cp)) { + configPaths.push(cp); + } + }); + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) diff --git a/yarn.lock b/yarn.lock index 00c91f7783..10c65d5df7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11336,11 +11336,6 @@ core-js@^2.5.0: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== -core-js@^3.0.0: - version "3.18.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" - integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== - core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14042,22 +14037,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock@^9.11.0: - version "9.11.0" - resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" - integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== - dependencies: - "@babel/core" "^7.0.0" - "@babel/runtime" "^7.0.0" - core-js "^3.0.0" - debug "^4.1.1" - glob-to-regexp "^0.4.0" - is-subset "^0.1.1" - lodash.isequal "^4.5.0" - path-to-regexp "^2.2.1" - querystring "^0.2.0" - whatwg-url "^6.5.0" - fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -14888,7 +14867,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -16872,11 +16851,6 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" -is-subset@^0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" - integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= - is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18745,7 +18719,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: +lodash.isequal@^4.0.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -21559,11 +21533,6 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -path-to-regexp@^2.2.1: - version "2.4.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -28010,7 +27979,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -whatwg-url@^6.4.1, whatwg-url@^6.5.0: +whatwg-url@^6.4.1: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From 5d4053c827ffc307bbeb3dc2ded8da137aee3b58 Mon Sep 17 00:00:00 2001 From: Matto Date: Mon, 27 Sep 2021 23:41:24 +1000 Subject: [PATCH 10/43] Read config from remote config server Signed-off-by: Matto --- .changeset/giant-years-help.md | 8 + packages/backend-common/src/config.ts | 10 +- packages/cli/package.json | 6 + packages/cli/src/lib/config.ts | 16 +- packages/config-loader/api-report.md | 34 +++- packages/config-loader/package.json | 10 +- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/loader.test.ts | 125 +++++++++++- packages/config-loader/src/loader.ts | 236 +++++++++++++++++++--- packages/integration/api-report.md | 5 + packages/integration/src/index.ts | 1 + yarn.lock | 74 ++++++- 12 files changed, 475 insertions(+), 52 deletions(-) create mode 100644 .changeset/giant-years-help.md diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md new file mode 100644 index 0000000000..f0ca12ef19 --- /dev/null +++ b/.changeset/giant-years-help.md @@ -0,0 +1,8 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/integration': patch +--- + +Reading app config from a remote server diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index fa49d5f015..90131ebbe7 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,8 @@ import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; -import { loadConfig } from '@backstage/config-loader'; +import { ConfigTarget, loadConfig } from '@backstage/config-loader'; +import { isValidUrl } from '@backstage/integration'; export class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -149,7 +150,10 @@ export async function loadBackendConfig(options: { argv: string[]; }): Promise { const args = parseArgs(options.argv); - const configPaths: string[] = [args.config ?? []].flat(); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); const config = new ObservableConfigProxy(options.logger); @@ -158,7 +162,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configPaths.map(opt => resolvePath(opt)), + configTargets: configTargets, watch: { onChange(newConfigs) { options.logger.info( diff --git a/packages/cli/package.json b/packages/cli/package.json index 62a008087e..d85adff199 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,10 +29,16 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.4", + "@babel/core": "^7.4.4", + "@babel/plugin-transform-modules-commonjs": "^7.4.4", + "@backstage/cli-common": "^0.1.3", + "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", + "@backstage/config-loader": "^0.6.8", + "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 3688d60089..79d30ee220 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; +import { + ConfigTarget, + loadConfig, + loadConfigSchema, +} from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; +import { isValidUrl } from '@backstage/integration'; type Options = { args: string[]; @@ -26,7 +31,12 @@ type Options = { }; export async function loadCliConfig(options: Options) { - const configPaths = options.args.map(arg => paths.resolveTarget(arg)); + const configTargets: ConfigTarget[] = []; + options.args.forEach(arg => { + if (!isValidUrl(arg)) { + configTargets.push({ path: paths.resolveTarget(arg) }); + } + }); // Consider all packages in the monorepo when loading in config const { Project } = require('@lerna/project'); @@ -48,7 +58,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, - configPaths, + configTargets: configTargets, }); // printing to stderr to not clobber stdout in case the cli command diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 19b5ee9e27..4ed409e3ce 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = { withFilteredKeys?: boolean; }; +// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ConfigTarget = + | { + path: string; + } + | { + url: string; + }; + // @public export type ConfigVisibility = 'frontend' | 'backend' | 'secret'; @@ -32,13 +43,11 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public export type LoadConfigOptions = { configRoot: string; - configPaths: string[]; + configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; - watch?: { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; - }; + remote?: Remote; + watch?: Watch; }; // @public @@ -64,6 +73,13 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; +// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Remote = { + reloadIntervalSeconds: number; +}; + // @public export type TransformFunc = ( value: T, @@ -71,4 +87,12 @@ export type TransformFunc = ( visibility: ConfigVisibility; }, ) => T | undefined; + +// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Watch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; ``` diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index a731225e2b..cab3a43fc5 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,10 +30,12 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", + "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", @@ -42,8 +44,10 @@ "json-schema-merge-allof": "^0.8.1", "json-schema-traverse": "^1.0.0", "typescript-json-schema": "^0.50.1", + "uuid": "^8.3.2", "yaml": "^1.9.2", - "yup": "^0.32.9" + "yup": "^0.32.9", + "node-fetch": "2.6.5" }, "devDependencies": { "@types/jest": "^26.0.7", @@ -51,7 +55,9 @@ "@types/mock-fs": "^4.10.0", "@types/node": "^14.14.32", "@types/yup": "^0.29.13", - "mock-fs": "^5.1.0" + "mock-fs": "^5.1.0", + "fetch-mock-jest": "1.5.1", + "fetch-mock": "^9.11.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 888c68ef70..ee3e6413ad 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -29,4 +29,4 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './loader'; +export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3f51e76936..190904a870 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,12 +18,33 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; +import { v4 as uuidv4 } from 'uuid'; + +const fetchMock = require('fetch-mock').sandbox(); +const nodeFetch = require('node-fetch'); + +nodeFetch.default = fetchMock; describe('loadConfig', () => { beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + method: 'GET', + }, + { + body: `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + ); + mockFs({ '/root/app-config.yaml': ` app: @@ -66,6 +87,7 @@ describe('loadConfig', () => { }); afterEach(() => { + fetchMock.restore(); mockFs.restore(); }); @@ -73,7 +95,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], env: 'production', }), ).resolves.toEqual([ @@ -90,11 +112,37 @@ describe('loadConfig', () => { ]); }); + it('load config from remote path', async () => { + const configUrl = 'https://some.domain.io/app-config.yaml'; + + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + env: 'production', + remote: { + reloadIntervalSeconds: 30, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), ).resolves.toEqual([ @@ -115,9 +163,9 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', + configTargets: [ + { path: '/root/app-config.yaml' }, + { path: '/root/app-config.development.yaml' }, ], env: 'development', }), @@ -155,7 +203,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.substitute.yaml'], + configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), ).resolves.toEqual([ @@ -180,7 +228,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: onChange.resolve, stopSignal: stopSignal.promise, @@ -218,12 +266,73 @@ describe('loadConfig', () => { stopSignal.resolve(); }); + it('watches remote config urls', async () => { + const onChange = defer(); + const stopSignal = defer(); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + remote: { + reloadIntervalSeconds: 1, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + }, + { + body: `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + { overwriteRoutes: true }, + ); + + await expect(onChange.promise).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'NEW ReMOTe ExaMPLe App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + it('stops watching config files', async () => { const stopSignal = defer(); await loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: () => { expect('not').toBe('called'); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 6803d61786..2c61b4d1ce 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,15 +17,66 @@ import fs from 'fs-extra'; import yaml from 'yaml'; import chokidar from 'chokidar'; -import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; import { AppConfig } from '@backstage/config'; import { ForwardedError } from '@backstage/errors'; import { applyConfigTransforms, - readEnvConfig, createIncludeTransform, createSubstitutionTransform, + EnvFunc, + readEnvConfig, } from './lib'; +import fetch from 'node-fetch'; +import { isValidUrl } from '@backstage/integration'; + +export type ConfigTarget = { path: string } | { url: string }; + +export type Watch = { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; +}; + +/** + * Options that control the loading of configuration files in the backend. + * + * @public + */ +export type Remote = { + /** + * An optional remote config reloading period, in seconds + */ + reloadIntervalSeconds: number; +}; + +export type RemoteConfigProp = { + /** + * URL of the remote config + */ + url: string; + + /** + * Contents of the remote config + */ + content: string | null; + + /** + * An optional new ETag header value. Used when checking for updated config. + */ + newETag?: string; + + /** + * An optional old ETag header value. Used when checking for updated config + */ + oldETag?: string; +}; /** * Options that control the loading of configuration files in the backend. @@ -36,8 +87,8 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; - // Absolute paths to load config files from. Configs from earlier paths have lower priority. - configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. + configTargets: ConfigTarget[]; /** @deprecated This option has been removed */ env?: string; @@ -49,22 +100,19 @@ export type LoadConfigOptions = { */ experimentalEnvFunc?: (name: string) => Promise; + /** + * An optional remote config + */ + remote?: Remote; + /** * An optional configuration that enables watching of config files. */ - watch?: { - /** - * A listener that is called when a config file is changed. - */ - onChange: (configs: AppConfig[]) => void; - - /** - * An optional signal that stops the watcher once the promise resolves. - */ - stopSignal?: Promise; - }; + watch?: Watch; }; +const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; + /** * Load configuration data. * @@ -73,12 +121,30 @@ export type LoadConfigOptions = { export async function loadConfig( options: LoadConfigOptions, ): Promise { - const { configRoot, experimentalEnvFunc: envFunc, watch } = options; - const configPaths = options.configPaths.slice(); + const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options; + + const configPaths: string[] = options.configTargets + .slice() + .filter((e): e is { path: string } => e.hasOwnProperty('path')) + .map(configTarget => configTarget.path); + + let configUrls: string[] = options.configTargets + .slice() + .filter((e): e is { url: string } => e.hasOwnProperty('url')) + .map(configTarget => configTarget.url); + + const remoteConfigProps: RemoteConfigProp[] = []; + + if (remote === undefined && configUrls.length > 0) { + console.warn( + `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, + ); + configUrls = []; + } // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` - if (configPaths.length === 0) { + if (configPaths.length === 0 && configUrls.length === 0) { configPaths.push(resolvePath(configRoot, 'app-config.yaml')); const localConfig = resolvePath(configRoot, 'app-config.local.yaml'); @@ -114,22 +180,79 @@ export async function loadConfig( return configs; }; - let fileConfigs; + const loadRemoteConfigFiles = async () => { + const configs: AppConfig[] = []; + + const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { + const response = await fetch(remoteConfigProp.url); + if (!response.ok) { + throw new Error( + `Could not read config file at ${remoteConfigProp.url}`, + ); + } + + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + remoteConfigProp.content = await response.text(); + + return remoteConfigProp; + }; + + for (let i = 0; i < configUrls.length; i++) { + const remoteConfigProp = await readConfigFromUrl({ + url: configUrls[i], + content: null, + }); + + if (!isValidUrl(remoteConfigProp.url)) { + throw new Error( + `Config load path is not valid: '${remoteConfigProp.url}'`, + ); + } + + const dir = configRoot; + if (!remoteConfigProp.content) { + throw new Error(`Config is not valid`); + } + const input = yaml.parse(remoteConfigProp.content); + const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ + substitutionTransform, + ]); + + configs.push({ data, context: remoteConfigProp.url }); + + remoteConfigProps.push(remoteConfigProp); + } + + return configs; + }; + + let fileConfigs: AppConfig[]; try { fileConfigs = await loadConfigFiles(); } catch (error) { throw new ForwardedError('Failed to read static configuration file', error); } + let remoteConfigs: AppConfig[] = []; + if (remote) { + try { + remoteConfigs = await loadRemoteConfigFiles(); + } catch (error) { + throw new Error(`Failed to read remote configuration file, ${error}`); + } + } + const envConfigs = await readEnvConfig(process.env); - // Set up config file watching if requested by the caller - if (watch) { - let currentSerializedConfig = JSON.stringify(fileConfigs); - + const watchConfigFile = (watchProp: Watch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); + + let currentSerializedConfig = JSON.stringify(fileConfigs); watcher.on('change', async () => { try { const newConfigs = await loadConfigFiles(); @@ -140,18 +263,77 @@ export async function loadConfig( } currentSerializedConfig = newSerializedConfig; - watch.onChange([...newConfigs, ...envConfigs]); + watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]); } catch (error) { console.error(`Failed to reload configuration files, ${error}`); } }); - if (watch.stopSignal) { - watch.stopSignal.then(() => { + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { watcher.close(); }); } + }; + + const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { + const requestProps = { method: 'HEAD' }; + const { headers } = await fetch(remoteConfigProp.url, requestProps); + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + + return ( + remoteConfigProp.oldETag !== undefined && + remoteConfigProp.newETag !== undefined && + remoteConfigProp.oldETag !== remoteConfigProp.newETag + ); + }; + + let handle: NodeJS.Timeout | undefined; + try { + handle = setInterval(async () => { + console.info(`Checking for config update`); + for (const remoteConfigProp of remoteConfigProps) { + if (await hasConfigChanged(remoteConfigProp)) { + console.info(`Remote config change, reloading config ...`); + const newRemoteConfigs = await loadRemoteConfigFiles(); + watchProp.onChange([ + ...newRemoteConfigs, + ...fileConfigs, + ...envConfigs, + ]); + console.info(`Remote config reloaded`); + break; + } + } + }, remoteProp.reloadIntervalSeconds * 1000); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { + if (handle !== undefined) { + console.info(`Stopping remote config watch`); + clearInterval(handle); + handle = undefined; + } + }); + } + }; + + // Set up config file watching if requested by the caller + if (watch) { + watchConfigFile(watch); } - return [...fileConfigs, ...envConfigs]; + if (watch && remote) { + watchRemoteConfig(watch, remote); + } + + return remote + ? [...remoteConfigs, ...fileConfigs, ...envConfigs] + : [...fileConfigs, ...envConfigs]; } diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 01f84383fe..e07f61eea6 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,6 +347,11 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function isValidUrl(url: string): boolean; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 3d5f336e8d..b8992f52d2 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,6 +27,7 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; +export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; diff --git a/yarn.lock b/yarn.lock index c905fbea6f..e45d685bf8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11555,6 +11555,11 @@ core-js@^2.4.0, core-js@^2.5.0, core-js@^2.6.10: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== +core-js@^3.0.0: + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== + core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14238,6 +14243,34 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== +fetch-blob@2.1.2: + version "2.1.2" + resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" + integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== + +fetch-mock-jest@1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" + integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== + dependencies: + fetch-mock "^9.11.0" + +fetch-mock@^9.11.0: + version "9.11.0" + resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" + integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== + dependencies: + "@babel/core" "^7.0.0" + "@babel/runtime" "^7.0.0" + core-js "^3.0.0" + debug "^4.1.1" + glob-to-regexp "^0.4.0" + is-subset "^0.1.1" + lodash.isequal "^4.5.0" + path-to-regexp "^2.2.1" + querystring "^0.2.0" + whatwg-url "^6.5.0" + fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -15062,7 +15095,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -17103,6 +17136,11 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" +is-subset@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" + integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= + is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18971,7 +19009,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0: +lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -20815,6 +20853,13 @@ node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -22153,6 +22198,11 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" +path-to-regexp@^2.2.1: + version "2.4.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -27219,6 +27269,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + "traverse@>=0.3.0 <0.4": version "0.3.9" resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" @@ -28449,6 +28504,11 @@ web-streams-polyfill@4.0.0-beta.1: resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.1.tgz#3b19b9817374b7cee06d374ba7eeb3aeb80e8c95" integrity sha512-3ux37gEX670UUphBF9AMCq8XM6iQ8Ac6A+DSRRjDoRBm1ufCkaCDdNVbaqq60PsEkdNlLKrGtv/YBP4EJXqNtQ== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -28671,7 +28731,15 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^6.4.1: +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +whatwg-url@^6.4.1, whatwg-url@^6.5.0: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From e51740f9c96cb51c3e742df3289b56ce92264925 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 19 Oct 2021 23:18:19 +1100 Subject: [PATCH 11/43] Added documentation Signed-off-by: Matto --- docs/conf/writing.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4288d23b17..86b844484b 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,7 +1,6 @@ --- -id: writing -title: Writing Backstage Configuration Files -description: Documentation on Writing Backstage Configuration Files +id: writing title: Writing Backstage Configuration Files description: Documentation on Writing +Backstage Configuration Files --- ## File Format @@ -67,13 +66,15 @@ production build. ## Configuration Files -It is possible to have multiple configuration files, both to support different -environments, but also to define configuration that is local to specific -packages. The configuration files to load are selected using a `--config ` -flag, and it is possible to load any number of files. Paths are relative to the -working directory of the executed process, for example `package/backend`. This -means that to select a config file in the repo root when running the backend, -you would use `--config ../../my-config.yaml`. +It is possible to have multiple configuration files (bundled and/or remote), +both to support different environments, but also to define configuration that is +local to specific packages. The configuration files to load are selected using a +`--config ` flag, and it is possible to load any number of +files. Paths are relative to the working directory of the executed process, for +example `package/backend`. This means that to select a config file in the repo +root when running the backend, you would use `--config ../../my-config.yaml`, +and for config file on a config server you would use +`--config https://some.domain.io/app-config.yaml` If no `config` flags are specified, the default behavior is to load `app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root. @@ -85,7 +86,7 @@ are NOT loaded. To include them you need to explicitly include them with a flag, for example: ```shell -yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml +yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml --config https://some.domain.io/app-config.yaml ``` All loaded configuration files are merged together using the following rules: From ce843364f09c59a32f3d4ba8c178d33e73c51d9b Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:20:36 +1100 Subject: [PATCH 12/43] Remove the dependency on Etag headers for reloading config Signed-off-by: Matto --- docs/conf/writing.md | 5 +- packages/cli/package.json | 2 - packages/config-loader/package.json | 4 +- packages/config-loader/src/loader.test.ts | 79 ++++++++++++---------- packages/config-loader/src/loader.ts | 82 +++++++---------------- yarn.lock | 7 -- 6 files changed, 73 insertions(+), 106 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 86b844484b..a0e29820ed 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,6 +1,7 @@ --- -id: writing title: Writing Backstage Configuration Files description: Documentation on Writing -Backstage Configuration Files +id: writing +title: Writing Backstage Configuration Files +description: Documentation on Writing Backstage Configuration Files --- ## File Format diff --git a/packages/cli/package.json b/packages/cli/package.json index d85adff199..b9ba9fe8a3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,14 +31,12 @@ "@backstage/cli-common": "^0.1.4", "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", - "@backstage/cli-common": "^0.1.3", "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", "@backstage/config-loader": "^0.6.8", - "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index cab3a43fc5..456bca90ad 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -44,7 +44,6 @@ "json-schema-merge-allof": "^0.8.1", "json-schema-traverse": "^1.0.0", "typescript-json-schema": "^0.50.1", - "uuid": "^8.3.2", "yaml": "^1.9.2", "yup": "^0.32.9", "node-fetch": "2.6.5" @@ -56,8 +55,7 @@ "@types/node": "^14.14.32", "@types/yup": "^0.29.13", "mock-fs": "^5.1.0", - "fetch-mock-jest": "1.5.1", - "fetch-mock": "^9.11.0" + "msw": "^0.29.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 190904a870..e8b5a8828b 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,33 +18,47 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; -import { v4 as uuidv4 } from 'uuid'; - -const fetchMock = require('fetch-mock').sandbox(); -const nodeFetch = require('node-fetch'); - -nodeFetch.default = fetchMock; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; describe('loadConfig', () => { + const server = setupServer(); + const initialLoaderHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + const reloadHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + beforeAll(() => server.listen()); + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - method: 'GET', - }, - { - body: `app: - title: Remote Example App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - ); - mockFs({ '/root/app-config.yaml': ` app: @@ -87,10 +101,12 @@ describe('loadConfig', () => { }); afterEach(() => { - fetchMock.restore(); mockFs.restore(); + server.resetHandlers(); }); + afterAll(() => server.close()); + it('load config from default path', async () => { await expect( loadConfig({ @@ -113,6 +129,8 @@ describe('loadConfig', () => { }); it('load config from remote path', async () => { + server.use(initialLoaderHandler); + const configUrl = 'https://some.domain.io/app-config.yaml'; await expect( @@ -267,6 +285,8 @@ describe('loadConfig', () => { }); it('watches remote config urls', async () => { + server.use(initialLoaderHandler); + const onChange = defer(); const stopSignal = defer(); @@ -296,20 +316,7 @@ describe('loadConfig', () => { }, ]); - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - }, - { - body: `app: - title: NEW ReMOTe ExaMPLe App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - { overwriteRoutes: true }, - ); + server.use(reloadHandler); await expect(onChange.promise).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2c61b4d1ce..c35b742f92 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -56,6 +56,7 @@ export type Remote = { reloadIntervalSeconds: number; }; +/** @public */ export type RemoteConfigProp = { /** * URL of the remote config @@ -111,8 +112,6 @@ export type LoadConfigOptions = { watch?: Watch; }; -const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; - /** * Load configuration data. * @@ -128,18 +127,15 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); - let configUrls: string[] = options.configTargets + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) .map(configTarget => configTarget.url); - const remoteConfigProps: RemoteConfigProp[] = []; - if (remote === undefined && configUrls.length > 0) { - console.warn( + throw new Error( `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, ); - configUrls = []; } // If no paths are provided, we default to reading @@ -183,47 +179,32 @@ export async function loadConfig( const loadRemoteConfigFiles = async () => { const configs: AppConfig[] = []; - const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { - const response = await fetch(remoteConfigProp.url); + const readConfigFromUrl = async (url: string) => { + const response = await fetch(url); if (!response.ok) { - throw new Error( - `Could not read config file at ${remoteConfigProp.url}`, - ); + throw new Error(`Could not read config file at ${url}`); } - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - remoteConfigProp.content = await response.text(); - - return remoteConfigProp; + return await response.text(); }; for (let i = 0; i < configUrls.length; i++) { - const remoteConfigProp = await readConfigFromUrl({ - url: configUrls[i], - content: null, - }); - - if (!isValidUrl(remoteConfigProp.url)) { - throw new Error( - `Config load path is not valid: '${remoteConfigProp.url}'`, - ); + const configUrl = configUrls[i]; + if (!isValidUrl(configUrl)) { + throw new Error(`Config load path is not valid: '${configUrl}'`); } - const dir = configRoot; - if (!remoteConfigProp.content) { + const remoteConfigContent = await readConfigFromUrl(configUrl); + if (!remoteConfigContent) { throw new Error(`Config is not valid`); } - const input = yaml.parse(remoteConfigProp.content); + const configYaml = yaml.parse(remoteConfigContent); const substitutionTransform = createSubstitutionTransform(env); - const data = await applyConfigTransforms(dir, input, [ + const data = await applyConfigTransforms(configRoot, configYaml, [ substitutionTransform, ]); - configs.push({ data, context: remoteConfigProp.url }); - - remoteConfigProps.push(remoteConfigProp); + configs.push({ data, context: configUrl }); } return configs; @@ -277,17 +258,12 @@ export async function loadConfig( }; const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { - const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { - const requestProps = { method: 'HEAD' }; - const { headers } = await fetch(remoteConfigProp.url, requestProps); - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - + const hasConfigChanged = async ( + oldRemoteConfigs: AppConfig[], + newRemoteConfigs: AppConfig[], + ) => { return ( - remoteConfigProp.oldETag !== undefined && - remoteConfigProp.newETag !== undefined && - remoteConfigProp.oldETag !== remoteConfigProp.newETag + JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs) ); }; @@ -295,18 +271,12 @@ export async function loadConfig( try { handle = setInterval(async () => { console.info(`Checking for config update`); - for (const remoteConfigProp of remoteConfigProps) { - if (await hasConfigChanged(remoteConfigProp)) { - console.info(`Remote config change, reloading config ...`); - const newRemoteConfigs = await loadRemoteConfigFiles(); - watchProp.onChange([ - ...newRemoteConfigs, - ...fileConfigs, - ...envConfigs, - ]); - console.info(`Remote config reloaded`); - break; - } + const newRemoteConfigs = await loadRemoteConfigFiles(); + if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) { + remoteConfigs = newRemoteConfigs; + console.info(`Remote config change, reloading config ...`); + watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]); + console.info(`Remote config reloaded`); } }, remoteProp.reloadIntervalSeconds * 1000); } catch (error) { diff --git a/yarn.lock b/yarn.lock index e45d685bf8..113b0d48c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14248,13 +14248,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock-jest@1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" - integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== - dependencies: - fetch-mock "^9.11.0" - fetch-mock@^9.11.0: version "9.11.0" resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" From 8e85a0bca056f8ea245405a7555911e2a001ca66 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 26 Oct 2021 21:50:49 +1100 Subject: [PATCH 13/43] Change variable name Signed-off-by: Matto --- packages/config-loader/api-report.md | 12 ++++++------ packages/config-loader/src/index.ts | 7 ++++++- packages/config-loader/src/loader.ts | 22 ++++++++++------------ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 4ed409e3ce..3b61b67d7b 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -46,8 +46,8 @@ export type LoadConfigOptions = { configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; - remote?: Remote; - watch?: Watch; + remote?: LoadConfigOptionsRemote; + watch?: LoadConfigOptionsWatch; }; // @public @@ -73,10 +73,10 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Remote = { +export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; @@ -88,10 +88,10 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Watch = { +export type LoadConfigOptionsWatch = { onChange: (configs: AppConfig[]) => void; stopSignal?: Promise; }; diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index ee3e6413ad..97f0d301a3 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -29,4 +29,9 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; +export type { + ConfigTarget, + LoadConfigOptions, + LoadConfigOptionsWatch, + LoadConfigOptionsRemote, +} from './loader'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index c35b742f92..8c4bbae9d6 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -32,7 +32,7 @@ import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; -export type Watch = { +export type LoadConfigOptionsWatch = { /** * A listener that is called when a config file is changed. */ @@ -44,12 +44,7 @@ export type Watch = { stopSignal?: Promise; }; -/** - * Options that control the loading of configuration files in the backend. - * - * @public - */ -export type Remote = { +export type LoadConfigOptionsRemote = { /** * An optional remote config reloading period, in seconds */ @@ -104,12 +99,12 @@ export type LoadConfigOptions = { /** * An optional remote config */ - remote?: Remote; + remote?: LoadConfigOptionsRemote; /** * An optional configuration that enables watching of config files. */ - watch?: Watch; + watch?: LoadConfigOptionsWatch; }; /** @@ -222,13 +217,13 @@ export async function loadConfig( try { remoteConfigs = await loadRemoteConfigFiles(); } catch (error) { - throw new Error(`Failed to read remote configuration file, ${error}`); + throw new ForwardedError(`Failed to read remote configuration file, ${error}`); } } const envConfigs = await readEnvConfig(process.env); - const watchConfigFile = (watchProp: Watch) => { + const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); @@ -257,7 +252,10 @@ export async function loadConfig( } }; - const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const watchRemoteConfig = ( + watchProp: LoadConfigOptionsWatch, + remoteProp: LoadConfigOptionsRemote, + ) => { const hasConfigChanged = async ( oldRemoteConfigs: AppConfig[], newRemoteConfigs: AppConfig[], From 1cecd737f98cc0fca824073af9e0edd9cd75410d Mon Sep 17 00:00:00 2001 From: Matto Date: Wed, 27 Oct 2021 20:33:33 +1100 Subject: [PATCH 14/43] Removed dependency, introduced isValidUrl, and reinstated `configPaths` Signed-off-by: Matto --- packages/backend-common/src/config.ts | 3 +- packages/backend-common/src/urls.test.ts | 34 ++++++++++++++ packages/backend-common/src/urls.ts | 25 +++++++++++ packages/cli/src/lib/config.ts | 3 +- packages/cli/src/lib/urls.test.ts | 34 ++++++++++++++ packages/cli/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/api-report.md | 1 + packages/config-loader/package.json | 1 - packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/urls.test.ts | 34 ++++++++++++++ packages/config-loader/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/src/loader.test.ts | 49 ++++++++++++++++++++- packages/config-loader/src/loader.ts | 14 +++++- yarn.lock | 37 ++-------------- 14 files changed, 247 insertions(+), 39 deletions(-) create mode 100644 packages/backend-common/src/urls.test.ts create mode 100644 packages/backend-common/src/urls.ts create mode 100644 packages/cli/src/lib/urls.test.ts create mode 100644 packages/cli/src/lib/urls.ts create mode 100644 packages/config-loader/src/lib/urls.test.ts create mode 100644 packages/config-loader/src/lib/urls.ts diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 90131ebbe7..9b0eb8dcd0 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -21,7 +21,7 @@ import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; import { ConfigTarget, loadConfig } from '@backstage/config-loader'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; export class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -162,6 +162,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, watch: { onChange(newConfigs) { diff --git a/packages/backend-common/src/urls.test.ts b/packages/backend-common/src/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-common/src/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-common/src/urls.ts b/packages/backend-common/src/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-common/src/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 79d30ee220..80b374039f 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -21,7 +21,7 @@ import { } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; type Options = { args: string[]; @@ -58,6 +58,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, }); diff --git a/packages/cli/src/lib/urls.test.ts b/packages/cli/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/cli/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/cli/src/lib/urls.ts b/packages/cli/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/cli/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 3b61b67d7b..be4678aa0f 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -43,6 +43,7 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public export type LoadConfigOptions = { configRoot: string; + configPaths: string[]; configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 456bca90ad..58b0061996 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,7 +30,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 32a0191cae..ca88b771ba 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readEnvConfig } from './env'; export * from './transform'; export * from './schema'; +export { isValidUrl } from './urls'; diff --git a/packages/config-loader/src/lib/urls.test.ts b/packages/config-loader/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/config-loader/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/config-loader/src/lib/urls.ts b/packages/config-loader/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/config-loader/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index e8b5a8828b..804254afa0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -67,6 +67,13 @@ describe('loadConfig', () => { $file: secrets/session-key.txt escaped: \$\${Escaped} `, + '/root/app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, '/root/app-config.development.yaml': ` app: sessionKey: development-key @@ -111,6 +118,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], env: 'production', }), @@ -136,6 +144,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], env: 'production', remote: { @@ -156,10 +165,43 @@ describe('loadConfig', () => { ]); }); - it('loads config with secrets', async () => { + it('loads config with secrets from two different files', async () => { await expect( loadConfig({ configRoot: '/root', + configPaths: ['/root/app-config2.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], + env: 'production', + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + { + context: 'app-config2.yaml', + data: { + app: { + title: 'Example App 2', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from single file', async () => { + await expect( + loadConfig({ + configRoot: '/root', + configPaths: ['/root/app-config.yaml'], configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), @@ -181,6 +223,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [ { path: '/root/app-config.yaml' }, { path: '/root/app-config.development.yaml' }, @@ -221,6 +264,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), @@ -246,6 +290,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: onChange.resolve, @@ -294,6 +339,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], watch: { onChange: onChange.resolve, @@ -339,6 +385,7 @@ describe('loadConfig', () => { await loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: () => { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 8c4bbae9d6..338f199132 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -25,10 +25,10 @@ import { createIncludeTransform, createSubstitutionTransform, EnvFunc, + isValidUrl, readEnvConfig, } from './lib'; import fetch from 'node-fetch'; -import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; @@ -83,6 +83,11 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; + /** Absolute paths to load config files from. Configs from earlier paths have lower priority. + * @deprecated Use {@link configTargets} instead. + */ + configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. configTargets: ConfigTarget[]; @@ -122,6 +127,13 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); + // Append deprecated configPaths to the absolute config paths received via configTargets. + options.configPaths.forEach(cp => { + if (!configPaths.includes(cp)) { + configPaths.push(cp); + } + }); + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) diff --git a/yarn.lock b/yarn.lock index 113b0d48c2..079cd96a5c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11555,11 +11555,6 @@ core-js@^2.4.0, core-js@^2.5.0, core-js@^2.6.10: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== -core-js@^3.0.0: - version "3.18.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" - integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== - core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14248,22 +14243,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock@^9.11.0: - version "9.11.0" - resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" - integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== - dependencies: - "@babel/core" "^7.0.0" - "@babel/runtime" "^7.0.0" - core-js "^3.0.0" - debug "^4.1.1" - glob-to-regexp "^0.4.0" - is-subset "^0.1.1" - lodash.isequal "^4.5.0" - path-to-regexp "^2.2.1" - querystring "^0.2.0" - whatwg-url "^6.5.0" - fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -15088,7 +15067,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -17129,11 +17108,6 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" -is-subset@^0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" - integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= - is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -19002,7 +18976,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: +lodash.isequal@^4.0.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -22191,11 +22165,6 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -path-to-regexp@^2.2.1: - version "2.4.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -28732,7 +28701,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -whatwg-url@^6.4.1, whatwg-url@^6.5.0: +whatwg-url@^6.4.1: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From 5b51907cbf2da15f03a9e9fccd4ec381b80fd9f9 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 14:56:31 +1100 Subject: [PATCH 15/43] resolved conflicts Signed-off-by: Matto --- packages/backend-common/src/config.ts | 16 +++++++++++++++ packages/config-loader/src/loader.ts | 29 ++++----------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 9b0eb8dcd0..5b09d66608 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -84,51 +84,66 @@ export class ObservableConfigProxy implements Config { has(key: string): boolean { return this.select(false)?.has(key) ?? false; } + keys(): string[] { return this.select(false)?.keys() ?? []; } + get(key?: string): T { return this.select(true).get(key); } + getOptional(key?: string): T | undefined { return this.select(false)?.getOptional(key); } + getConfig(key: string): Config { return new ObservableConfigProxy(this.logger, this, key); } + getOptionalConfig(key: string): Config | undefined { if (this.select(false)?.has(key)) { return new ObservableConfigProxy(this.logger, this, key); } return undefined; } + getConfigArray(key: string): Config[] { return this.select(true).getConfigArray(key); } + getOptionalConfigArray(key: string): Config[] | undefined { return this.select(false)?.getOptionalConfigArray(key); } + getNumber(key: string): number { return this.select(true).getNumber(key); } + getOptionalNumber(key: string): number | undefined { return this.select(false)?.getOptionalNumber(key); } + getBoolean(key: string): boolean { return this.select(true).getBoolean(key); } + getOptionalBoolean(key: string): boolean | undefined { return this.select(false)?.getOptionalBoolean(key); } + getString(key: string): string { return this.select(true).getString(key); } + getOptionalString(key: string): string | undefined { return this.select(false)?.getOptionalString(key); } + getStringArray(key: string): string[] { return this.select(true).getStringArray(key); } + getOptionalStringArray(key: string): string[] | undefined { return this.select(false)?.getOptionalStringArray(key); } @@ -184,6 +199,7 @@ export async function loadBackendConfig(options: { } }), }, + remote: { reloadIntervalSeconds: 10 }, }); options.logger.info( diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 338f199132..b8fab12917 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -24,7 +24,6 @@ import { applyConfigTransforms, createIncludeTransform, createSubstitutionTransform, - EnvFunc, isValidUrl, readEnvConfig, } from './lib'; @@ -51,29 +50,6 @@ export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; -/** @public */ -export type RemoteConfigProp = { - /** - * URL of the remote config - */ - url: string; - - /** - * Contents of the remote config - */ - content: string | null; - - /** - * An optional new ETag header value. Used when checking for updated config. - */ - newETag?: string; - - /** - * An optional old ETag header value. Used when checking for updated config - */ - oldETag?: string; -}; - /** * Options that control the loading of configuration files in the backend. * @@ -229,7 +205,10 @@ export async function loadConfig( try { remoteConfigs = await loadRemoteConfigFiles(); } catch (error) { - throw new ForwardedError(`Failed to read remote configuration file, ${error}`); + throw new ForwardedError( + `Failed to read remote configuration file`, + error, + ); } } From 5cce78f968099c3701303fbb035918c8914e50c0 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 15:11:54 +1100 Subject: [PATCH 16/43] Resolved conflicts Signed-off-by: Matto --- .changeset/giant-years-help.md | 1 - ADOPTERS.md | 122 +++++++++++++------------- packages/backend-common/src/config.ts | 15 ---- packages/cli/package.json | 4 - packages/config-loader/package.json | 1 - packages/integration/api-report.md | 5 -- packages/integration/src/index.ts | 1 - 7 files changed, 61 insertions(+), 88 deletions(-) diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md index f0ca12ef19..c6ca0def30 100644 --- a/.changeset/giant-years-help.md +++ b/.changeset/giant-years-help.md @@ -2,7 +2,6 @@ '@backstage/backend-common': patch '@backstage/cli': patch '@backstage/config-loader': patch -'@backstage/integration': patch --- Reading app config from a remote server diff --git a/ADOPTERS.md b/ADOPTERS.md index 2d2583be95..db751bfb9b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,61 +1,61 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 31869c685a..9b0eb8dcd0 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -84,66 +84,51 @@ export class ObservableConfigProxy implements Config { has(key: string): boolean { return this.select(false)?.has(key) ?? false; } - keys(): string[] { return this.select(false)?.keys() ?? []; } - get(key?: string): T { return this.select(true).get(key); } - getOptional(key?: string): T | undefined { return this.select(false)?.getOptional(key); } - getConfig(key: string): Config { return new ObservableConfigProxy(this.logger, this, key); } - getOptionalConfig(key: string): Config | undefined { if (this.select(false)?.has(key)) { return new ObservableConfigProxy(this.logger, this, key); } return undefined; } - getConfigArray(key: string): Config[] { return this.select(true).getConfigArray(key); } - getOptionalConfigArray(key: string): Config[] | undefined { return this.select(false)?.getOptionalConfigArray(key); } - getNumber(key: string): number { return this.select(true).getNumber(key); } - getOptionalNumber(key: string): number | undefined { return this.select(false)?.getOptionalNumber(key); } - getBoolean(key: string): boolean { return this.select(true).getBoolean(key); } - getOptionalBoolean(key: string): boolean | undefined { return this.select(false)?.getOptionalBoolean(key); } - getString(key: string): string { return this.select(true).getString(key); } - getOptionalString(key: string): string | undefined { return this.select(false)?.getOptionalString(key); } - getStringArray(key: string): string[] { return this.select(true).getStringArray(key); } - getOptionalStringArray(key: string): string[] | undefined { return this.select(false)?.getOptionalStringArray(key); } diff --git a/packages/cli/package.json b/packages/cli/package.json index b9ba9fe8a3..62a008087e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,14 +29,10 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.4", - "@babel/core": "^7.4.4", - "@babel/plugin-transform-modules-commonjs": "^7.4.4", - "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", - "@backstage/config-loader": "^0.6.8", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 58b0061996..efcf753295 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,7 +34,6 @@ "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", - "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index e07f61eea6..01f84383fe 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,11 +347,6 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; -// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export function isValidUrl(url: string): boolean; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index b8992f52d2..3d5f336e8d 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,7 +27,6 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; -export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; From f7cd672c7f655427162a7ca0260e3438742537d8 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 16:39:13 +1100 Subject: [PATCH 17/43] Updated error message Signed-off-by: Matto --- packages/config-loader/src/loader.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index b8fab12917..4a5660f46c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -116,9 +116,7 @@ export async function loadConfig( .map(configTarget => configTarget.url); if (remote === undefined && configUrls.length > 0) { - throw new Error( - `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, - ); + throw new Error(`Remote config detected but this feature is turned off`); } // If no paths are provided, we default to reading From 7e6186cf6b522c0130d683a8cd6dcaa5cde2f018 Mon Sep 17 00:00:00 2001 From: Elliot Greenwood Date: Wed, 27 Oct 2021 13:35:00 +0100 Subject: [PATCH 18/43] Add NotModifiedError for the AWSS3UrlReader Signed-off-by: Elliot Greenwood --- .../src/reading/AwsS3UrlReader.test.ts | 209 ++++++++++++------ .../src/reading/AwsS3UrlReader.ts | 6 +- 2 files changed, 142 insertions(+), 73 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 301543e8c9..a3ae8b647f 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -27,6 +27,7 @@ import { UrlReaderPredicateTuple } from './types'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +import { NotModifiedError } from '@backstage/errors'; const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), @@ -131,29 +132,37 @@ describe('AwsS3UrlReader', () => { }); describe('read', () => { - AWSMock.setSDKInstance(aws); - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - ); - const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), ), - ), - { s3, treeResponseFactory }, - ); + { s3, treeResponseFactory }, + ); + }); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.read( @@ -176,32 +185,39 @@ describe('AwsS3UrlReader', () => { }); describe('readUrl', () => { - AWSMock.setSDKInstance(aws); + let awsS3UrlReader: AwsS3UrlReader; - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), + beforeAll(() => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - ); + ); - const s3 = new aws.S3(); + const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), ), - ), - { s3, treeResponseFactory }, - ); + { s3, treeResponseFactory }, + ); + }); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readUrl( @@ -223,40 +239,89 @@ describe('AwsS3UrlReader', () => { ); }); }); + + describe('readUrl with etag', () => { + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock('S3', 'getObject', (_, callback) => { + callback({ statusCode: 304 }, null); + }); + + const s3 = new aws.S3(); + + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + + it('returns contents of an object in a bucket', async () => { + await expect( + awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + { + etag: 'abc123', + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); + describe('readTree', () => { - const object: aws.S3.Types.Object = { - Key: 'awsS3-mock-object.yaml', - }; - const objectList: aws.S3.ObjectList = [object]; - const output: aws.S3.Types.ListObjectsV2Output = { - Contents: objectList, - }; - AWSMock.setSDKInstance(aws); - AWSMock.mock('S3', 'listObjectsV2', output); + let awsS3UrlReader: AwsS3UrlReader; - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), - ), - ), - ); + beforeAll(() => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; - const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: '.amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - { s3, treeResponseFactory }, - ); + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readTree( 'https://test.s3.us-east-2.amazonaws.com', diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 7fd0f6ba19..05a14e26ec 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,7 +27,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; -import { ForwardedError } from '@backstage/errors'; +import { ForwardedError, NotModifiedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( @@ -163,6 +163,10 @@ export class AwsS3UrlReader implements UrlReader { etag: etag, }; } catch (e) { + if (e.statusCode === 304) { + throw new NotModifiedError(); + } + throw new ForwardedError('Could not retrieve file from S3', e); } } From 8c4cad0bf2e51533109fe6f5147fde33cac16618 Mon Sep 17 00:00:00 2001 From: Elliot Greenwood Date: Wed, 27 Oct 2021 14:33:08 +0100 Subject: [PATCH 19/43] Add changeset for backstage common Signed-off-by: Elliot Greenwood --- .changeset/few-waves-dream.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/few-waves-dream.md diff --git a/.changeset/few-waves-dream.md b/.changeset/few-waves-dream.md new file mode 100644 index 0000000000..72123171b8 --- /dev/null +++ b/.changeset/few-waves-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +AWSS3UrlReader now throws a `NotModifiedError` (exported from @backstage/backend-common) when s3 returns a 304 response. From e9206f437c8b4262d8c959546947fac046ca52ec Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Fri, 29 Oct 2021 13:39:35 -0600 Subject: [PATCH 20/43] Make theme banner warning color optional Signed-off-by: Tim Hansen --- .../src/components/DismissableBanner/DismissableBanner.tsx | 3 ++- packages/theme/src/types.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 40082925c6..92b449f705 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -80,7 +80,8 @@ const useStyles = makeStyles( backgroundColor: theme.palette.banner.error, }, warning: { - backgroundColor: theme.palette.banner.warning, + backgroundColor: + theme.palette.banner.warning ?? theme.palette.banner.error, }, }), { name: 'BackstageDismissableBanner' }, diff --git a/packages/theme/src/types.ts b/packages/theme/src/types.ts index 40f45ee739..e1acba3a01 100644 --- a/packages/theme/src/types.ts +++ b/packages/theme/src/types.ts @@ -76,7 +76,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; - warning: string; + warning?: string; }; }; From c01e26327efc80555add4bc87171308006116d1a Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Fri, 29 Oct 2021 13:43:28 -0600 Subject: [PATCH 21/43] update changeset Signed-off-by: Tim Hansen --- .changeset/dry-spies-cover.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md index 7006dee5c4..aae683faaf 100644 --- a/.changeset/dry-spies-cover.md +++ b/.changeset/dry-spies-cover.md @@ -3,4 +3,6 @@ '@backstage/theme': patch --- -Will Add warning variant to `DismissableBanner` component. +Added a warning variant to `DismissableBanner` component. If you are using a +custom theme, you will need to add the optional `palette.banner.warning` color, +otherwise this variant will fall back to the `palette.banner.error` color. From 311d604bea6d5a7885942ca1adfbec15f7a4cf72 Mon Sep 17 00:00:00 2001 From: rodion Date: Sun, 31 Oct 2021 22:43:24 +0300 Subject: [PATCH 22/43] fix: plugin sentry id token / private method Signed-off-by: rodion --- plugins/sentry/api-report.md | 11 ----------- plugins/sentry/src/api/production-api.ts | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/plugins/sentry/api-report.md b/plugins/sentry/api-report.md index 1b7a248e48..6d83ddc5e6 100644 --- a/plugins/sentry/api-report.md +++ b/plugins/sentry/api-report.md @@ -41,17 +41,6 @@ export class ProductionSentryApi implements SentryApi { identityApi?: IdentityApi | undefined, ); // (undocumented) - authOptions(): Promise< - | { - headers?: undefined; - } - | { - headers: { - authorization: string; - }; - } - >; - // (undocumented) fetchIssues( project: string, statsFor: string, diff --git a/plugins/sentry/src/api/production-api.ts b/plugins/sentry/src/api/production-api.ts index 37a735df14..1fc6ece37f 100644 --- a/plugins/sentry/src/api/production-api.ts +++ b/plugins/sentry/src/api/production-api.ts @@ -51,7 +51,7 @@ export class ProductionSentryApi implements SentryApi { return (await response.json()) as SentryIssue[]; } - async authOptions() { + private async authOptions() { if (!this.identityApi) { return {}; } From 777daa10ad6896b5cfe161721120d4d7e00c3d32 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sun, 31 Oct 2021 19:56:06 -0500 Subject: [PATCH 23/43] Fixed s3 path join to work under any os Signed-off-by: Andre Wanlin --- .changeset/tender-bugs-sort.md | 5 +++++ packages/techdocs-common/src/stages/publish/awsS3.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/tender-bugs-sort.md diff --git a/.changeset/tender-bugs-sort.md b/.changeset/tender-bugs-sort.md new file mode 100644 index 0000000000..b6ad29ba66 --- /dev/null +++ b/.changeset/tender-bugs-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Updated to properly join URL segments under any OS diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 29eecb8d03..5606ff1fd4 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -357,7 +357,7 @@ export class AwsS3Publish implements PublisherBase { : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot); // Re-prepend the root path to the relative file path - const filePath = path.join(this.bucketRootPath, filePathNoRoot); + const filePath = path.posix.join(this.bucketRootPath, filePathNoRoot); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); From 4848c9cb2094e9691340c9e68ed65e4688b47753 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sun, 31 Oct 2021 19:58:51 -0500 Subject: [PATCH 24/43] Fixed other instances of same issue Signed-off-by: Andre Wanlin --- packages/techdocs-common/src/stages/publish/awsS3.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 5606ff1fd4..f0616e4cd3 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -306,7 +306,7 @@ export class AwsS3Publish implements PublisherBase { ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); - const entityRootDir = path.join(this.bucketRootPath, entityDir); + const entityRootDir = path.posix.join(this.bucketRootPath, entityDir); const stream = this.storageClient .getObject({ @@ -396,7 +396,7 @@ export class AwsS3Publish implements PublisherBase { ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); - const entityRootDir = path.join(this.bucketRootPath, entityDir); + const entityRootDir = path.posix.join(this.bucketRootPath, entityDir); await this.storageClient .headObject({ From ef01b7c7a7a22623c3fd5e7f85d274842b1cb8db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 04:15:34 +0000 Subject: [PATCH 25/43] build(deps-dev): bump @storybook/addons from 6.3.11 to 6.3.12 Bumps [@storybook/addons](https://github.com/storybookjs/storybook/tree/HEAD/lib/addons) from 6.3.11 to 6.3.12. - [Release notes](https://github.com/storybookjs/storybook/releases) - [Changelog](https://github.com/storybookjs/storybook/blob/next/CHANGELOG.md) - [Commits](https://github.com/storybookjs/storybook/commits/v6.3.12/lib/addons) --- updated-dependencies: - dependency-name: "@storybook/addons" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 3fec636e32..e879cff365 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5531,7 +5531,7 @@ react-syntax-highlighter "^13.5.3" regenerator-runtime "^0.13.7" -"@storybook/addons@6.3.11", "@storybook/addons@^6.1.11": +"@storybook/addons@6.3.11": version "6.3.11" resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.3.11.tgz#4b5e88793bcce7ef823340e9010a96e35e3284cc" integrity sha512-2Y03lOwzWDRB/glISa/4luBMM5uyYhkIBixbZF9miIb2SCWRlNmom5NCnKsR18Wu6g7zI7os3aAMfKr24aSofQ== @@ -5561,6 +5561,21 @@ global "^4.4.0" regenerator-runtime "^0.13.7" +"@storybook/addons@^6.1.11": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.3.12.tgz#8773dcc113c5086dfff722388b7b65580e43b65b" + integrity sha512-UgoMyr7Qr0FS3ezt8u6hMEcHgyynQS9ucr5mAwZky3wpXRPFyUTmMto9r4BBUdqyUvTUj/LRKIcmLBfj+/l0Fg== + dependencies: + "@storybook/api" "6.3.12" + "@storybook/channels" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/core-events" "6.3.12" + "@storybook/router" "6.3.12" + "@storybook/theming" "6.3.12" + core-js "^3.8.2" + global "^4.4.0" + regenerator-runtime "^0.13.7" + "@storybook/api@6.3.11": version "6.3.11" resolved "https://registry.npmjs.org/@storybook/api/-/api-6.3.11.tgz#ea3806a0570da65bfb5b39e4edb90289b5ba701e" @@ -5587,6 +5602,32 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" +"@storybook/api@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/api/-/api-6.3.12.tgz#2845c20464d5348d676d09665e8ab527825ed7b5" + integrity sha512-LScRXUeCWEW/OP+jiooNMQICVdusv7azTmULxtm72fhkXFRiQs2CdRNTiqNg46JLLC9z95f1W+pGK66X6HiiQA== + dependencies: + "@reach/router" "^1.3.4" + "@storybook/channels" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/core-events" "6.3.12" + "@storybook/csf" "0.0.1" + "@storybook/router" "6.3.12" + "@storybook/semver" "^7.3.2" + "@storybook/theming" "6.3.12" + "@types/reach__router" "^1.3.7" + core-js "^3.8.2" + fast-deep-equal "^3.1.3" + global "^4.4.0" + lodash "^4.17.20" + memoizerific "^1.11.3" + qs "^6.10.0" + regenerator-runtime "^0.13.7" + store2 "^2.12.0" + telejson "^5.3.2" + ts-dedent "^2.0.0" + util-deprecate "^1.0.2" + "@storybook/api@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/api/-/api-6.3.7.tgz#88b8a51422cd0739c91bde0b1d65fb6d8a8485d0" @@ -5724,6 +5765,15 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" +"@storybook/channels@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/channels/-/channels-6.3.12.tgz#aa0d793895a8b211f0ad3459c61c1bcafd0093c7" + integrity sha512-l4sA+g1PdUV8YCbgs47fIKREdEQAKNdQIZw0b7BfTvY9t0x5yfBywgQhYON/lIeiNGz2OlIuD+VUtqYfCtNSyw== + dependencies: + core-js "^3.8.2" + ts-dedent "^2.0.0" + util-deprecate "^1.0.2" + "@storybook/channels@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/channels/-/channels-6.3.7.tgz#85ed5925522b802d959810f78d37aacde7fea66e" @@ -5789,6 +5839,14 @@ core-js "^3.8.2" global "^4.4.0" +"@storybook/client-logger@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-6.3.12.tgz#6585c98923b49fcb25dbceeeb96ef2a83e28e0f4" + integrity sha512-zNDsamZvHnuqLznDdP9dUeGgQ9TyFh4ray3t1VGO7ZqWVZ2xtVCCXjDvMnOXI2ifMpX5UsrOvshIPeE9fMBmiQ== + dependencies: + core-js "^3.8.2" + global "^4.4.0" + "@storybook/client-logger@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-6.3.7.tgz#ff17b7494e7e9e23089b0d5c5364c371c726bdd1" @@ -5941,6 +5999,13 @@ dependencies: core-js "^3.8.2" +"@storybook/core-events@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/core-events/-/core-events-6.3.12.tgz#73f6271d485ef2576234e578bb07705b92805290" + integrity sha512-SXfD7xUUMazaeFkB92qOTUV8Y/RghE4SkEYe5slAdjeocSaH7Nz2WV0rqNEgChg0AQc+JUI66no8L9g0+lw4Gw== + dependencies: + core-js "^3.8.2" + "@storybook/core-events@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/core-events/-/core-events-6.3.7.tgz#c5bc7cae7dc295de73b6b9f671ecbe582582e9bd" @@ -6135,6 +6200,22 @@ qs "^6.10.0" ts-dedent "^2.0.0" +"@storybook/router@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/router/-/router-6.3.12.tgz#0d572ec795f588ca886f39cb9b27b94ff3683f84" + integrity sha512-G/pNGCnrJRetCwyEZulHPT+YOcqEj/vkPVDTUfii2qgqukup6K0cjwgd7IukAURnAnnzTi1gmgFuEKUi8GE/KA== + dependencies: + "@reach/router" "^1.3.4" + "@storybook/client-logger" "6.3.12" + "@types/reach__router" "^1.3.7" + core-js "^3.8.2" + fast-deep-equal "^3.1.3" + global "^4.4.0" + lodash "^4.17.20" + memoizerific "^1.11.3" + qs "^6.10.0" + ts-dedent "^2.0.0" + "@storybook/router@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/router/-/router-6.3.7.tgz#1714a99a58a7b9f08b6fcfe2b678dad6ca896736" @@ -6193,6 +6274,24 @@ resolve-from "^5.0.0" ts-dedent "^2.0.0" +"@storybook/theming@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/theming/-/theming-6.3.12.tgz#5bddf9bd90a60709b5ab238ecdb7d9055dd7862e" + integrity sha512-wOJdTEa/VFyFB2UyoqyYGaZdym6EN7RALuQOAMT6zHA282FBmKw8nL5DETHEbctpnHdcrMC/391teK4nNSrdOA== + dependencies: + "@emotion/core" "^10.1.1" + "@emotion/is-prop-valid" "^0.8.6" + "@emotion/styled" "^10.0.27" + "@storybook/client-logger" "6.3.12" + core-js "^3.8.2" + deep-object-diff "^1.1.0" + emotion-theming "^10.0.27" + global "^4.4.0" + memoizerific "^1.11.3" + polished "^4.0.5" + resolve-from "^5.0.0" + ts-dedent "^2.0.0" + "@storybook/theming@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/theming/-/theming-6.3.7.tgz#6daf9a21b26ed607f3c28a82acd90c0248e76d8b" From 290fbb3ec2c6fbd4a1425e6eb906a35e4d879999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarek=20=C5=81ukow?= Date: Mon, 1 Nov 2021 11:17:00 +0100 Subject: [PATCH 26/43] Improve API docs in Scaffolder action plugins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jarek Łukow --- .changeset/wicked-boats-lie.md | 7 +++++++ .../api-report.md | 4 +--- .../src/actions/fetch/cookiecutter.ts | 9 +++++++++ .../src/index.ts | 2 +- plugins/scaffolder-backend-module-rails/api-report.md | 4 +--- .../src/actions/fetch/rails/index.ts | 10 ++++++++++ plugins/scaffolder-backend-module-rails/src/index.ts | 2 +- plugins/scaffolder-backend-module-yeoman/api-report.md | 6 +----- .../src/actions/run/yeoman.ts | 9 +++++++++ plugins/scaffolder-backend-module-yeoman/src/index.ts | 7 +++++++ 10 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 .changeset/wicked-boats-lie.md diff --git a/.changeset/wicked-boats-lie.md b/.changeset/wicked-boats-lie.md new file mode 100644 index 0000000000..8c5da92d9c --- /dev/null +++ b/.changeset/wicked-boats-lie.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-scaffolder-backend-module-cookiecutter': patch +'@backstage/plugin-scaffolder-backend-module-rails': patch +'@backstage/plugin-scaffolder-backend-module-yeoman': patch +--- + +Add missing API docs to scaffolder action plugins diff --git a/plugins/scaffolder-backend-module-cookiecutter/api-report.md b/plugins/scaffolder-backend-module-cookiecutter/api-report.md index 6fa57fd0f3..132d3d4225 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/api-report.md +++ b/plugins/scaffolder-backend-module-cookiecutter/api-report.md @@ -10,9 +10,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '@backstage/plugin-scaffolder-backend'; import { UrlReader } from '@backstage/backend-common'; -// Warning: (ae-missing-release-tag) "createFetchCookiecutterAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export function createFetchCookiecutterAction(options: { reader: UrlReader; integrations: ScmIntegrations; diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts index 5120cd3b27..c4d7f1e3e8 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -121,6 +121,15 @@ export class CookiecutterRunner { } } +/** + * Creates a `fetch:cookiecutter` Scaffolder action. + * + * @remarks + * + * See {@link https://cookiecutter.readthedocs.io/} and {@link https://backstage.io/docs/features/software-templates/writing-custom-actions}. + * @param options - Templating configuration. + * @public + */ export function createFetchCookiecutterAction(options: { reader: UrlReader; integrations: ScmIntegrations; diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/index.ts b/plugins/scaffolder-backend-module-cookiecutter/src/index.ts index cce8011059..4f0ba9a407 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/index.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/index.ts @@ -15,7 +15,7 @@ */ /** - * A module for the scaffolder backend that lets you template projects using cookiecutter + * A module for the scaffolder backend that lets you template projects using {@link https://cookiecutter.readthedocs.io/ | cookiecutter}. * * @packageDocumentation */ diff --git a/plugins/scaffolder-backend-module-rails/api-report.md b/plugins/scaffolder-backend-module-rails/api-report.md index 477483f70e..9c0610d4f2 100644 --- a/plugins/scaffolder-backend-module-rails/api-report.md +++ b/plugins/scaffolder-backend-module-rails/api-report.md @@ -8,9 +8,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '@backstage/plugin-scaffolder-backend'; import { UrlReader } from '@backstage/backend-common'; -// Warning: (ae-missing-release-tag) "createFetchRailsAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export function createFetchRailsAction(options: { reader: UrlReader; integrations: ScmIntegrations; diff --git a/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.ts b/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.ts index 5ce7d46689..00a5690c0a 100644 --- a/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.ts +++ b/plugins/scaffolder-backend-module-rails/src/actions/fetch/rails/index.ts @@ -27,6 +27,16 @@ import { import { resolve as resolvePath } from 'path'; import { RailsNewRunner } from './railsNewRunner'; +/** + * Creates the `fetch:rails` Scaffolder action. + * + * @remarks + * + * See {@link https://guides.rubyonrails.org/rails_application_templates.html} and {@link https://backstage.io/docs/features/software-templates/writing-custom-actions}. + * + * @param options - Configuration of the templater. + * @public + */ export function createFetchRailsAction(options: { reader: UrlReader; integrations: ScmIntegrations; diff --git a/plugins/scaffolder-backend-module-rails/src/index.ts b/plugins/scaffolder-backend-module-rails/src/index.ts index eb51c06c49..773e1feabd 100644 --- a/plugins/scaffolder-backend-module-rails/src/index.ts +++ b/plugins/scaffolder-backend-module-rails/src/index.ts @@ -15,7 +15,7 @@ */ /** - * A module for the scaffolder backend that lets you template projects using Rails + * A module for the scaffolder backend that lets you template projects using {@link https://guides.rubyonrails.org/rails_application_templates.html | Rails}. * * @packageDocumentation */ diff --git a/plugins/scaffolder-backend-module-yeoman/api-report.md b/plugins/scaffolder-backend-module-yeoman/api-report.md index 1f38d3945c..fc18814faa 100644 --- a/plugins/scaffolder-backend-module-yeoman/api-report.md +++ b/plugins/scaffolder-backend-module-yeoman/api-report.md @@ -5,10 +5,6 @@ ```ts import { TemplateAction } from '@backstage/plugin-scaffolder-backend'; -// Warning: (ae-missing-release-tag) "createRunYeomanAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export function createRunYeomanAction(): TemplateAction; - -// (No @packageDocumentation comment for this package) ``` diff --git a/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts b/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts index f6a29037a3..f7b0c54b41 100644 --- a/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts +++ b/plugins/scaffolder-backend-module-yeoman/src/actions/run/yeoman.ts @@ -18,6 +18,15 @@ import { JsonObject } from '@backstage/types'; import { createTemplateAction } from '@backstage/plugin-scaffolder-backend'; import { yeomanRun } from './yeomanRun'; +/** + * Creates a `run:yeoman` Scaffolder action. + * + * @remarks + * + * See {@link https://yeoman.io/} and {@link https://backstage.io/docs/features/software-templates/writing-custom-actions}. + * + * @public + */ export function createRunYeomanAction() { return createTemplateAction<{ namespace: string; diff --git a/plugins/scaffolder-backend-module-yeoman/src/index.ts b/plugins/scaffolder-backend-module-yeoman/src/index.ts index 4f06b14a86..1aac8e12f9 100644 --- a/plugins/scaffolder-backend-module-yeoman/src/index.ts +++ b/plugins/scaffolder-backend-module-yeoman/src/index.ts @@ -13,4 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/** + * A module for the scaffolder backend that lets you template projects using + * {@link https://yeoman.io/ | Yeoman}. + * + * @packageDocumentation + */ export * from './actions'; From 9587f6f7d26ad9cff89ed0f2fb9f310d86b8cf36 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 25 Oct 2021 11:26:13 +0200 Subject: [PATCH 27/43] Add tests for the scaffolder eventstream Signed-off-by: Dominik Henneke --- .../src/service/router.test.ts | 179 ++++++++++++++++-- .../scaffolder-backend/src/service/router.ts | 43 +++-- 2 files changed, 183 insertions(+), 39 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 1c5343d458..5157af4eaf 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -29,17 +29,17 @@ jest.doMock('fs-extra', () => ({ })); import { + DatabaseManager, + DockerContainerRunner, getVoidLogger, PluginDatabaseManager, - DatabaseManager, UrlReaders, - DockerContainerRunner, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; -import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; /** * TODO: The following should import directly from the router file. * Due to a circular dependency between this plugin and the @@ -47,6 +47,12 @@ import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function */ import { createRouter } from '../index'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; + +jest.mock('../scaffolder/tasks'); + +const MockStorageTaskBroker: jest.MockedClass = + StorageTaskBroker as any; const createCatalogClient = (templates: any[] = []) => ({ @@ -102,7 +108,7 @@ describe('createRouter', () => { }, }; - beforeAll(async () => { + beforeEach(async () => { const router = await createRouter({ logger: getVoidLogger(), config: new ConfigReader({}), @@ -114,7 +120,7 @@ describe('createRouter', () => { app = express().use(router); }); - beforeEach(() => { + afterEach(() => { jest.resetAllMocks(); }); @@ -142,6 +148,10 @@ describe('createRouter', () => { }); it('return the template id', async () => { + MockStorageTaskBroker.prototype.dispatch.mockResolvedValue({ + taskId: 'a-random-id', + }); + const response = await request(app) .post('/v2/tasks') .send({ @@ -151,28 +161,161 @@ describe('createRouter', () => { }, }); - expect(response.body.id).toBeDefined(); + expect(response.body.id).toBe('a-random-id'); expect(response.status).toEqual(201); }); }); describe('GET /v2/tasks/:taskId', () => { it('does not divulge secrets', async () => { - const postResponse = await request(app) - .post('/v2/tasks') - .set('Authorization', 'Bearer secret') - .send({ - templateName: 'create-react-app-template', - values: { - required: 'required-value', - }, - }); + MockStorageTaskBroker.prototype.get.mockResolvedValue({ + id: 'a-random-id', + spec: {} as any, + status: 'completed', + createdAt: '', + secrets: { token: 'secret' }, + }); - const response = await request(app) - .get(`/v2/tasks/${postResponse.body.id}`) - .send(); + const response = await request(app).get(`/v2/tasks/a-random-id`); expect(response.status).toEqual(200); + expect(response.body.status).toBe('completed'); expect(response.body.secrets).toBeUndefined(); }); }); + + describe('GET /v2/tasks/:taskId/eventstream', () => { + it('should return log messages', async () => { + const unsubscribe = jest.fn(); + MockStorageTaskBroker.prototype.observe.mockImplementation( + ({ taskId }, callback) => { + // emit after this function returned + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ], + }); + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + }); + + return unsubscribe; + }, + ); + + let statusCode: any = undefined; + let headers: any = {}; + const responseDataFn = jest.fn(); + + const req = request(app) + .get('/v2/tasks/a-random-id/eventstream') + .set('accept', 'text/event-stream') + .parse((res, _) => { + ({ statusCode, headers } = res as any); + + res.on('data', chunk => { + responseDataFn(chunk.toString()); + + // the server expects the client to abort the request + if (chunk.includes('completion')) { + req.abort(); + } + }); + }); + + // wait for the request to finish + await req.catch(() => { + // ignore 'aborted' error + }); + + expect(statusCode).toBe(200); + expect(headers['content-type']).toBe('text/event-stream'); + expect(responseDataFn).toBeCalledTimes(2); + expect(responseDataFn).toBeCalledWith(`event: log +data: {"id":0,"taskId":"a-random-id","type":"log","createdAt":"","body":{"message":"My log message"}} + +`); + expect(responseDataFn).toBeCalledWith(`event: completion +data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{"message":"Finished!"}} + +`); + + expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); + expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + { taskId: 'a-random-id' }, + expect.any(Function), + ); + + expect(unsubscribe).toBeCalledTimes(1); + }); + + it('should return log messages with after query', async () => { + const unsubscribe = jest.fn(); + MockStorageTaskBroker.prototype.observe.mockImplementation( + ({ taskId }, callback) => { + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + }); + return unsubscribe; + }, + ); + + let statusCode: any = undefined; + let headers: any = {}; + + const req = request(app) + .get('/v2/tasks/a-random-id/eventstream') + .query({ after: 10 }) + .set('accept', 'text/event-stream') + .parse((res, _) => { + ({ statusCode, headers } = res as any); + + res.on('data', () => { + // close immediately + req.abort(); + }); + }); + + // wait for the request to finish + await req.catch(() => { + // ignore 'aborted' error + }); + + expect(statusCode).toBe(200); + expect(headers['content-type']).toBe('text/event-stream'); + + expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); + expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + { taskId: 'a-random-id', after: 10 }, + expect.any(Function), + ); + + expect(unsubscribe).toBeCalledTimes(1); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a78b04da78..fce3b885c6 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -14,33 +14,33 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import express from 'express'; -import Router from 'express-promise-router'; -import { Logger } from 'winston'; -import { CatalogEntityClient } from '../lib/catalog'; -import { validate } from 'jsonschema'; -import { - DatabaseTaskStore, - TemplateActionRegistry, - TaskWorker, - TemplateAction, - createBuiltinActions, -} from '../scaffolder'; -import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; -import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, PluginDatabaseManager, UrlReader, } from '@backstage/backend-common'; -import { InputError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; -import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; -import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; - +import { Entity, TemplateEntityV1beta2 } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { InputError, NotFoundError } from '@backstage/errors'; import { ScmIntegrations } from '@backstage/integration'; -import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; +import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; +import express from 'express'; +import Router from 'express-promise-router'; +import { validate } from 'jsonschema'; +import { Logger } from 'winston'; +import { CatalogEntityClient } from '../lib/catalog'; +import { + createBuiltinActions, + DatabaseTaskStore, + TaskBroker, + TaskSpec, + TaskWorker, + TemplateAction, + TemplateActionRegistry, +} from '../scaffolder'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; +import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; /** * RouterOptions @@ -278,7 +278,8 @@ export async function createRouter( // to automatically reconnect because it lost connection. } } - res.flush(); + // res.flush() is only available with the compression middleware + res.flush?.(); if (shouldUnsubscribe) unsubscribe(); }, ); From 95a48698bc5d64195076790d00baaebf07ee01e6 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 25 Oct 2021 14:17:37 +0200 Subject: [PATCH 28/43] Add tests for the scaffolder eventstream frontend client Signed-off-by: Dominik Henneke --- plugins/scaffolder/package.json | 1 + plugins/scaffolder/src/api.test.ts | 76 +++++++++++++++++++++++++--- plugins/scaffolder/src/setupTests.ts | 3 ++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 0b517671ba..20b8e934f7 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -77,6 +77,7 @@ "@types/jest": "^26.0.7", "@types/node": "^14.14.32", "cross-fetch": "^3.0.6", + "event-source-polyfill": "^1.0.25", "msw": "^0.35.0" }, "files": [ diff --git a/plugins/scaffolder/src/api.test.ts b/plugins/scaffolder/src/api.test.ts index cc4b70283c..f11b2cf06c 100644 --- a/plugins/scaffolder/src/api.test.ts +++ b/plugins/scaffolder/src/api.test.ts @@ -14,12 +14,18 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/core-app-api'; import { ScmIntegrations } from '@backstage/integration'; import { ScaffolderClient } from './api'; -import { ConfigReader } from '@backstage/core-app-api'; + +const MockedEventSource = global.EventSource as jest.MockedClass< + typeof EventSource +>; describe('api', () => { - const discoveryApi = {} as any; + const mockBaseUrl = 'http://backstage/api'; + + const discoveryApi = { getBaseUrl: async () => mockBaseUrl }; const identityApi = {} as any; const scmIntegrationsApi = ScmIntegrations.fromConfig( new ConfigReader({ @@ -32,10 +38,14 @@ describe('api', () => { }, }), ); - const apiClient = new ScaffolderClient({ - scmIntegrationsApi, - discoveryApi, - identityApi, + + let apiClient: ScaffolderClient; + beforeEach(() => { + apiClient = new ScaffolderClient({ + scmIntegrationsApi, + discoveryApi, + identityApi, + }); }); it('should return default and custom integrations', async () => { @@ -51,4 +61,58 @@ describe('api', () => { expect(allowedHosts).toContain(integration.host), ); }); + + describe('streamEvents', () => { + describe('eventsource', () => { + it('should work', async () => { + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (typeof fn !== 'function') { + return; + } + + if (type === 'log') { + fn({ + data: '{"id":1,"taskId":"a-random-id","type":"log","createdAt":"","body":{"message":"My log message"}}', + } as any); + } else if (type === 'completion') { + fn({ + data: '{"id":2,"taskId":"a-random-id","type":"completion","createdAt":"","body":{"message":"Finished!"}}', + } as any); + } + }, + ); + + const next = jest.fn(); + + await new Promise(complete => { + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }); + }); + + expect(MockedEventSource).toBeCalledWith( + 'http://backstage/api/v2/tasks/a-random-task-id/eventstream', + { withCredentials: true }, + ); + expect(MockedEventSource.prototype.close).toBeCalled(); + + expect(next).toBeCalledTimes(2); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + }); + }); }); diff --git a/plugins/scaffolder/src/setupTests.ts b/plugins/scaffolder/src/setupTests.ts index 963c0f188b..84589060ec 100644 --- a/plugins/scaffolder/src/setupTests.ts +++ b/plugins/scaffolder/src/setupTests.ts @@ -15,3 +15,6 @@ */ import '@testing-library/jest-dom'; + +const { EventSourcePolyfill } = jest.requireMock('event-source-polyfill'); +global.EventSource = EventSourcePolyfill; From b45a34fb15e50593470afc73c14647d4d78a9ed2 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Mon, 25 Oct 2021 11:27:28 +0200 Subject: [PATCH 29/43] Adds a new endpoint for consuming logs from the Scaffolder that uses long polling instead of Server Sent Events Signed-off-by: Dominik Henneke --- .changeset/selfish-countries-prove.md | 42 ++++ .../src/service/router.test.ts | 80 ++++++++ .../scaffolder-backend/src/service/router.ts | 41 +++- plugins/scaffolder/api-report.md | 9 +- plugins/scaffolder/package.json | 1 + plugins/scaffolder/src/api.test.ts | 179 ++++++++++++++++++ plugins/scaffolder/src/api.ts | 66 ++++++- plugins/scaffolder/src/setupTests.ts | 1 + 8 files changed, 404 insertions(+), 15 deletions(-) create mode 100644 .changeset/selfish-countries-prove.md diff --git a/.changeset/selfish-countries-prove.md b/.changeset/selfish-countries-prove.md new file mode 100644 index 0000000000..03e36f6d80 --- /dev/null +++ b/.changeset/selfish-countries-prove.md @@ -0,0 +1,42 @@ +--- +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Adds a new endpoint for consuming logs from the Scaffolder that uses long polling instead of Server Sent Events. + +This is useful if Backstage is accessed from an environment that doesn't support SSE correctly, which happens in combination with certain enterprise HTTP Proxy servers. + +It is intended to switch the endpoint globally for the whole instance. +If you want to use it, you can provide a reconfigured API to the `scaffolderApiRef`: + +```tsx +// packages/app/src/apis.ts + +// ... +import { + scaffolderApiRef, + ScaffolderClient, +} from '@backstage/plugin-scaffolder'; + +export const apis: AnyApiFactory[] = [ + // ... + + createApiFactory({ + api: scaffolderApiRef, + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + scmIntegrationsApi: scmIntegrationsApiRef, + }, + factory: ({ discoveryApi, identityApi, scmIntegrationsApi }) => + new ScaffolderClient({ + discoveryApi, + identityApi, + scmIntegrationsApi, + // use long polling instead of an eventsource + useLongPollingLogs: true, + }), + }), +]; +``` diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 5157af4eaf..d6136fe872 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -318,4 +318,84 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ expect(unsubscribe).toBeCalledTimes(1); }); }); + + describe('GET /v2/tasks/:taskId/logs', () => { + it('should return log messages', async () => { + const unsubscribe = jest.fn(); + MockStorageTaskBroker.prototype.observe.mockImplementation( + ({ taskId }, callback) => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + return unsubscribe; + }, + ); + + const response = await request(app).get('/v2/tasks/a-random-id/logs'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([ + { + id: 0, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + { + id: 1, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]); + + expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); + expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + { taskId: 'a-random-id' }, + expect.any(Function), + ); + expect(unsubscribe).toBeCalledTimes(1); + }); + + it('should return log messages with after query', async () => { + const unsubscribe = jest.fn(); + MockStorageTaskBroker.prototype.observe.mockImplementation( + (_, callback) => { + callback(undefined, { events: [] }); + return unsubscribe; + }, + ); + + const response = await request(app) + .get('/v2/tasks/a-random-id/logs') + .query({ after: 10 }); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([]); + + expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); + expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + { taskId: 'a-random-id', after: 10 }, + expect.any(Function), + ); + expect(unsubscribe).toBeCalledTimes(1); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fce3b885c6..076a7b0036 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -247,7 +247,9 @@ export async function createRouter( }) .get('/v2/tasks/:taskId/eventstream', async (req, res) => { const { taskId } = req.params; - const after = Number(req.query.after) || undefined; + const after = + req.query.after !== undefined ? Number(req.query.after) : undefined; + logger.debug(`Event stream observing taskId '${taskId}' opened`); // Mandatory headers and http status to keep connection open @@ -289,6 +291,43 @@ export async function createRouter( unsubscribe(); logger.debug(`Event stream observing taskId '${taskId}' closed`); }); + }) + .get('/v2/tasks/:taskId/logs', async (req, res) => { + const { taskId } = req.params; + const after = Number(req.query.after) || undefined; + + let unsubscribe = () => {}; + + // cancel the request after 30 seconds. this aligns with the recommendations of RFC 6202. + const timeout = setTimeout(() => { + unsubscribe(); + res.json([]); + }, 30_000); + + // Get all known events after an id (always includes the completion event) and return the first callback + unsubscribe = taskBroker.observe( + { taskId, after }, + (error, { events }) => { + // stop the timeout + clearTimeout(timeout); + unsubscribe(); + + if (error) { + logger.error( + `Received error from log when observing taskId '${taskId}', ${error}`, + ); + } + + res.json(events); + }, + ); + + // When client closes connection we update the clients list + // avoiding the disconnected one + req.on('close', () => { + unsubscribe(); + clearTimeout(timeout); + }); }); const app = express(); diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 554fccabb7..0ac6910264 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -178,6 +178,7 @@ export class ScaffolderClient implements ScaffolderApi { discoveryApi: DiscoveryApi; identityApi: IdentityApi; scmIntegrationsApi: ScmIntegrationRegistry; + useLongPollingLogs?: boolean; }); // (undocumented) getIntegrationsList(options: { allowedHosts: string[] }): Promise< @@ -199,13 +200,7 @@ export class ScaffolderClient implements ScaffolderApi { // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen scaffold(templateName: string, values: Record): Promise; // (undocumented) - streamLogs({ - taskId, - after, - }: { - taskId: string; - after?: number; - }): Observable; + streamLogs(opts: { taskId: string; after?: number }): Observable; } // Warning: (ae-missing-release-tag) "ScaffolderFieldExtensions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 20b8e934f7..58bee32201 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -55,6 +55,7 @@ "json-schema": "^0.3.0", "lodash": "^4.17.21", "luxon": "^2.0.2", + "qs": "^6.9.4", "react": "^16.13.1", "react-dom": "^16.13.1", "react-lazylog": "^4.5.2", diff --git a/plugins/scaffolder/src/api.test.ts b/plugins/scaffolder/src/api.test.ts index f11b2cf06c..8b53029cd7 100644 --- a/plugins/scaffolder/src/api.test.ts +++ b/plugins/scaffolder/src/api.test.ts @@ -16,13 +16,19 @@ import { ConfigReader } from '@backstage/core-app-api'; import { ScmIntegrations } from '@backstage/integration'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; import { ScaffolderClient } from './api'; const MockedEventSource = global.EventSource as jest.MockedClass< typeof EventSource >; +const server = setupServer(); + describe('api', () => { + setupRequestMockHandlers(server); const mockBaseUrl = 'http://backstage/api'; const discoveryApi = { getBaseUrl: async () => mockBaseUrl }; @@ -114,5 +120,178 @@ describe('api', () => { }); }); }); + + describe('longPolling', () => { + beforeEach(() => { + apiClient = new ScaffolderClient({ + scmIntegrationsApi, + discoveryApi, + identityApi, + useLongPollingLogs: true, + }); + }); + + it('should work', async () => { + server.use( + rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (req, res, ctx) => { + const { taskId } = req.params; + const after = req.url.searchParams.get('after'); + + if (taskId === 'a-random-task-id') { + if (!after) { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } else if (after === '1') { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } + } + + return res(ctx.status(500)); + }), + ); + + const next = jest.fn(); + + await new Promise(complete => + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }), + ); + + expect(next).toBeCalledTimes(2); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + + it('should unsubscribe', async () => { + expect.assertions(3); + + server.use( + rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (req, res, ctx) => { + const { taskId } = req.params; + + const after = req.url.searchParams.get('after'); + + // use assertion to make sure it is not called after unsubscribing + expect(after).toBe(null); + + if (taskId === 'a-random-task-id') { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } + + return res(ctx.status(500)); + }), + ); + + const next = jest.fn(); + + await new Promise(complete => { + const subscription = apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ + next: (...args) => { + next(...args); + subscription.unsubscribe(); + complete(); + }, + }); + }); + + expect(next).toBeCalledTimes(1); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + }); + + it('should continue after error', async () => { + const called = jest.fn(); + + server.use( + rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (_req, res, ctx) => { + called(); + + if (called.mock.calls.length > 1) { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } + + return res(ctx.status(500)); + }), + ); + + const next = jest.fn(); + + await new Promise(complete => + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }), + ); + + expect(called).toBeCalledTimes(2); + + expect(next).toBeCalledTimes(1); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + }); }); }); diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 1b531b652d..5f2874d653 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -15,17 +15,18 @@ */ import { EntityName } from '@backstage/catalog-model'; -import { JsonObject, JsonValue, Observable } from '@backstage/types'; -import { ResponseError } from '@backstage/errors'; -import { ScmIntegrationRegistry } from '@backstage/integration'; -import { Field, FieldValidation } from '@rjsf/core'; -import ObservableImpl from 'zen-observable'; -import { ListActionsResponse, ScaffolderTask, Status } from './types'; import { createApiRef, DiscoveryApi, IdentityApi, } from '@backstage/core-plugin-api'; +import { ResponseError } from '@backstage/errors'; +import { ScmIntegrationRegistry } from '@backstage/integration'; +import { JsonObject, JsonValue, Observable } from '@backstage/types'; +import { Field, FieldValidation } from '@rjsf/core'; +import qs from 'qs'; +import ObservableImpl from 'zen-observable'; +import { ListActionsResponse, ScaffolderTask, Status } from './types'; export const scaffolderApiRef = createApiRef({ id: 'plugin.scaffolder.service', @@ -94,15 +95,18 @@ export class ScaffolderClient implements ScaffolderApi { private readonly discoveryApi: DiscoveryApi; private readonly identityApi: IdentityApi; private readonly scmIntegrationsApi: ScmIntegrationRegistry; + private readonly useLongPollingLogs: boolean; constructor(options: { discoveryApi: DiscoveryApi; identityApi: IdentityApi; scmIntegrationsApi: ScmIntegrationRegistry; + useLongPollingLogs?: boolean; }) { this.discoveryApi = options.discoveryApi; this.identityApi = options.identityApi; this.scmIntegrationsApi = options.scmIntegrationsApi; + this.useLongPollingLogs = options.useLongPollingLogs ?? false; } async getIntegrationsList(options: { allowedHosts: string[] }) { @@ -189,7 +193,15 @@ export class ScaffolderClient implements ScaffolderApi { return await response.json(); } - streamLogs({ + streamLogs(opts: { taskId: string; after?: number }): Observable { + if (this.useLongPollingLogs) { + return this.streamLogsPolling(opts); + } + + return this.streamLogsEventStream(opts); + } + + private streamLogsEventStream({ taskId, after, }: { @@ -239,6 +251,46 @@ export class ScaffolderClient implements ScaffolderApi { }); } + private streamLogsPolling({ + taskId, + after: inputAfter, + }: { + taskId: string; + after?: number; + }): Observable { + let after = inputAfter; + + return new ObservableImpl(subscriber => { + this.discoveryApi.getBaseUrl('scaffolder').then(async baseUrl => { + while (!subscriber.closed) { + const url = `${baseUrl}/v2/tasks/${encodeURIComponent( + taskId, + )}/logs?${qs.stringify({ after })}`; + const response = await fetch(url); + + if (!response.ok) { + // wait for one second to not run into an + await new Promise(resolve => setTimeout(resolve, 1000)); + continue; + } + + const logs = (await response.json()) as LogEvent[]; + + for (const event of logs) { + after = Number(event.id); + + subscriber.next(event); + + if (event.type === 'completion') { + subscriber.complete(); + return; + } + } + } + }); + }); + } + /** * @returns ListActionsResponse containing all registered actions. */ diff --git a/plugins/scaffolder/src/setupTests.ts b/plugins/scaffolder/src/setupTests.ts index 84589060ec..10252413a7 100644 --- a/plugins/scaffolder/src/setupTests.ts +++ b/plugins/scaffolder/src/setupTests.ts @@ -15,6 +15,7 @@ */ import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; const { EventSourcePolyfill } = jest.requireMock('event-source-polyfill'); global.EventSource = EventSourcePolyfill; From 471b9243dead0d1fa0fe027137307096f30d2e91 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Thu, 28 Oct 2021 15:46:16 +0200 Subject: [PATCH 30/43] Rename endpoint to /events Signed-off-by: Dominik Henneke --- .../src/service/router.test.ts | 6 +- .../scaffolder-backend/src/service/router.ts | 2 +- plugins/scaffolder/src/api.test.ts | 145 ++++++++++-------- plugins/scaffolder/src/api.ts | 2 +- 4 files changed, 82 insertions(+), 73 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index d6136fe872..5b9b248097 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -319,7 +319,7 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ }); }); - describe('GET /v2/tasks/:taskId/logs', () => { + describe('GET /v2/tasks/:taskId/events', () => { it('should return log messages', async () => { const unsubscribe = jest.fn(); MockStorageTaskBroker.prototype.observe.mockImplementation( @@ -346,7 +346,7 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ }, ); - const response = await request(app).get('/v2/tasks/a-random-id/logs'); + const response = await request(app).get('/v2/tasks/a-random-id/events'); expect(response.status).toEqual(200); expect(response.body).toEqual([ @@ -384,7 +384,7 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ ); const response = await request(app) - .get('/v2/tasks/a-random-id/logs') + .get('/v2/tasks/a-random-id/events') .query({ after: 10 }); expect(response.status).toEqual(200); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 076a7b0036..35bedb3651 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -292,7 +292,7 @@ export async function createRouter( logger.debug(`Event stream observing taskId '${taskId}' closed`); }); }) - .get('/v2/tasks/:taskId/logs', async (req, res) => { + .get('/v2/tasks/:taskId/events', async (req, res) => { const { taskId } = req.params; const after = Number(req.query.after) || undefined; diff --git a/plugins/scaffolder/src/api.test.ts b/plugins/scaffolder/src/api.test.ts index 8b53029cd7..01463a6e1c 100644 --- a/plugins/scaffolder/src/api.test.ts +++ b/plugins/scaffolder/src/api.test.ts @@ -133,40 +133,43 @@ describe('api', () => { it('should work', async () => { server.use( - rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (req, res, ctx) => { - const { taskId } = req.params; - const after = req.url.searchParams.get('after'); + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (req, res, ctx) => { + const { taskId } = req.params; + const after = req.url.searchParams.get('after'); - if (taskId === 'a-random-task-id') { - if (!after) { - return res( - ctx.json([ - { - id: 1, - taskId: 'a-random-id', - type: 'log', - createdAt: '', - body: { message: 'My log message' }, - }, - ]), - ); - } else if (after === '1') { - return res( - ctx.json([ - { - id: 2, - taskId: 'a-random-id', - type: 'completion', - createdAt: '', - body: { message: 'Finished!' }, - }, - ]), - ); + if (taskId === 'a-random-task-id') { + if (!after) { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } else if (after === '1') { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } } - } - return res(ctx.status(500)); - }), + return res(ctx.status(500)); + }, + ), ); const next = jest.fn(); @@ -198,30 +201,33 @@ describe('api', () => { expect.assertions(3); server.use( - rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (req, res, ctx) => { - const { taskId } = req.params; + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (req, res, ctx) => { + const { taskId } = req.params; - const after = req.url.searchParams.get('after'); + const after = req.url.searchParams.get('after'); - // use assertion to make sure it is not called after unsubscribing - expect(after).toBe(null); + // use assertion to make sure it is not called after unsubscribing + expect(after).toBe(null); - if (taskId === 'a-random-task-id') { - return res( - ctx.json([ - { - id: 1, - taskId: 'a-random-id', - type: 'log', - createdAt: '', - body: { message: 'My log message' }, - }, - ]), - ); - } + if (taskId === 'a-random-task-id') { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } - return res(ctx.status(500)); - }), + return res(ctx.status(500)); + }, + ), ); const next = jest.fn(); @@ -252,25 +258,28 @@ describe('api', () => { const called = jest.fn(); server.use( - rest.get(`${mockBaseUrl}/v2/tasks/:taskId/logs`, (_req, res, ctx) => { - called(); + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (_req, res, ctx) => { + called(); - if (called.mock.calls.length > 1) { - return res( - ctx.json([ - { - id: 2, - taskId: 'a-random-id', - type: 'completion', - createdAt: '', - body: { message: 'Finished!' }, - }, - ]), - ); - } + if (called.mock.calls.length > 1) { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } - return res(ctx.status(500)); - }), + return res(ctx.status(500)); + }, + ), ); const next = jest.fn(); diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 5f2874d653..a16ab59053 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -265,7 +265,7 @@ export class ScaffolderClient implements ScaffolderApi { while (!subscriber.closed) { const url = `${baseUrl}/v2/tasks/${encodeURIComponent( taskId, - )}/logs?${qs.stringify({ after })}`; + )}/events?${qs.stringify({ after })}`; const response = await fetch(url); if (!response.ok) { From 7c7b0b27cb3fcc3a8a7bfefaee9fb51d9d03babc Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 2 Nov 2021 13:51:17 +0100 Subject: [PATCH 31/43] Adapt code to the refactored classes and types Signed-off-by: Dominik Henneke --- .../src/service/router.test.ts | 191 +++++++++--------- .../scaffolder-backend/src/service/router.ts | 4 +- 2 files changed, 102 insertions(+), 93 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 5b9b248097..d8ba15279f 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -46,14 +46,9 @@ import request from 'supertest'; * plugin-scaffolder-backend-module-cookiecutter plugin, it results in an error: * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function */ -import { createRouter } from '../index'; +import { createRouter, DatabaseTaskStore, TaskBroker } from '../index'; import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; -jest.mock('../scaffolder/tasks'); - -const MockStorageTaskBroker: jest.MockedClass = - StorageTaskBroker as any; - const createCatalogClient = (templates: any[] = []) => ({ getEntities: async () => ({ items: templates }), @@ -79,6 +74,7 @@ const mockUrlReader = UrlReaders.default({ describe('createRouter', () => { let app: express.Express; + let taskBroker: TaskBroker; const template: TemplateEntityV1beta2 = { apiVersion: 'backstage.io/v1beta2', kind: 'Template', @@ -109,6 +105,16 @@ describe('createRouter', () => { }; beforeEach(async () => { + const logger = getVoidLogger(); + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await createDatabase().getClient(), + }); + taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + + jest.spyOn(taskBroker, 'dispatch'); + jest.spyOn(taskBroker, 'get'); + jest.spyOn(taskBroker, 'observe'); + const router = await createRouter({ logger: getVoidLogger(), config: new ConfigReader({}), @@ -116,6 +122,7 @@ describe('createRouter', () => { catalogClient: createCatalogClient([template]), containerRunner: new DockerContainerRunner({} as any), reader: mockUrlReader, + taskBroker, }); app = express().use(router); }); @@ -148,7 +155,9 @@ describe('createRouter', () => { }); it('return the template id', async () => { - MockStorageTaskBroker.prototype.dispatch.mockResolvedValue({ + ( + taskBroker.dispatch as jest.Mocked['dispatch'] + ).mockResolvedValue({ taskId: 'a-random-id', }); @@ -168,7 +177,7 @@ describe('createRouter', () => { describe('GET /v2/tasks/:taskId', () => { it('does not divulge secrets', async () => { - MockStorageTaskBroker.prototype.get.mockResolvedValue({ + (taskBroker.get as jest.Mocked['get']).mockResolvedValue({ id: 'a-random-id', spec: {} as any, status: 'completed', @@ -186,37 +195,37 @@ describe('createRouter', () => { describe('GET /v2/tasks/:taskId/eventstream', () => { it('should return log messages', async () => { const unsubscribe = jest.fn(); - MockStorageTaskBroker.prototype.observe.mockImplementation( - ({ taskId }, callback) => { - // emit after this function returned - setImmediate(() => { - callback(undefined, { - events: [ - { - id: 0, - taskId, - type: 'log', - createdAt: '', - body: { message: 'My log message' }, - }, - ], - }); - callback(undefined, { - events: [ - { - id: 1, - taskId, - type: 'completion', - createdAt: '', - body: { message: 'Finished!' }, - }, - ], - }); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + // emit after this function returned + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ], }); + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + }); - return unsubscribe; - }, - ); + return { unsubscribe }; + }); let statusCode: any = undefined; let headers: any = {}; @@ -255,8 +264,8 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ `); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( { taskId: 'a-random-id' }, expect.any(Function), ); @@ -266,24 +275,24 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ it('should return log messages with after query', async () => { const unsubscribe = jest.fn(); - MockStorageTaskBroker.prototype.observe.mockImplementation( - ({ taskId }, callback) => { - setImmediate(() => { - callback(undefined, { - events: [ - { - id: 1, - taskId, - type: 'completion', - createdAt: '', - body: { message: 'Finished!' }, - }, - ], - }); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], }); - return unsubscribe; - }, - ); + }); + return { unsubscribe }; + }); let statusCode: any = undefined; let headers: any = {}; @@ -309,8 +318,8 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ expect(statusCode).toBe(200); expect(headers['content-type']).toBe('text/event-stream'); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( { taskId: 'a-random-id', after: 10 }, expect.any(Function), ); @@ -322,29 +331,29 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ describe('GET /v2/tasks/:taskId/events', () => { it('should return log messages', async () => { const unsubscribe = jest.fn(); - MockStorageTaskBroker.prototype.observe.mockImplementation( - ({ taskId }, callback) => { - callback(undefined, { - events: [ - { - id: 0, - taskId, - type: 'log', - createdAt: '', - body: { message: 'My log message' }, - }, - { - id: 1, - taskId, - type: 'completion', - createdAt: '', - body: { message: 'Finished!' }, - }, - ], - }); - return unsubscribe; - }, - ); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + return { unsubscribe }; + }); const response = await request(app).get('/v2/tasks/a-random-id/events'); @@ -366,8 +375,8 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ }, ]); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( { taskId: 'a-random-id' }, expect.any(Function), ); @@ -376,12 +385,12 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ it('should return log messages with after query', async () => { const unsubscribe = jest.fn(); - MockStorageTaskBroker.prototype.observe.mockImplementation( - (_, callback) => { - callback(undefined, { events: [] }); - return unsubscribe; - }, - ); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation((_, callback) => { + callback(undefined, { events: [] }); + return { unsubscribe }; + }); const response = await request(app) .get('/v2/tasks/a-random-id/events') @@ -390,8 +399,8 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{ expect(response.status).toEqual(200); expect(response.body).toEqual([]); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledTimes(1); - expect(MockStorageTaskBroker.prototype.observe).toBeCalledWith( + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( { taskId: 'a-random-id', after: 10 }, expect.any(Function), ); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 35bedb3651..2ae30593f2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -305,7 +305,7 @@ export async function createRouter( }, 30_000); // Get all known events after an id (always includes the completion event) and return the first callback - unsubscribe = taskBroker.observe( + ({ unsubscribe } = taskBroker.observe( { taskId, after }, (error, { events }) => { // stop the timeout @@ -320,7 +320,7 @@ export async function createRouter( res.json(events); }, - ); + )); // When client closes connection we update the clients list // avoiding the disconnected one From 7240d33df360b4f4ac67f74e803eec71da8561c6 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Wed, 3 Nov 2021 07:30:56 -0500 Subject: [PATCH 32/43] Also fixing for Google Storage Signed-off-by: Andre Wanlin --- .../techdocs-common/src/stages/publish/googleStorage.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 5a142e800a..77c366315c 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -231,7 +231,7 @@ export class GoogleGCSPublish implements PublisherBase { ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); - const entityRootDir = path.join(this.bucketRootPath, entityDir); + const entityRootDir = path.posix.join(this.bucketRootPath, entityDir); const fileStreamChunks: Array = []; this.storageClient @@ -270,7 +270,7 @@ export class GoogleGCSPublish implements PublisherBase { : lowerCaseEntityTripletInStoragePath(decodedUriNoRoot); // Re-prepend the root path to the relative file path - const filePath = path.join(this.bucketRootPath, filePathNoRoot); + const filePath = path.posix.join(this.bucketRootPath, filePathNoRoot); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -310,7 +310,7 @@ export class GoogleGCSPublish implements PublisherBase { ? entityTriplet : lowerCaseEntityTriplet(entityTriplet); - const entityRootDir = path.join(this.bucketRootPath, entityDir); + const entityRootDir = path.posix.join(this.bucketRootPath, entityDir); this.storageClient .bucket(this.bucketName) From ade04904d5f66278e85095692b32baf8b03b4a38 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 4 Nov 2021 13:37:34 +0530 Subject: [PATCH 33/43] fix api reports Signed-off-by: Himanshu Mishra --- packages/theme/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/theme/api-report.md b/packages/theme/api-report.md index 30dcb7b040..2dc6ab7b55 100644 --- a/packages/theme/api-report.md +++ b/packages/theme/api-report.md @@ -64,7 +64,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; - warning: string; + warning?: string; }; }; From bd93a7811b8b5511488f50a0d782005366f055b1 Mon Sep 17 00:00:00 2001 From: Lykke Axlin Date: Thu, 4 Nov 2021 16:48:30 +0100 Subject: [PATCH 34/43] changed @date-io/luxon from 2.x to 1.0 to be compatible with material-ui-pickers Signed-off-by: Lykke Axlin --- plugins/ilert/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ilert/package.json b/plugins/ilert/package.json index f33e34beed..f93a5b4870 100644 --- a/plugins/ilert/package.json +++ b/plugins/ilert/package.json @@ -27,7 +27,7 @@ "@backstage/errors": "^0.1.3", "@backstage/plugin-catalog-react": "^0.6.1", "@backstage/theme": "^0.2.12", - "@date-io/luxon": "2.x", + "@date-io/luxon": "1.x", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", From fa325fd89dcbc40f610a70e946107835df073b15 Mon Sep 17 00:00:00 2001 From: Lykke Axlin Date: Thu, 4 Nov 2021 17:06:08 +0100 Subject: [PATCH 35/43] added changeset Signed-off-by: Lykke Axlin --- .changeset/light-knives-camp.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/light-knives-camp.md diff --git a/.changeset/light-knives-camp.md b/.changeset/light-knives-camp.md new file mode 100644 index 0000000000..a68f3ebb9a --- /dev/null +++ b/.changeset/light-knives-camp.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-ilert': patch +--- + +Change the version of @date-io/luxon from 2.x to 1.x to make it compatible with material-ui-pickers From 98b8ef555fbeb8d41141360307b21ef7546d6092 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 11:03:35 +1100 Subject: [PATCH 36/43] Re-ran prettier on all files Signed-off-by: Matto --- ADOPTERS.md | 128 ++++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index a2015086c3..797d178270 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,64 +1,64 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | -| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | -| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation.| -| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc.| +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | +| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | +| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | From e37a84085624703018b48746fad5337883317a97 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 11:17:01 +1100 Subject: [PATCH 37/43] Re-ran prettier on all files Signed-off-by: Matto --- ADOPTERS.md | 128 +++++++++++++------------- packages/backend-common/src/config.ts | 2 +- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index a2015086c3..797d178270 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,64 +1,64 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | -| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | -| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation.| -| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc.| +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems🚀. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | +| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | +| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 3c5a2d7b26..3fd5e1b44e 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -22,7 +22,7 @@ import { loadConfigSchema, loadConfig, ConfigSchema, - ConfigTarget + ConfigTarget, } from '@backstage/config-loader'; import { AppConfig, Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; From 89e9f486f1f925684741d0f81391f73c5c89977e Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 12:45:14 +1100 Subject: [PATCH 38/43] api-report fix Signed-off-by: Matto --- packages/config-loader/api-report.md | 30 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index be4678aa0f..6f568df942 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -51,6 +51,21 @@ export type LoadConfigOptions = { watch?: LoadConfigOptionsWatch; }; +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsRemote = { + reloadIntervalSeconds: number; +}; + +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsWatch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; + // @public export function loadConfigSchema( options: LoadConfigSchemaOptions, @@ -74,13 +89,6 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type LoadConfigOptionsRemote = { - reloadIntervalSeconds: number; -}; - // @public export type TransformFunc = ( value: T, @@ -89,11 +97,7 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warnings were encountered during analysis: // -// @public (undocumented) -export type LoadConfigOptionsWatch = { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; -}; +// src/loader.d.ts:33:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/config-loader" does not have an export "configTargets" ``` From 42e97f470890daa4cefda0fc239a9c3084d0f1b9 Mon Sep 17 00:00:00 2001 From: Lykke Axlin Date: Fri, 5 Nov 2021 08:09:36 +0100 Subject: [PATCH 39/43] update yarn.lock and fixed typo Signed-off-by: Lykke Axlin --- .changeset/light-knives-camp.md | 2 +- yarn.lock | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.changeset/light-knives-camp.md b/.changeset/light-knives-camp.md index a68f3ebb9a..1e7fa37082 100644 --- a/.changeset/light-knives-camp.md +++ b/.changeset/light-knives-camp.md @@ -2,4 +2,4 @@ '@backstage/plugin-ilert': patch --- -Change the version of @date-io/luxon from 2.x to 1.x to make it compatible with material-ui-pickers +Change the version of `@date-io/luxon` from 2.x to 1.x to make it compatible with material-ui-pickers diff --git a/yarn.lock b/yarn.lock index 3fec636e32..67e23eb89d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2684,11 +2684,6 @@ resolved "https://registry.npmjs.org/@date-io/core/-/core-2.10.7.tgz#0fe1fa0ef02c827919e23c2802a4b25589ac522d" integrity sha512-EG/1qDiQvd12RoNJ6H+sZcHVswC/3uMx/ySvfaJ24vB30rLjkgHggEXbgMbfgki7wMuiQ/zXI8QlmF1k3kWRGQ== -"@date-io/core@^2.10.11": - version "2.10.11" - resolved "https://registry.npmjs.org/@date-io/core/-/core-2.10.11.tgz#b1a3d57730f3eaaab54d5658be4a71727297e357" - integrity sha512-keXQnwH0LM8wyvu+j5Z2KGK56D+eItjy7DnwuWl/oV+DM2UEYl0z5WhdPMpfswSyt/kjuPOzcVF/7u/skMLaoA== - "@date-io/date-fns@^1.3.13": version "1.3.13" resolved "https://registry.npmjs.org/@date-io/date-fns/-/date-fns-1.3.13.tgz#7798844041640ab393f7e21a7769a65d672f4735" @@ -2696,12 +2691,12 @@ dependencies: "@date-io/core" "^1.3.13" -"@date-io/luxon@2.x": - version "2.10.11" - resolved "https://registry.npmjs.org/@date-io/luxon/-/luxon-2.10.11.tgz#d0981b9fdf5e5f17f8ce59265a3ac6c335565fac" - integrity sha512-SS6SIkp0Y9GFwpQycCTUAyW3OZTW05CWI1DJu10hUzcg8SmjJfhjs7hQY3TOeW+JT6VtXGTVGwbWPUBJsNkhZg== +"@date-io/luxon@1.x": + version "1.3.13" + resolved "https://registry.npmjs.org/@date-io/luxon/-/luxon-1.3.13.tgz#68f0134bb38ef486b2ed6df01981f814c633e28a" + integrity sha512-9wUrJCNSMZJeYAiH+dbb45oGpnHeFP7TOH/Lt26If47gjFCkjvyINzWx+K5AGsnlP0Qosxc7hkF1yLi6ecutxw== dependencies: - "@date-io/core" "^2.10.11" + "@date-io/core" "^1.3.13" "@discoveryjs/json-ext@^0.5.3": version "0.5.5" From f3c7eec64be17a50c38d7f13270773e14fd50386 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 5 Nov 2021 14:48:44 -0500 Subject: [PATCH 40/43] Updated changeset to reflect changes made Signed-off-by: Andre Wanlin --- .changeset/techdocs-tender-bugs-sort.md | 5 +++++ .changeset/tender-bugs-sort.md | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .changeset/techdocs-tender-bugs-sort.md delete mode 100644 .changeset/tender-bugs-sort.md diff --git a/.changeset/techdocs-tender-bugs-sort.md b/.changeset/techdocs-tender-bugs-sort.md new file mode 100644 index 0000000000..be670b06ff --- /dev/null +++ b/.changeset/techdocs-tender-bugs-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Updated to properly join URL segments under any OS for both AWS S3 and GCP diff --git a/.changeset/tender-bugs-sort.md b/.changeset/tender-bugs-sort.md deleted file mode 100644 index b6ad29ba66..0000000000 --- a/.changeset/tender-bugs-sort.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/techdocs-common': patch ---- - -Updated to properly join URL segments under any OS From ec64d9590cbb4dad0f8832da7f41e7a02e8115bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sat, 6 Nov 2021 10:25:17 +0100 Subject: [PATCH 41/43] Call the super constructor early MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/nice-tools-cheat.md | 5 +++++ packages/cli/src/lib/errors.ts | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .changeset/nice-tools-cheat.md diff --git a/.changeset/nice-tools-cheat.md b/.changeset/nice-tools-cheat.md new file mode 100644 index 0000000000..c4f0e162bf --- /dev/null +++ b/.changeset/nice-tools-cheat.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Make `ExitCodeError` call `super` early to avoid compiler warnings diff --git a/packages/cli/src/lib/errors.ts b/packages/cli/src/lib/errors.ts index c0b4c45b6c..dd3955a10a 100644 --- a/packages/cli/src/lib/errors.ts +++ b/packages/cli/src/lib/errors.ts @@ -26,11 +26,11 @@ export class ExitCodeError extends CustomError { readonly code: number; constructor(code: number, command?: string) { - if (command) { - super(`Command '${command}' exited with code ${code}`); - } else { - super(`Child exited with code ${code}`); - } + super( + command + ? `Command '${command}' exited with code ${code}` + : `Child exited with code ${code}`, + ); this.code = code; } } From e7e4626fc9308f63375cd8b07b9196fd57d14079 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 8 Nov 2021 10:38:42 +0100 Subject: [PATCH 42/43] chore: remove warning Signed-off-by: blam --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 13b7c945cf..1078631926 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ # [Backstage](https://backstage.io) -> 🏖 All of the maintainers will be taking a wellness break Nov. 1–5. The repo and Discord may be quieter than usual, but not to worry. We’ll have coverage plans in place and be back in full force, rested and restored, on Nov. 8. 🏖 - [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) From 44308cc80daeafd9ae5debc148d724f1e06cbeae Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 8 Nov 2021 11:40:31 +0100 Subject: [PATCH 43/43] chore: remove some duplicate dependencies Signed-off-by: blam --- yarn.lock | 119 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 93 insertions(+), 26 deletions(-) diff --git a/yarn.lock b/yarn.lock index e879cff365..905d2e7bef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5449,18 +5449,18 @@ integrity sha512-dOI13j1uHMZkRxhZuge/ugOE7Aqcg7Nxki932lDZuXyY4G8CGxkc/66PeQ8pR4PCzThHORXo7Ptvau6bh101lQ== "@storybook/addon-a11y@^6.3.4": - version "6.3.7" - resolved "https://registry.npmjs.org/@storybook/addon-a11y/-/addon-a11y-6.3.7.tgz#a802455f2d932eda07314e3d44a96c94bbd22b3d" - integrity sha512-Z5Lhxm8r5CkPW9FYf6zmAk9c7IhUeUQZxKZeEWGZdOvcjQ32rtg4IYvO2SHgWNrEKBdxxFm3pMiyK3wylQLfsQ== + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/addon-a11y/-/addon-a11y-6.3.12.tgz#2f930fc84fc275a4ed43a716fc09cc12caf4e110" + integrity sha512-q1NdRHFJV6sLEEJw0hatCc5ZIthELqM/AWdrEWDyhcJNyiq7Tq4nKqQBMTQSYwHiUAmxVgw7i4oa1vM2M51/3g== dependencies: - "@storybook/addons" "6.3.7" - "@storybook/api" "6.3.7" - "@storybook/channels" "6.3.7" - "@storybook/client-api" "6.3.7" - "@storybook/client-logger" "6.3.7" - "@storybook/components" "6.3.7" - "@storybook/core-events" "6.3.7" - "@storybook/theming" "6.3.7" + "@storybook/addons" "6.3.12" + "@storybook/api" "6.3.12" + "@storybook/channels" "6.3.12" + "@storybook/client-api" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/components" "6.3.12" + "@storybook/core-events" "6.3.12" + "@storybook/theming" "6.3.12" axe-core "^4.2.0" core-js "^3.8.2" global "^4.4.0" @@ -5546,6 +5546,21 @@ global "^4.4.0" regenerator-runtime "^0.13.7" +"@storybook/addons@6.3.12", "@storybook/addons@^6.1.11": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.3.12.tgz#8773dcc113c5086dfff722388b7b65580e43b65b" + integrity sha512-UgoMyr7Qr0FS3ezt8u6hMEcHgyynQS9ucr5mAwZky3wpXRPFyUTmMto9r4BBUdqyUvTUj/LRKIcmLBfj+/l0Fg== + dependencies: + "@storybook/api" "6.3.12" + "@storybook/channels" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/core-events" "6.3.12" + "@storybook/router" "6.3.12" + "@storybook/theming" "6.3.12" + core-js "^3.8.2" + global "^4.4.0" + regenerator-runtime "^0.13.7" + "@storybook/addons@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.3.7.tgz#7c6b8d11b65f67b1884f6140437fe996dc39537a" @@ -5561,21 +5576,6 @@ global "^4.4.0" regenerator-runtime "^0.13.7" -"@storybook/addons@^6.1.11": - version "6.3.12" - resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.3.12.tgz#8773dcc113c5086dfff722388b7b65580e43b65b" - integrity sha512-UgoMyr7Qr0FS3ezt8u6hMEcHgyynQS9ucr5mAwZky3wpXRPFyUTmMto9r4BBUdqyUvTUj/LRKIcmLBfj+/l0Fg== - dependencies: - "@storybook/api" "6.3.12" - "@storybook/channels" "6.3.12" - "@storybook/client-logger" "6.3.12" - "@storybook/core-events" "6.3.12" - "@storybook/router" "6.3.12" - "@storybook/theming" "6.3.12" - core-js "^3.8.2" - global "^4.4.0" - regenerator-runtime "^0.13.7" - "@storybook/api@6.3.11": version "6.3.11" resolved "https://registry.npmjs.org/@storybook/api/-/api-6.3.11.tgz#ea3806a0570da65bfb5b39e4edb90289b5ba701e" @@ -5743,6 +5743,19 @@ qs "^6.10.0" telejson "^5.3.2" +"@storybook/channel-postmessage@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/channel-postmessage/-/channel-postmessage-6.3.12.tgz#3ff9412ac0f445e3b8b44dd414e783a5a47ff7c1" + integrity sha512-Ou/2Ga3JRTZ/4sSv7ikMgUgLTeZMsXXWLXuscz4oaYhmOqAU9CrJw0G1NitwBgK/+qC83lEFSLujHkWcoQDOKg== + dependencies: + "@storybook/channels" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/core-events" "6.3.12" + core-js "^3.8.2" + global "^4.4.0" + qs "^6.10.0" + telejson "^5.3.2" + "@storybook/channel-postmessage@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/channel-postmessage/-/channel-postmessage-6.3.7.tgz#bd4edf84a29aa2cd4a22d26115c60194d289a840" @@ -5807,6 +5820,30 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" +"@storybook/client-api@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/client-api/-/client-api-6.3.12.tgz#a0c6d72a871d1cb02b4b98675472839061e39b5b" + integrity sha512-xnW+lKKK2T774z+rOr9Wopt1aYTStfb86PSs9p3Fpnc2Btcftln+C3NtiHZl8Ccqft8Mz/chLGgewRui6tNI8g== + dependencies: + "@storybook/addons" "6.3.12" + "@storybook/channel-postmessage" "6.3.12" + "@storybook/channels" "6.3.12" + "@storybook/client-logger" "6.3.12" + "@storybook/core-events" "6.3.12" + "@storybook/csf" "0.0.1" + "@types/qs" "^6.9.5" + "@types/webpack-env" "^1.16.0" + core-js "^3.8.2" + global "^4.4.0" + lodash "^4.17.20" + memoizerific "^1.11.3" + qs "^6.10.0" + regenerator-runtime "^0.13.7" + stable "^0.1.8" + store2 "^2.12.0" + ts-dedent "^2.0.0" + util-deprecate "^1.0.2" + "@storybook/client-api@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/client-api/-/client-api-6.3.7.tgz#cb1dca05467d777bd09aadbbdd1dd22ca537ce14" @@ -5885,6 +5922,36 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" +"@storybook/components@6.3.12": + version "6.3.12" + resolved "https://registry.npmjs.org/@storybook/components/-/components-6.3.12.tgz#0c7967c60354c84afa20dfab4753105e49b1927d" + integrity sha512-kdQt8toUjynYAxDLrJzuG7YSNL6as1wJoyzNUaCfG06YPhvIAlKo7le9tS2mThVFN5e9nbKrW3N1V1sp6ypZXQ== + dependencies: + "@popperjs/core" "^2.6.0" + "@storybook/client-logger" "6.3.12" + "@storybook/csf" "0.0.1" + "@storybook/theming" "6.3.12" + "@types/color-convert" "^2.0.0" + "@types/overlayscrollbars" "^1.12.0" + "@types/react-syntax-highlighter" "11.0.5" + color-convert "^2.0.1" + core-js "^3.8.2" + fast-deep-equal "^3.1.3" + global "^4.4.0" + lodash "^4.17.20" + markdown-to-jsx "^7.1.3" + memoizerific "^1.11.3" + overlayscrollbars "^1.13.1" + polished "^4.0.5" + prop-types "^15.7.2" + react-colorful "^5.1.2" + react-popper-tooltip "^3.1.1" + react-syntax-highlighter "^13.5.3" + react-textarea-autosize "^8.3.0" + regenerator-runtime "^0.13.7" + ts-dedent "^2.0.0" + util-deprecate "^1.0.2" + "@storybook/components@6.3.7": version "6.3.7" resolved "https://registry.npmjs.org/@storybook/components/-/components-6.3.7.tgz#42b1ca6d24e388e02eab82aa9ed3365db2266ecc"