From 0611f3b3e2d5e4e742abbf17d6e17c1bd0c95541 Mon Sep 17 00:00:00 2001 From: Matto Date: Mon, 27 Sep 2021 23:41:24 +1000 Subject: [PATCH 01/17] Read config from remote config server Signed-off-by: Matto --- .changeset/giant-years-help.md | 8 + packages/backend-common/src/config.ts | 10 +- packages/cli/package.json | 3 +- packages/cli/src/lib/config.ts | 16 +- packages/config-loader/api-report.md | 34 +++- packages/config-loader/package.json | 10 +- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/loader.test.ts | 125 +++++++++++- packages/config-loader/src/loader.ts | 232 +++++++++++++++++++--- packages/integration/api-report.md | 5 + packages/integration/src/index.ts | 1 + yarn.lock | 69 ++++++- 12 files changed, 461 insertions(+), 54 deletions(-) create mode 100644 .changeset/giant-years-help.md diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md new file mode 100644 index 0000000000..f0ca12ef19 --- /dev/null +++ b/.changeset/giant-years-help.md @@ -0,0 +1,8 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/integration': patch +--- + +Reading app config from a remote server diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 96907870cb..afdb4fd0f7 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -19,7 +19,8 @@ import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader, JsonValue } from '@backstage/config'; -import { loadConfig } from '@backstage/config-loader'; +import { ConfigTarget, loadConfig } from '@backstage/config-loader'; +import { isValidUrl } from '@backstage/integration'; class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -117,7 +118,10 @@ export async function loadBackendConfig(options: { argv: string[]; }): Promise { const args = parseArgs(options.argv); - const configPaths: string[] = [args.config ?? []].flat(); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); const config = new ObservableConfigProxy(options.logger); @@ -126,7 +130,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configPaths.map(opt => resolvePath(opt)), + configTargets: configTargets, watch: { onChange(newConfigs) { options.logger.info( diff --git a/packages/cli/package.json b/packages/cli/package.json index 0dc4dc3cf7..47475780a6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,8 +31,10 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.3", + "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", + "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -117,7 +119,6 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.9.4", "@backstage/config": "^0.1.10", "@backstage/core-components": "^0.5.0", "@backstage/core-plugin-api": "^0.1.8", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index a93f040122..218771beb6 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; +import { + ConfigTarget, + loadConfig, + loadConfigSchema, +} from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; +import { isValidUrl } from '@backstage/integration'; type Options = { args: string[]; @@ -26,7 +31,12 @@ type Options = { }; export async function loadCliConfig(options: Options) { - const configPaths = options.args.map(arg => paths.resolveTarget(arg)); + const configTargets: ConfigTarget[] = []; + options.args.forEach(arg => { + if (!isValidUrl(arg)) { + configTargets.push({ path: paths.resolveTarget(arg) }); + } + }); // Consider all packages in the monorepo when loading in config const { Project } = require('@lerna/project'); @@ -46,7 +56,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, - configPaths, + configTargets: configTargets, }); // printing to stderr to not clobber stdout in case the cli command diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index bb3ae50d23..bb36bbe750 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = { withFilteredKeys?: boolean; }; +// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ConfigTarget = + | { + path: string; + } + | { + url: string; + }; + // @public export type ConfigVisibility = 'frontend' | 'backend' | 'secret'; @@ -35,13 +46,11 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public (undocumented) export type LoadConfigOptions = { configRoot: string; - configPaths: string[]; + configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; - watch?: { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; - }; + remote?: Remote; + watch?: Watch; }; // @public @@ -66,6 +75,13 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; +// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Remote = { + reloadIntervalSeconds: number; +}; + // @public export type TransformFunc = ( value: T, @@ -73,4 +89,12 @@ export type TransformFunc = ( visibility: ConfigVisibility; }, ) => T | undefined; + +// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Watch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; ``` diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 2f2878799e..d20c77bedc 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,8 +30,10 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", + "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", @@ -39,8 +41,10 @@ "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", "typescript-json-schema": "^0.50.1", + "uuid": "^8.3.2", "yaml": "^1.9.2", - "yup": "^0.29.3" + "yup": "^0.29.3", + "node-fetch": "2.6.5" }, "devDependencies": { "@types/jest": "^26.0.7", @@ -48,7 +52,9 @@ "@types/mock-fs": "^4.10.0", "@types/node": "^14.14.32", "@types/yup": "^0.29.8", - "mock-fs": "^5.1.0" + "mock-fs": "^5.1.0", + "fetch-mock-jest": "1.5.1", + "fetch-mock": "^9.11.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 0e66c59ec4..e0eafc00bc 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -30,4 +30,4 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './loader'; +export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3f51e76936..190904a870 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,12 +18,33 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; +import { v4 as uuidv4 } from 'uuid'; + +const fetchMock = require('fetch-mock').sandbox(); +const nodeFetch = require('node-fetch'); + +nodeFetch.default = fetchMock; describe('loadConfig', () => { beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + method: 'GET', + }, + { + body: `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + ); + mockFs({ '/root/app-config.yaml': ` app: @@ -66,6 +87,7 @@ describe('loadConfig', () => { }); afterEach(() => { + fetchMock.restore(); mockFs.restore(); }); @@ -73,7 +95,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], env: 'production', }), ).resolves.toEqual([ @@ -90,11 +112,37 @@ describe('loadConfig', () => { ]); }); + it('load config from remote path', async () => { + const configUrl = 'https://some.domain.io/app-config.yaml'; + + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + env: 'production', + remote: { + reloadIntervalSeconds: 30, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), ).resolves.toEqual([ @@ -115,9 +163,9 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', + configTargets: [ + { path: '/root/app-config.yaml' }, + { path: '/root/app-config.development.yaml' }, ], env: 'development', }), @@ -155,7 +203,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.substitute.yaml'], + configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), ).resolves.toEqual([ @@ -180,7 +228,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: onChange.resolve, stopSignal: stopSignal.promise, @@ -218,12 +266,73 @@ describe('loadConfig', () => { stopSignal.resolve(); }); + it('watches remote config urls', async () => { + const onChange = defer(); + const stopSignal = defer(); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + remote: { + reloadIntervalSeconds: 1, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + }, + { + body: `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + { overwriteRoutes: true }, + ); + + await expect(onChange.promise).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'NEW ReMOTe ExaMPLe App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + it('stops watching config files', async () => { const stopSignal = defer(); await loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: () => { expect('not').toBe('called'); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 315b44c4ee..fb8d0aab33 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,23 +17,67 @@ import fs from 'fs-extra'; import yaml from 'yaml'; import chokidar from 'chokidar'; -import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; import { AppConfig } from '@backstage/config'; import { applyConfigTransforms, - readEnvConfig, createIncludeTransform, createSubstitutionTransform, + EnvFunc, + readEnvConfig, } from './lib'; -import { EnvFunc } from './lib/transform/types'; +import fetch from 'node-fetch'; +import { isValidUrl } from '@backstage/integration'; +export type ConfigTarget = { path: string } | { url: string }; + +export type Watch = { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; +}; + +export type Remote = { + /** + * An optional remote config reloading period, in seconds + */ + reloadIntervalSeconds: number; +}; + +export type RemoteConfigProp = { + /** + * URL of the remote config + */ + url: string; + + /** + * Contents of the remote config + */ + content: string | null; + + /** + * An optional new ETag header value. Used when checking for updated config. + */ + newETag?: string; + + /** + * An optional old ETag header value. Used when checking for updated config + */ + oldETag?: string; +}; /** @public */ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; - // Absolute paths to load config files from. Configs from earlier paths have lower priority. - configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. + configTargets: ConfigTarget[]; /** @deprecated This option has been removed */ env?: string; @@ -45,22 +89,19 @@ export type LoadConfigOptions = { */ experimentalEnvFunc?: EnvFunc; + /** + * An optional remote config + */ + remote?: Remote; + /** * An optional configuration that enables watching of config files. */ - watch?: { - /** - * A listener that is called when a config file is changed. - */ - onChange: (configs: AppConfig[]) => void; - - /** - * An optional signal that stops the watcher once the promise resolves. - */ - stopSignal?: Promise; - }; + watch?: Watch; }; +const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; + /** * Load configuration data. * @@ -69,12 +110,30 @@ export type LoadConfigOptions = { export async function loadConfig( options: LoadConfigOptions, ): Promise { - const { configRoot, experimentalEnvFunc: envFunc, watch } = options; - const configPaths = options.configPaths.slice(); + const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options; + + const configPaths: string[] = options.configTargets + .slice() + .filter((e): e is { path: string } => e.hasOwnProperty('path')) + .map(configTarget => configTarget.path); + + let configUrls: string[] = options.configTargets + .slice() + .filter((e): e is { url: string } => e.hasOwnProperty('url')) + .map(configTarget => configTarget.url); + + const remoteConfigProps: RemoteConfigProp[] = []; + + if (remote === undefined && configUrls.length > 0) { + console.warn( + `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, + ); + configUrls = []; + } // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` - if (configPaths.length === 0) { + if (configPaths.length === 0 && configUrls.length === 0) { configPaths.push(resolvePath(configRoot, 'app-config.yaml')); const localConfig = resolvePath(configRoot, 'app-config.local.yaml'); @@ -99,6 +158,7 @@ export async function loadConfig( const input = yaml.parse(await readFile(configPath)); const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ createIncludeTransform(env, readFile, substitutionTransform), substitutionTransform, @@ -110,7 +170,56 @@ export async function loadConfig( return configs; }; - let fileConfigs; + const loadRemoteConfigFiles = async () => { + const configs: AppConfig[] = []; + + const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { + const response = await fetch(remoteConfigProp.url); + if (!response.ok) { + throw new Error( + `Could not read config file at ${remoteConfigProp.url}`, + ); + } + + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + remoteConfigProp.content = await response.text(); + + return remoteConfigProp; + }; + + for (let i = 0; i < configUrls.length; i++) { + const remoteConfigProp = await readConfigFromUrl({ + url: configUrls[i], + content: null, + }); + + if (!isValidUrl(remoteConfigProp.url)) { + throw new Error( + `Config load path is not valid: '${remoteConfigProp.url}'`, + ); + } + + const dir = configRoot; + if (!remoteConfigProp.content) { + throw new Error(`Config is not valid`); + } + const input = yaml.parse(remoteConfigProp.content); + const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ + substitutionTransform, + ]); + + configs.push({ data, context: remoteConfigProp.url }); + + remoteConfigProps.push(remoteConfigProp); + } + + return configs; + }; + + let fileConfigs: AppConfig[]; try { fileConfigs = await loadConfigFiles(); } catch (error) { @@ -119,15 +228,23 @@ export async function loadConfig( ); } + let remoteConfigs: AppConfig[] = []; + if (remote) { + try { + remoteConfigs = await loadRemoteConfigFiles(); + } catch (error) { + throw new Error(`Failed to read remote configuration file, ${error}`); + } + } + const envConfigs = await readEnvConfig(process.env); - // Set up config file watching if requested by the caller - if (watch) { - let currentSerializedConfig = JSON.stringify(fileConfigs); - + const watchConfigFile = (watchProp: Watch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); + + let currentSerializedConfig = JSON.stringify(fileConfigs); watcher.on('change', async () => { try { const newConfigs = await loadConfigFiles(); @@ -138,18 +255,77 @@ export async function loadConfig( } currentSerializedConfig = newSerializedConfig; - watch.onChange([...newConfigs, ...envConfigs]); + watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]); } catch (error) { console.error(`Failed to reload configuration files, ${error}`); } }); - if (watch.stopSignal) { - watch.stopSignal.then(() => { + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { watcher.close(); }); } + }; + + const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { + const requestProps = { method: 'HEAD' }; + const { headers } = await fetch(remoteConfigProp.url, requestProps); + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + + return ( + remoteConfigProp.oldETag !== undefined && + remoteConfigProp.newETag !== undefined && + remoteConfigProp.oldETag !== remoteConfigProp.newETag + ); + }; + + let handle: NodeJS.Timeout | undefined; + try { + handle = setInterval(async () => { + console.info(`Checking for config update`); + for (const remoteConfigProp of remoteConfigProps) { + if (await hasConfigChanged(remoteConfigProp)) { + console.info(`Remote config change, reloading config ...`); + const newRemoteConfigs = await loadRemoteConfigFiles(); + watchProp.onChange([ + ...newRemoteConfigs, + ...fileConfigs, + ...envConfigs, + ]); + console.info(`Remote config reloaded`); + break; + } + } + }, remoteProp.reloadIntervalSeconds * 1000); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { + if (handle !== undefined) { + console.info(`Stopping remote config watch`); + clearInterval(handle); + handle = undefined; + } + }); + } + }; + + // Set up config file watching if requested by the caller + if (watch) { + watchConfigFile(watch); } - return [...fileConfigs, ...envConfigs]; + if (watch && remote) { + watchRemoteConfig(watch, remote); + } + + return remote + ? [...remoteConfigs, ...fileConfigs, ...envConfigs] + : [...fileConfigs, ...envConfigs]; } diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 01f84383fe..e07f61eea6 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,6 +347,11 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function isValidUrl(url: string): boolean; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 3d5f336e8d..b8992f52d2 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,6 +27,7 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; +export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; diff --git a/yarn.lock b/yarn.lock index 7657498ff7..87991e1bb5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11336,6 +11336,11 @@ core-js@^2.5.0: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== +core-js@^3.0.0: + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== + core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14037,6 +14042,29 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== +fetch-mock-jest@1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" + integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== + dependencies: + fetch-mock "^9.11.0" + +fetch-mock@^9.11.0: + version "9.11.0" + resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" + integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== + dependencies: + "@babel/core" "^7.0.0" + "@babel/runtime" "^7.0.0" + core-js "^3.0.0" + debug "^4.1.1" + glob-to-regexp "^0.4.0" + is-subset "^0.1.1" + lodash.isequal "^4.5.0" + path-to-regexp "^2.2.1" + querystring "^0.2.0" + whatwg-url "^6.5.0" + fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -14867,7 +14895,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -16851,6 +16879,11 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" +is-subset@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" + integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= + is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18719,7 +18752,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0: +lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -20192,6 +20225,13 @@ node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -21526,6 +21566,11 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" +path-to-regexp@^2.2.1: + version "2.4.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -26576,6 +26621,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + "traverse@>=0.3.0 <0.4": version "0.3.9" resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" @@ -27702,6 +27752,11 @@ web-namespaces@^1.0.0: resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz#bc98a3de60dadd7faefc403d1076d529f5e030ec" integrity sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -27954,7 +28009,15 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^6.4.1: +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +whatwg-url@^6.4.1, whatwg-url@^6.5.0: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From befe7d122f5095122740d0878058ad684219c182 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 19 Oct 2021 23:18:19 +1100 Subject: [PATCH 02/17] Added documentation Signed-off-by: Matto --- docs/conf/writing.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4288d23b17..86b844484b 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,7 +1,6 @@ --- -id: writing -title: Writing Backstage Configuration Files -description: Documentation on Writing Backstage Configuration Files +id: writing title: Writing Backstage Configuration Files description: Documentation on Writing +Backstage Configuration Files --- ## File Format @@ -67,13 +66,15 @@ production build. ## Configuration Files -It is possible to have multiple configuration files, both to support different -environments, but also to define configuration that is local to specific -packages. The configuration files to load are selected using a `--config ` -flag, and it is possible to load any number of files. Paths are relative to the -working directory of the executed process, for example `package/backend`. This -means that to select a config file in the repo root when running the backend, -you would use `--config ../../my-config.yaml`. +It is possible to have multiple configuration files (bundled and/or remote), +both to support different environments, but also to define configuration that is +local to specific packages. The configuration files to load are selected using a +`--config ` flag, and it is possible to load any number of +files. Paths are relative to the working directory of the executed process, for +example `package/backend`. This means that to select a config file in the repo +root when running the backend, you would use `--config ../../my-config.yaml`, +and for config file on a config server you would use +`--config https://some.domain.io/app-config.yaml` If no `config` flags are specified, the default behavior is to load `app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root. @@ -85,7 +86,7 @@ are NOT loaded. To include them you need to explicitly include them with a flag, for example: ```shell -yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml +yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml --config https://some.domain.io/app-config.yaml ``` All loaded configuration files are merged together using the following rules: From 74ceaa64175d5eecd33577a735ce447081b7f0bc Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:20:36 +1100 Subject: [PATCH 03/17] Remove the dependency on Etag headers for reloading config Signed-off-by: Matto --- docs/conf/writing.md | 5 +- packages/cli/package.json | 2 +- packages/config-loader/package.json | 4 +- packages/config-loader/src/loader.test.ts | 79 +++++++++-------- packages/config-loader/src/loader.ts | 102 ++++++---------------- yarn.lock | 7 -- 6 files changed, 73 insertions(+), 126 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 86b844484b..a0e29820ed 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,6 +1,7 @@ --- -id: writing title: Writing Backstage Configuration Files description: Documentation on Writing -Backstage Configuration Files +id: writing +title: Writing Backstage Configuration Files +description: Documentation on Writing Backstage Configuration Files --- ## File Format diff --git a/packages/cli/package.json b/packages/cli/package.json index 47475780a6..519b45a5ad 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -34,7 +34,6 @@ "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", - "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -119,6 +118,7 @@ "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-common": "^0.9.4", "@backstage/config": "^0.1.10", "@backstage/core-components": "^0.5.0", "@backstage/core-plugin-api": "^0.1.8", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index d20c77bedc..80e4121971 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -41,7 +41,6 @@ "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", "typescript-json-schema": "^0.50.1", - "uuid": "^8.3.2", "yaml": "^1.9.2", "yup": "^0.29.3", "node-fetch": "2.6.5" @@ -53,8 +52,7 @@ "@types/node": "^14.14.32", "@types/yup": "^0.29.8", "mock-fs": "^5.1.0", - "fetch-mock-jest": "1.5.1", - "fetch-mock": "^9.11.0" + "msw": "^0.29.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 190904a870..e8b5a8828b 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,33 +18,47 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; -import { v4 as uuidv4 } from 'uuid'; - -const fetchMock = require('fetch-mock').sandbox(); -const nodeFetch = require('node-fetch'); - -nodeFetch.default = fetchMock; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; describe('loadConfig', () => { + const server = setupServer(); + const initialLoaderHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + const reloadHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + beforeAll(() => server.listen()); + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - method: 'GET', - }, - { - body: `app: - title: Remote Example App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - ); - mockFs({ '/root/app-config.yaml': ` app: @@ -87,10 +101,12 @@ describe('loadConfig', () => { }); afterEach(() => { - fetchMock.restore(); mockFs.restore(); + server.resetHandlers(); }); + afterAll(() => server.close()); + it('load config from default path', async () => { await expect( loadConfig({ @@ -113,6 +129,8 @@ describe('loadConfig', () => { }); it('load config from remote path', async () => { + server.use(initialLoaderHandler); + const configUrl = 'https://some.domain.io/app-config.yaml'; await expect( @@ -267,6 +285,8 @@ describe('loadConfig', () => { }); it('watches remote config urls', async () => { + server.use(initialLoaderHandler); + const onChange = defer(); const stopSignal = defer(); @@ -296,20 +316,7 @@ describe('loadConfig', () => { }, ]); - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - }, - { - body: `app: - title: NEW ReMOTe ExaMPLe App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - { overwriteRoutes: true }, - ); + server.use(reloadHandler); await expect(onChange.promise).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index fb8d0aab33..b853d30d8a 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -50,27 +50,6 @@ export type Remote = { reloadIntervalSeconds: number; }; -export type RemoteConfigProp = { - /** - * URL of the remote config - */ - url: string; - - /** - * Contents of the remote config - */ - content: string | null; - - /** - * An optional new ETag header value. Used when checking for updated config. - */ - newETag?: string; - - /** - * An optional old ETag header value. Used when checking for updated config - */ - oldETag?: string; -}; /** @public */ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -100,8 +79,6 @@ export type LoadConfigOptions = { watch?: Watch; }; -const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; - /** * Load configuration data. * @@ -117,18 +94,15 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); - let configUrls: string[] = options.configTargets + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) .map(configTarget => configTarget.url); - const remoteConfigProps: RemoteConfigProp[] = []; - if (remote === undefined && configUrls.length > 0) { - console.warn( + throw new Error( `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, ); - configUrls = []; } // If no paths are provided, we default to reading @@ -173,47 +147,32 @@ export async function loadConfig( const loadRemoteConfigFiles = async () => { const configs: AppConfig[] = []; - const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { - const response = await fetch(remoteConfigProp.url); + const readConfigFromUrl = async (url: string) => { + const response = await fetch(url); if (!response.ok) { - throw new Error( - `Could not read config file at ${remoteConfigProp.url}`, - ); + throw new Error(`Could not read config file at ${url}`); } - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - remoteConfigProp.content = await response.text(); - - return remoteConfigProp; + return await response.text(); }; for (let i = 0; i < configUrls.length; i++) { - const remoteConfigProp = await readConfigFromUrl({ - url: configUrls[i], - content: null, - }); - - if (!isValidUrl(remoteConfigProp.url)) { - throw new Error( - `Config load path is not valid: '${remoteConfigProp.url}'`, - ); + const configUrl = configUrls[i]; + if (!isValidUrl(configUrl)) { + throw new Error(`Config load path is not valid: '${configUrl}'`); } - const dir = configRoot; - if (!remoteConfigProp.content) { + const remoteConfigContent = await readConfigFromUrl(configUrl); + if (!remoteConfigContent) { throw new Error(`Config is not valid`); } - const input = yaml.parse(remoteConfigProp.content); + const configYaml = yaml.parse(remoteConfigContent); const substitutionTransform = createSubstitutionTransform(env); - const data = await applyConfigTransforms(dir, input, [ + const data = await applyConfigTransforms(configRoot, configYaml, [ substitutionTransform, ]); - configs.push({ data, context: remoteConfigProp.url }); - - remoteConfigProps.push(remoteConfigProp); + configs.push({ data, context: configUrl }); } return configs; @@ -269,17 +228,12 @@ export async function loadConfig( }; const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { - const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { - const requestProps = { method: 'HEAD' }; - const { headers } = await fetch(remoteConfigProp.url, requestProps); - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - + const hasConfigChanged = async ( + oldRemoteConfigs: AppConfig[], + newRemoteConfigs: AppConfig[], + ) => { return ( - remoteConfigProp.oldETag !== undefined && - remoteConfigProp.newETag !== undefined && - remoteConfigProp.oldETag !== remoteConfigProp.newETag + JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs) ); }; @@ -287,18 +241,12 @@ export async function loadConfig( try { handle = setInterval(async () => { console.info(`Checking for config update`); - for (const remoteConfigProp of remoteConfigProps) { - if (await hasConfigChanged(remoteConfigProp)) { - console.info(`Remote config change, reloading config ...`); - const newRemoteConfigs = await loadRemoteConfigFiles(); - watchProp.onChange([ - ...newRemoteConfigs, - ...fileConfigs, - ...envConfigs, - ]); - console.info(`Remote config reloaded`); - break; - } + const newRemoteConfigs = await loadRemoteConfigFiles(); + if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) { + remoteConfigs = newRemoteConfigs; + console.info(`Remote config change, reloading config ...`); + watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]); + console.info(`Remote config reloaded`); } }, remoteProp.reloadIntervalSeconds * 1000); } catch (error) { diff --git a/yarn.lock b/yarn.lock index 87991e1bb5..00c91f7783 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14042,13 +14042,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock-jest@1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" - integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== - dependencies: - fetch-mock "^9.11.0" - fetch-mock@^9.11.0: version "9.11.0" resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" From 988c5b8421a82a733cebafc7afcaf01683149a5c Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:26:08 +1100 Subject: [PATCH 04/17] Removed unused dependency Signed-off-by: Matto --- packages/config-loader/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 80e4121971..4b5afb3d29 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -33,7 +33,6 @@ "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", - "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", From 855d460611216a54d045df5b269c65f8641e94f9 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 26 Oct 2021 21:50:49 +1100 Subject: [PATCH 05/17] Change variable name Signed-off-by: Matto --- packages/config-loader/api-report.md | 12 ++++++------ packages/config-loader/src/index.ts | 7 ++++++- packages/config-loader/src/loader.ts | 15 +++++++++------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index bb36bbe750..1dad108c38 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -49,8 +49,8 @@ export type LoadConfigOptions = { configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; - remote?: Remote; - watch?: Watch; + remote?: LoadConfigOptionsRemote; + watch?: LoadConfigOptionsWatch; }; // @public @@ -75,10 +75,10 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Remote = { +export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; @@ -90,10 +90,10 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Watch = { +export type LoadConfigOptionsWatch = { onChange: (configs: AppConfig[]) => void; stopSignal?: Promise; }; diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index e0eafc00bc..c6b8ef8cef 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -30,4 +30,9 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; +export type { + ConfigTarget, + LoadConfigOptions, + LoadConfigOptionsWatch, + LoadConfigOptionsRemote, +} from './loader'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index b853d30d8a..2e7a0df801 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -31,7 +31,7 @@ import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; -export type Watch = { +export type LoadConfigOptionsWatch = { /** * A listener that is called when a config file is changed. */ @@ -43,7 +43,7 @@ export type Watch = { stopSignal?: Promise; }; -export type Remote = { +export type LoadConfigOptionsRemote = { /** * An optional remote config reloading period, in seconds */ @@ -71,12 +71,12 @@ export type LoadConfigOptions = { /** * An optional remote config */ - remote?: Remote; + remote?: LoadConfigOptionsRemote; /** * An optional configuration that enables watching of config files. */ - watch?: Watch; + watch?: LoadConfigOptionsWatch; }; /** @@ -198,7 +198,7 @@ export async function loadConfig( const envConfigs = await readEnvConfig(process.env); - const watchConfigFile = (watchProp: Watch) => { + const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); @@ -227,7 +227,10 @@ export async function loadConfig( } }; - const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const watchRemoteConfig = ( + watchProp: LoadConfigOptionsWatch, + remoteProp: LoadConfigOptionsRemote, + ) => { const hasConfigChanged = async ( oldRemoteConfigs: AppConfig[], newRemoteConfigs: AppConfig[], From a57927f5d5a41b70b2d90f4f889c65a6f22003a1 Mon Sep 17 00:00:00 2001 From: Matto Date: Wed, 27 Oct 2021 20:33:33 +1100 Subject: [PATCH 06/17] Removed dependency, introduced isValidUrl, and reinstated `configPaths` Signed-off-by: Matto --- packages/backend-common/package.json | 1 - packages/backend-common/src/config.ts | 3 +- packages/backend-common/src/urls.test.ts | 34 ++++++++++++++ packages/backend-common/src/urls.ts | 25 +++++++++++ packages/cli/package.json | 1 - packages/cli/src/lib/config.ts | 3 +- packages/cli/src/lib/urls.test.ts | 34 ++++++++++++++ packages/cli/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/api-report.md | 1 + packages/config-loader/package.json | 1 - packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/urls.test.ts | 34 ++++++++++++++ packages/config-loader/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/src/loader.test.ts | 49 ++++++++++++++++++++- packages/config-loader/src/loader.ts | 14 +++++- yarn.lock | 37 ++-------------- 16 files changed, 247 insertions(+), 41 deletions(-) create mode 100644 packages/backend-common/src/urls.test.ts create mode 100644 packages/backend-common/src/urls.ts create mode 100644 packages/cli/src/lib/urls.test.ts create mode 100644 packages/cli/src/lib/urls.ts create mode 100644 packages/config-loader/src/lib/urls.test.ts create mode 100644 packages/config-loader/src/lib/urls.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 67a2c1a853..0c368d7805 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -33,7 +33,6 @@ "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", "@backstage/errors": "^0.1.2", - "@backstage/integration": "^0.6.5", "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index afdb4fd0f7..c93f2cedf9 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,7 @@ import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader, JsonValue } from '@backstage/config'; import { ConfigTarget, loadConfig } from '@backstage/config-loader'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -130,6 +130,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, watch: { onChange(newConfigs) { diff --git a/packages/backend-common/src/urls.test.ts b/packages/backend-common/src/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-common/src/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-common/src/urls.ts b/packages/backend-common/src/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-common/src/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/cli/package.json b/packages/cli/package.json index 519b45a5ad..0dc4dc3cf7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,7 +31,6 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.3", - "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.6.8", "@hot-loader/react-dom": "^16.13.0", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 218771beb6..013328d9f3 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -21,7 +21,7 @@ import { } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; type Options = { args: string[]; @@ -56,6 +56,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, }); diff --git a/packages/cli/src/lib/urls.test.ts b/packages/cli/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/cli/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/cli/src/lib/urls.ts b/packages/cli/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/cli/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 1dad108c38..c8e01f27b0 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -46,6 +46,7 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public (undocumented) export type LoadConfigOptions = { configRoot: string; + configPaths: string[]; configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: EnvFunc; diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 4b5afb3d29..9475058b24 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,7 +30,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.3", "@backstage/config": "^0.1.9", "@types/json-schema": "^7.0.6", diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 32a0191cae..ca88b771ba 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readEnvConfig } from './env'; export * from './transform'; export * from './schema'; +export { isValidUrl } from './urls'; diff --git a/packages/config-loader/src/lib/urls.test.ts b/packages/config-loader/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/config-loader/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/config-loader/src/lib/urls.ts b/packages/config-loader/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/config-loader/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index e8b5a8828b..804254afa0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -67,6 +67,13 @@ describe('loadConfig', () => { $file: secrets/session-key.txt escaped: \$\${Escaped} `, + '/root/app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, '/root/app-config.development.yaml': ` app: sessionKey: development-key @@ -111,6 +118,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], env: 'production', }), @@ -136,6 +144,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], env: 'production', remote: { @@ -156,10 +165,43 @@ describe('loadConfig', () => { ]); }); - it('loads config with secrets', async () => { + it('loads config with secrets from two different files', async () => { await expect( loadConfig({ configRoot: '/root', + configPaths: ['/root/app-config2.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], + env: 'production', + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + { + context: 'app-config2.yaml', + data: { + app: { + title: 'Example App 2', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from single file', async () => { + await expect( + loadConfig({ + configRoot: '/root', + configPaths: ['/root/app-config.yaml'], configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), @@ -181,6 +223,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [ { path: '/root/app-config.yaml' }, { path: '/root/app-config.development.yaml' }, @@ -221,6 +264,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), @@ -246,6 +290,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: onChange.resolve, @@ -294,6 +339,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], watch: { onChange: onChange.resolve, @@ -339,6 +385,7 @@ describe('loadConfig', () => { await loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: () => { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2e7a0df801..1f20c4e437 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -24,10 +24,10 @@ import { createIncludeTransform, createSubstitutionTransform, EnvFunc, + isValidUrl, readEnvConfig, } from './lib'; import fetch from 'node-fetch'; -import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; @@ -55,6 +55,11 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; + /** Absolute paths to load config files from. Configs from earlier paths have lower priority. + * @deprecated Use {@link configTargets} instead. + */ + configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. configTargets: ConfigTarget[]; @@ -94,6 +99,13 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); + // Append deprecated configPaths to the absolute config paths received via configTargets. + options.configPaths.forEach(cp => { + if (!configPaths.includes(cp)) { + configPaths.push(cp); + } + }); + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) diff --git a/yarn.lock b/yarn.lock index 00c91f7783..10c65d5df7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11336,11 +11336,6 @@ core-js@^2.5.0: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== -core-js@^3.0.0: - version "3.18.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" - integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== - core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14042,22 +14037,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock@^9.11.0: - version "9.11.0" - resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" - integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== - dependencies: - "@babel/core" "^7.0.0" - "@babel/runtime" "^7.0.0" - core-js "^3.0.0" - debug "^4.1.1" - glob-to-regexp "^0.4.0" - is-subset "^0.1.1" - lodash.isequal "^4.5.0" - path-to-regexp "^2.2.1" - querystring "^0.2.0" - whatwg-url "^6.5.0" - fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -14888,7 +14867,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -16872,11 +16851,6 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" -is-subset@^0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" - integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= - is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18745,7 +18719,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: +lodash.isequal@^4.0.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -21559,11 +21533,6 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -path-to-regexp@^2.2.1: - version "2.4.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -28010,7 +27979,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -whatwg-url@^6.4.1, whatwg-url@^6.5.0: +whatwg-url@^6.4.1: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From 5d4053c827ffc307bbeb3dc2ded8da137aee3b58 Mon Sep 17 00:00:00 2001 From: Matto Date: Mon, 27 Sep 2021 23:41:24 +1000 Subject: [PATCH 07/17] Read config from remote config server Signed-off-by: Matto --- .changeset/giant-years-help.md | 8 + packages/backend-common/src/config.ts | 10 +- packages/cli/package.json | 6 + packages/cli/src/lib/config.ts | 16 +- packages/config-loader/api-report.md | 34 +++- packages/config-loader/package.json | 10 +- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/loader.test.ts | 125 +++++++++++- packages/config-loader/src/loader.ts | 236 +++++++++++++++++++--- packages/integration/api-report.md | 5 + packages/integration/src/index.ts | 1 + yarn.lock | 74 ++++++- 12 files changed, 475 insertions(+), 52 deletions(-) create mode 100644 .changeset/giant-years-help.md diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md new file mode 100644 index 0000000000..f0ca12ef19 --- /dev/null +++ b/.changeset/giant-years-help.md @@ -0,0 +1,8 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +'@backstage/integration': patch +--- + +Reading app config from a remote server diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index fa49d5f015..90131ebbe7 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,8 @@ import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; -import { loadConfig } from '@backstage/config-loader'; +import { ConfigTarget, loadConfig } from '@backstage/config-loader'; +import { isValidUrl } from '@backstage/integration'; export class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -149,7 +150,10 @@ export async function loadBackendConfig(options: { argv: string[]; }): Promise { const args = parseArgs(options.argv); - const configPaths: string[] = [args.config ?? []].flat(); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); const config = new ObservableConfigProxy(options.logger); @@ -158,7 +162,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configPaths.map(opt => resolvePath(opt)), + configTargets: configTargets, watch: { onChange(newConfigs) { options.logger.info( diff --git a/packages/cli/package.json b/packages/cli/package.json index 62a008087e..d85adff199 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,10 +29,16 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.4", + "@babel/core": "^7.4.4", + "@babel/plugin-transform-modules-commonjs": "^7.4.4", + "@backstage/cli-common": "^0.1.3", + "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", + "@backstage/config-loader": "^0.6.8", + "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 3688d60089..79d30ee220 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; +import { + ConfigTarget, + loadConfig, + loadConfigSchema, +} from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; +import { isValidUrl } from '@backstage/integration'; type Options = { args: string[]; @@ -26,7 +31,12 @@ type Options = { }; export async function loadCliConfig(options: Options) { - const configPaths = options.args.map(arg => paths.resolveTarget(arg)); + const configTargets: ConfigTarget[] = []; + options.args.forEach(arg => { + if (!isValidUrl(arg)) { + configTargets.push({ path: paths.resolveTarget(arg) }); + } + }); // Consider all packages in the monorepo when loading in config const { Project } = require('@lerna/project'); @@ -48,7 +58,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, - configPaths, + configTargets: configTargets, }); // printing to stderr to not clobber stdout in case the cli command diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 19b5ee9e27..4ed409e3ce 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = { withFilteredKeys?: boolean; }; +// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ConfigTarget = + | { + path: string; + } + | { + url: string; + }; + // @public export type ConfigVisibility = 'frontend' | 'backend' | 'secret'; @@ -32,13 +43,11 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public export type LoadConfigOptions = { configRoot: string; - configPaths: string[]; + configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; - watch?: { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; - }; + remote?: Remote; + watch?: Watch; }; // @public @@ -64,6 +73,13 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; +// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Remote = { + reloadIntervalSeconds: number; +}; + // @public export type TransformFunc = ( value: T, @@ -71,4 +87,12 @@ export type TransformFunc = ( visibility: ConfigVisibility; }, ) => T | undefined; + +// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Watch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; ``` diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index a731225e2b..cab3a43fc5 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,10 +30,12 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", + "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", @@ -42,8 +44,10 @@ "json-schema-merge-allof": "^0.8.1", "json-schema-traverse": "^1.0.0", "typescript-json-schema": "^0.50.1", + "uuid": "^8.3.2", "yaml": "^1.9.2", - "yup": "^0.32.9" + "yup": "^0.32.9", + "node-fetch": "2.6.5" }, "devDependencies": { "@types/jest": "^26.0.7", @@ -51,7 +55,9 @@ "@types/mock-fs": "^4.10.0", "@types/node": "^14.14.32", "@types/yup": "^0.29.13", - "mock-fs": "^5.1.0" + "mock-fs": "^5.1.0", + "fetch-mock-jest": "1.5.1", + "fetch-mock": "^9.11.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 888c68ef70..ee3e6413ad 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -29,4 +29,4 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './loader'; +export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3f51e76936..190904a870 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,12 +18,33 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; +import { v4 as uuidv4 } from 'uuid'; + +const fetchMock = require('fetch-mock').sandbox(); +const nodeFetch = require('node-fetch'); + +nodeFetch.default = fetchMock; describe('loadConfig', () => { beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + method: 'GET', + }, + { + body: `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + ); + mockFs({ '/root/app-config.yaml': ` app: @@ -66,6 +87,7 @@ describe('loadConfig', () => { }); afterEach(() => { + fetchMock.restore(); mockFs.restore(); }); @@ -73,7 +95,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], env: 'production', }), ).resolves.toEqual([ @@ -90,11 +112,37 @@ describe('loadConfig', () => { ]); }); + it('load config from remote path', async () => { + const configUrl = 'https://some.domain.io/app-config.yaml'; + + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + env: 'production', + remote: { + reloadIntervalSeconds: 30, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), ).resolves.toEqual([ @@ -115,9 +163,9 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', + configTargets: [ + { path: '/root/app-config.yaml' }, + { path: '/root/app-config.development.yaml' }, ], env: 'development', }), @@ -155,7 +203,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.substitute.yaml'], + configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), ).resolves.toEqual([ @@ -180,7 +228,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: onChange.resolve, stopSignal: stopSignal.promise, @@ -218,12 +266,73 @@ describe('loadConfig', () => { stopSignal.resolve(); }); + it('watches remote config urls', async () => { + const onChange = defer(); + const stopSignal = defer(); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + await expect( + loadConfig({ + configRoot: '/root', + configTargets: [{ url: configUrl }], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + remote: { + reloadIntervalSeconds: 1, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + fetchMock.mock( + { + url: 'https://some.domain.io/app-config.yaml', + }, + { + body: `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + headers: { ETag: uuidv4().toString() }, + }, + { overwriteRoutes: true }, + ); + + await expect(onChange.promise).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'NEW ReMOTe ExaMPLe App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + it('stops watching config files', async () => { const stopSignal = defer(); await loadConfig({ configRoot: '/root', - configPaths: [], + configTargets: [], watch: { onChange: () => { expect('not').toBe('called'); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 6803d61786..2c61b4d1ce 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,15 +17,66 @@ import fs from 'fs-extra'; import yaml from 'yaml'; import chokidar from 'chokidar'; -import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; import { AppConfig } from '@backstage/config'; import { ForwardedError } from '@backstage/errors'; import { applyConfigTransforms, - readEnvConfig, createIncludeTransform, createSubstitutionTransform, + EnvFunc, + readEnvConfig, } from './lib'; +import fetch from 'node-fetch'; +import { isValidUrl } from '@backstage/integration'; + +export type ConfigTarget = { path: string } | { url: string }; + +export type Watch = { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; +}; + +/** + * Options that control the loading of configuration files in the backend. + * + * @public + */ +export type Remote = { + /** + * An optional remote config reloading period, in seconds + */ + reloadIntervalSeconds: number; +}; + +export type RemoteConfigProp = { + /** + * URL of the remote config + */ + url: string; + + /** + * Contents of the remote config + */ + content: string | null; + + /** + * An optional new ETag header value. Used when checking for updated config. + */ + newETag?: string; + + /** + * An optional old ETag header value. Used when checking for updated config + */ + oldETag?: string; +}; /** * Options that control the loading of configuration files in the backend. @@ -36,8 +87,8 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; - // Absolute paths to load config files from. Configs from earlier paths have lower priority. - configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. + configTargets: ConfigTarget[]; /** @deprecated This option has been removed */ env?: string; @@ -49,22 +100,19 @@ export type LoadConfigOptions = { */ experimentalEnvFunc?: (name: string) => Promise; + /** + * An optional remote config + */ + remote?: Remote; + /** * An optional configuration that enables watching of config files. */ - watch?: { - /** - * A listener that is called when a config file is changed. - */ - onChange: (configs: AppConfig[]) => void; - - /** - * An optional signal that stops the watcher once the promise resolves. - */ - stopSignal?: Promise; - }; + watch?: Watch; }; +const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; + /** * Load configuration data. * @@ -73,12 +121,30 @@ export type LoadConfigOptions = { export async function loadConfig( options: LoadConfigOptions, ): Promise { - const { configRoot, experimentalEnvFunc: envFunc, watch } = options; - const configPaths = options.configPaths.slice(); + const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options; + + const configPaths: string[] = options.configTargets + .slice() + .filter((e): e is { path: string } => e.hasOwnProperty('path')) + .map(configTarget => configTarget.path); + + let configUrls: string[] = options.configTargets + .slice() + .filter((e): e is { url: string } => e.hasOwnProperty('url')) + .map(configTarget => configTarget.url); + + const remoteConfigProps: RemoteConfigProp[] = []; + + if (remote === undefined && configUrls.length > 0) { + console.warn( + `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, + ); + configUrls = []; + } // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` - if (configPaths.length === 0) { + if (configPaths.length === 0 && configUrls.length === 0) { configPaths.push(resolvePath(configRoot, 'app-config.yaml')); const localConfig = resolvePath(configRoot, 'app-config.local.yaml'); @@ -114,22 +180,79 @@ export async function loadConfig( return configs; }; - let fileConfigs; + const loadRemoteConfigFiles = async () => { + const configs: AppConfig[] = []; + + const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { + const response = await fetch(remoteConfigProp.url); + if (!response.ok) { + throw new Error( + `Could not read config file at ${remoteConfigProp.url}`, + ); + } + + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + remoteConfigProp.content = await response.text(); + + return remoteConfigProp; + }; + + for (let i = 0; i < configUrls.length; i++) { + const remoteConfigProp = await readConfigFromUrl({ + url: configUrls[i], + content: null, + }); + + if (!isValidUrl(remoteConfigProp.url)) { + throw new Error( + `Config load path is not valid: '${remoteConfigProp.url}'`, + ); + } + + const dir = configRoot; + if (!remoteConfigProp.content) { + throw new Error(`Config is not valid`); + } + const input = yaml.parse(remoteConfigProp.content); + const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(dir, input, [ + substitutionTransform, + ]); + + configs.push({ data, context: remoteConfigProp.url }); + + remoteConfigProps.push(remoteConfigProp); + } + + return configs; + }; + + let fileConfigs: AppConfig[]; try { fileConfigs = await loadConfigFiles(); } catch (error) { throw new ForwardedError('Failed to read static configuration file', error); } + let remoteConfigs: AppConfig[] = []; + if (remote) { + try { + remoteConfigs = await loadRemoteConfigFiles(); + } catch (error) { + throw new Error(`Failed to read remote configuration file, ${error}`); + } + } + const envConfigs = await readEnvConfig(process.env); - // Set up config file watching if requested by the caller - if (watch) { - let currentSerializedConfig = JSON.stringify(fileConfigs); - + const watchConfigFile = (watchProp: Watch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); + + let currentSerializedConfig = JSON.stringify(fileConfigs); watcher.on('change', async () => { try { const newConfigs = await loadConfigFiles(); @@ -140,18 +263,77 @@ export async function loadConfig( } currentSerializedConfig = newSerializedConfig; - watch.onChange([...newConfigs, ...envConfigs]); + watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]); } catch (error) { console.error(`Failed to reload configuration files, ${error}`); } }); - if (watch.stopSignal) { - watch.stopSignal.then(() => { + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { watcher.close(); }); } + }; + + const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { + const requestProps = { method: 'HEAD' }; + const { headers } = await fetch(remoteConfigProp.url, requestProps); + remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; + remoteConfigProp.newETag = + headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; + + return ( + remoteConfigProp.oldETag !== undefined && + remoteConfigProp.newETag !== undefined && + remoteConfigProp.oldETag !== remoteConfigProp.newETag + ); + }; + + let handle: NodeJS.Timeout | undefined; + try { + handle = setInterval(async () => { + console.info(`Checking for config update`); + for (const remoteConfigProp of remoteConfigProps) { + if (await hasConfigChanged(remoteConfigProp)) { + console.info(`Remote config change, reloading config ...`); + const newRemoteConfigs = await loadRemoteConfigFiles(); + watchProp.onChange([ + ...newRemoteConfigs, + ...fileConfigs, + ...envConfigs, + ]); + console.info(`Remote config reloaded`); + break; + } + } + }, remoteProp.reloadIntervalSeconds * 1000); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { + if (handle !== undefined) { + console.info(`Stopping remote config watch`); + clearInterval(handle); + handle = undefined; + } + }); + } + }; + + // Set up config file watching if requested by the caller + if (watch) { + watchConfigFile(watch); } - return [...fileConfigs, ...envConfigs]; + if (watch && remote) { + watchRemoteConfig(watch, remote); + } + + return remote + ? [...remoteConfigs, ...fileConfigs, ...envConfigs] + : [...fileConfigs, ...envConfigs]; } diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 01f84383fe..e07f61eea6 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,6 +347,11 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function isValidUrl(url: string): boolean; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 3d5f336e8d..b8992f52d2 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,6 +27,7 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; +export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; diff --git a/yarn.lock b/yarn.lock index c905fbea6f..e45d685bf8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11555,6 +11555,11 @@ core-js@^2.4.0, core-js@^2.5.0, core-js@^2.6.10: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== +core-js@^3.0.0: + version "3.18.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" + integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== + core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14238,6 +14243,34 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== +fetch-blob@2.1.2: + version "2.1.2" + resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" + integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== + +fetch-mock-jest@1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" + integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== + dependencies: + fetch-mock "^9.11.0" + +fetch-mock@^9.11.0: + version "9.11.0" + resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" + integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== + dependencies: + "@babel/core" "^7.0.0" + "@babel/runtime" "^7.0.0" + core-js "^3.0.0" + debug "^4.1.1" + glob-to-regexp "^0.4.0" + is-subset "^0.1.1" + lodash.isequal "^4.5.0" + path-to-regexp "^2.2.1" + querystring "^0.2.0" + whatwg-url "^6.5.0" + fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -15062,7 +15095,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -17103,6 +17136,11 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" +is-subset@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" + integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= + is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -18971,7 +19009,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0: +lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -20815,6 +20853,13 @@ node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -22153,6 +22198,11 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" +path-to-regexp@^2.2.1: + version "2.4.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" + integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== + path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -27219,6 +27269,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + "traverse@>=0.3.0 <0.4": version "0.3.9" resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" @@ -28449,6 +28504,11 @@ web-streams-polyfill@4.0.0-beta.1: resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.1.tgz#3b19b9817374b7cee06d374ba7eeb3aeb80e8c95" integrity sha512-3ux37gEX670UUphBF9AMCq8XM6iQ8Ac6A+DSRRjDoRBm1ufCkaCDdNVbaqq60PsEkdNlLKrGtv/YBP4EJXqNtQ== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -28671,7 +28731,15 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^6.4.1: +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +whatwg-url@^6.4.1, whatwg-url@^6.5.0: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From e51740f9c96cb51c3e742df3289b56ce92264925 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 19 Oct 2021 23:18:19 +1100 Subject: [PATCH 08/17] Added documentation Signed-off-by: Matto --- docs/conf/writing.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4288d23b17..86b844484b 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,7 +1,6 @@ --- -id: writing -title: Writing Backstage Configuration Files -description: Documentation on Writing Backstage Configuration Files +id: writing title: Writing Backstage Configuration Files description: Documentation on Writing +Backstage Configuration Files --- ## File Format @@ -67,13 +66,15 @@ production build. ## Configuration Files -It is possible to have multiple configuration files, both to support different -environments, but also to define configuration that is local to specific -packages. The configuration files to load are selected using a `--config ` -flag, and it is possible to load any number of files. Paths are relative to the -working directory of the executed process, for example `package/backend`. This -means that to select a config file in the repo root when running the backend, -you would use `--config ../../my-config.yaml`. +It is possible to have multiple configuration files (bundled and/or remote), +both to support different environments, but also to define configuration that is +local to specific packages. The configuration files to load are selected using a +`--config ` flag, and it is possible to load any number of +files. Paths are relative to the working directory of the executed process, for +example `package/backend`. This means that to select a config file in the repo +root when running the backend, you would use `--config ../../my-config.yaml`, +and for config file on a config server you would use +`--config https://some.domain.io/app-config.yaml` If no `config` flags are specified, the default behavior is to load `app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root. @@ -85,7 +86,7 @@ are NOT loaded. To include them you need to explicitly include them with a flag, for example: ```shell -yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml +yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml --config https://some.domain.io/app-config.yaml ``` All loaded configuration files are merged together using the following rules: From ce843364f09c59a32f3d4ba8c178d33e73c51d9b Mon Sep 17 00:00:00 2001 From: Matto Date: Thu, 21 Oct 2021 18:20:36 +1100 Subject: [PATCH 09/17] Remove the dependency on Etag headers for reloading config Signed-off-by: Matto --- docs/conf/writing.md | 5 +- packages/cli/package.json | 2 - packages/config-loader/package.json | 4 +- packages/config-loader/src/loader.test.ts | 79 ++++++++++++---------- packages/config-loader/src/loader.ts | 82 +++++++---------------- yarn.lock | 7 -- 6 files changed, 73 insertions(+), 106 deletions(-) diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 86b844484b..a0e29820ed 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -1,6 +1,7 @@ --- -id: writing title: Writing Backstage Configuration Files description: Documentation on Writing -Backstage Configuration Files +id: writing +title: Writing Backstage Configuration Files +description: Documentation on Writing Backstage Configuration Files --- ## File Format diff --git a/packages/cli/package.json b/packages/cli/package.json index d85adff199..b9ba9fe8a3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,14 +31,12 @@ "@backstage/cli-common": "^0.1.4", "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", - "@backstage/cli-common": "^0.1.3", "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", "@backstage/config-loader": "^0.6.8", - "@backstage/backend-common": "^0.9.4", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index cab3a43fc5..456bca90ad 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -44,7 +44,6 @@ "json-schema-merge-allof": "^0.8.1", "json-schema-traverse": "^1.0.0", "typescript-json-schema": "^0.50.1", - "uuid": "^8.3.2", "yaml": "^1.9.2", "yup": "^0.32.9", "node-fetch": "2.6.5" @@ -56,8 +55,7 @@ "@types/node": "^14.14.32", "@types/yup": "^0.29.13", "mock-fs": "^5.1.0", - "fetch-mock-jest": "1.5.1", - "fetch-mock": "^9.11.0" + "msw": "^0.29.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 190904a870..e8b5a8828b 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,33 +18,47 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; -import { v4 as uuidv4 } from 'uuid'; - -const fetchMock = require('fetch-mock').sandbox(); -const nodeFetch = require('node-fetch'); - -nodeFetch.default = fetchMock; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; describe('loadConfig', () => { + const server = setupServer(); + const initialLoaderHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + const reloadHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + beforeAll(() => server.listen()); + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - method: 'GET', - }, - { - body: `app: - title: Remote Example App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - ); - mockFs({ '/root/app-config.yaml': ` app: @@ -87,10 +101,12 @@ describe('loadConfig', () => { }); afterEach(() => { - fetchMock.restore(); mockFs.restore(); + server.resetHandlers(); }); + afterAll(() => server.close()); + it('load config from default path', async () => { await expect( loadConfig({ @@ -113,6 +129,8 @@ describe('loadConfig', () => { }); it('load config from remote path', async () => { + server.use(initialLoaderHandler); + const configUrl = 'https://some.domain.io/app-config.yaml'; await expect( @@ -267,6 +285,8 @@ describe('loadConfig', () => { }); it('watches remote config urls', async () => { + server.use(initialLoaderHandler); + const onChange = defer(); const stopSignal = defer(); @@ -296,20 +316,7 @@ describe('loadConfig', () => { }, ]); - fetchMock.mock( - { - url: 'https://some.domain.io/app-config.yaml', - }, - { - body: `app: - title: NEW ReMOTe ExaMPLe App - sessionKey: 'abc123' - escaped: \$\${Escaped} - `, - headers: { ETag: uuidv4().toString() }, - }, - { overwriteRoutes: true }, - ); + server.use(reloadHandler); await expect(onChange.promise).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2c61b4d1ce..c35b742f92 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -56,6 +56,7 @@ export type Remote = { reloadIntervalSeconds: number; }; +/** @public */ export type RemoteConfigProp = { /** * URL of the remote config @@ -111,8 +112,6 @@ export type LoadConfigOptions = { watch?: Watch; }; -const HTTP_RESPONSE_HEADER_ETAG = 'ETag'; - /** * Load configuration data. * @@ -128,18 +127,15 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); - let configUrls: string[] = options.configTargets + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) .map(configTarget => configTarget.url); - const remoteConfigProps: RemoteConfigProp[] = []; - if (remote === undefined && configUrls.length > 0) { - console.warn( + throw new Error( `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, ); - configUrls = []; } // If no paths are provided, we default to reading @@ -183,47 +179,32 @@ export async function loadConfig( const loadRemoteConfigFiles = async () => { const configs: AppConfig[] = []; - const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => { - const response = await fetch(remoteConfigProp.url); + const readConfigFromUrl = async (url: string) => { + const response = await fetch(url); if (!response.ok) { - throw new Error( - `Could not read config file at ${remoteConfigProp.url}`, - ); + throw new Error(`Could not read config file at ${url}`); } - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - remoteConfigProp.content = await response.text(); - - return remoteConfigProp; + return await response.text(); }; for (let i = 0; i < configUrls.length; i++) { - const remoteConfigProp = await readConfigFromUrl({ - url: configUrls[i], - content: null, - }); - - if (!isValidUrl(remoteConfigProp.url)) { - throw new Error( - `Config load path is not valid: '${remoteConfigProp.url}'`, - ); + const configUrl = configUrls[i]; + if (!isValidUrl(configUrl)) { + throw new Error(`Config load path is not valid: '${configUrl}'`); } - const dir = configRoot; - if (!remoteConfigProp.content) { + const remoteConfigContent = await readConfigFromUrl(configUrl); + if (!remoteConfigContent) { throw new Error(`Config is not valid`); } - const input = yaml.parse(remoteConfigProp.content); + const configYaml = yaml.parse(remoteConfigContent); const substitutionTransform = createSubstitutionTransform(env); - const data = await applyConfigTransforms(dir, input, [ + const data = await applyConfigTransforms(configRoot, configYaml, [ substitutionTransform, ]); - configs.push({ data, context: remoteConfigProp.url }); - - remoteConfigProps.push(remoteConfigProp); + configs.push({ data, context: configUrl }); } return configs; @@ -277,17 +258,12 @@ export async function loadConfig( }; const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { - const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => { - const requestProps = { method: 'HEAD' }; - const { headers } = await fetch(remoteConfigProp.url, requestProps); - remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined; - remoteConfigProp.newETag = - headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined; - + const hasConfigChanged = async ( + oldRemoteConfigs: AppConfig[], + newRemoteConfigs: AppConfig[], + ) => { return ( - remoteConfigProp.oldETag !== undefined && - remoteConfigProp.newETag !== undefined && - remoteConfigProp.oldETag !== remoteConfigProp.newETag + JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs) ); }; @@ -295,18 +271,12 @@ export async function loadConfig( try { handle = setInterval(async () => { console.info(`Checking for config update`); - for (const remoteConfigProp of remoteConfigProps) { - if (await hasConfigChanged(remoteConfigProp)) { - console.info(`Remote config change, reloading config ...`); - const newRemoteConfigs = await loadRemoteConfigFiles(); - watchProp.onChange([ - ...newRemoteConfigs, - ...fileConfigs, - ...envConfigs, - ]); - console.info(`Remote config reloaded`); - break; - } + const newRemoteConfigs = await loadRemoteConfigFiles(); + if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) { + remoteConfigs = newRemoteConfigs; + console.info(`Remote config change, reloading config ...`); + watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]); + console.info(`Remote config reloaded`); } }, remoteProp.reloadIntervalSeconds * 1000); } catch (error) { diff --git a/yarn.lock b/yarn.lock index e45d685bf8..113b0d48c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14248,13 +14248,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock-jest@1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/fetch-mock-jest/-/fetch-mock-jest-1.5.1.tgz#0e13df990d286d9239e284f12b279ed509bf53cd" - integrity sha512-+utwzP8C+Pax1GSka3nFXILWMY3Er2L+s090FOgqVNrNCPp0fDqgXnAHAJf12PLHi0z4PhcTaZNTz8e7K3fjqQ== - dependencies: - fetch-mock "^9.11.0" - fetch-mock@^9.11.0: version "9.11.0" resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" From 8e85a0bca056f8ea245405a7555911e2a001ca66 Mon Sep 17 00:00:00 2001 From: Matto Date: Tue, 26 Oct 2021 21:50:49 +1100 Subject: [PATCH 10/17] Change variable name Signed-off-by: Matto --- packages/config-loader/api-report.md | 12 ++++++------ packages/config-loader/src/index.ts | 7 ++++++- packages/config-loader/src/loader.ts | 22 ++++++++++------------ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 4ed409e3ce..3b61b67d7b 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -46,8 +46,8 @@ export type LoadConfigOptions = { configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; - remote?: Remote; - watch?: Watch; + remote?: LoadConfigOptionsRemote; + watch?: LoadConfigOptionsWatch; }; // @public @@ -73,10 +73,10 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Remote = { +export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; @@ -88,10 +88,10 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type Watch = { +export type LoadConfigOptionsWatch = { onChange: (configs: AppConfig[]) => void; stopSignal?: Promise; }; diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index ee3e6413ad..97f0d301a3 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -29,4 +29,9 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader'; +export type { + ConfigTarget, + LoadConfigOptions, + LoadConfigOptionsWatch, + LoadConfigOptionsRemote, +} from './loader'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index c35b742f92..8c4bbae9d6 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -32,7 +32,7 @@ import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; -export type Watch = { +export type LoadConfigOptionsWatch = { /** * A listener that is called when a config file is changed. */ @@ -44,12 +44,7 @@ export type Watch = { stopSignal?: Promise; }; -/** - * Options that control the loading of configuration files in the backend. - * - * @public - */ -export type Remote = { +export type LoadConfigOptionsRemote = { /** * An optional remote config reloading period, in seconds */ @@ -104,12 +99,12 @@ export type LoadConfigOptions = { /** * An optional remote config */ - remote?: Remote; + remote?: LoadConfigOptionsRemote; /** * An optional configuration that enables watching of config files. */ - watch?: Watch; + watch?: LoadConfigOptionsWatch; }; /** @@ -222,13 +217,13 @@ export async function loadConfig( try { remoteConfigs = await loadRemoteConfigFiles(); } catch (error) { - throw new Error(`Failed to read remote configuration file, ${error}`); + throw new ForwardedError(`Failed to read remote configuration file, ${error}`); } } const envConfigs = await readEnvConfig(process.env); - const watchConfigFile = (watchProp: Watch) => { + const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); @@ -257,7 +252,10 @@ export async function loadConfig( } }; - const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => { + const watchRemoteConfig = ( + watchProp: LoadConfigOptionsWatch, + remoteProp: LoadConfigOptionsRemote, + ) => { const hasConfigChanged = async ( oldRemoteConfigs: AppConfig[], newRemoteConfigs: AppConfig[], From 1cecd737f98cc0fca824073af9e0edd9cd75410d Mon Sep 17 00:00:00 2001 From: Matto Date: Wed, 27 Oct 2021 20:33:33 +1100 Subject: [PATCH 11/17] Removed dependency, introduced isValidUrl, and reinstated `configPaths` Signed-off-by: Matto --- packages/backend-common/src/config.ts | 3 +- packages/backend-common/src/urls.test.ts | 34 ++++++++++++++ packages/backend-common/src/urls.ts | 25 +++++++++++ packages/cli/src/lib/config.ts | 3 +- packages/cli/src/lib/urls.test.ts | 34 ++++++++++++++ packages/cli/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/api-report.md | 1 + packages/config-loader/package.json | 1 - packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/urls.test.ts | 34 ++++++++++++++ packages/config-loader/src/lib/urls.ts | 25 +++++++++++ packages/config-loader/src/loader.test.ts | 49 ++++++++++++++++++++- packages/config-loader/src/loader.ts | 14 +++++- yarn.lock | 37 ++-------------- 14 files changed, 247 insertions(+), 39 deletions(-) create mode 100644 packages/backend-common/src/urls.test.ts create mode 100644 packages/backend-common/src/urls.ts create mode 100644 packages/cli/src/lib/urls.test.ts create mode 100644 packages/cli/src/lib/urls.ts create mode 100644 packages/config-loader/src/lib/urls.test.ts create mode 100644 packages/config-loader/src/lib/urls.ts diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 90131ebbe7..9b0eb8dcd0 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -21,7 +21,7 @@ import { findPaths } from '@backstage/cli-common'; import { Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; import { ConfigTarget, loadConfig } from '@backstage/config-loader'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; export class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -162,6 +162,7 @@ export async function loadBackendConfig(options: { const configs = await loadConfig({ configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, watch: { onChange(newConfigs) { diff --git a/packages/backend-common/src/urls.test.ts b/packages/backend-common/src/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-common/src/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-common/src/urls.ts b/packages/backend-common/src/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-common/src/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 79d30ee220..80b374039f 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -21,7 +21,7 @@ import { } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -import { isValidUrl } from '@backstage/integration'; +import { isValidUrl } from './urls'; type Options = { args: string[]; @@ -58,6 +58,7 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, + configPaths: [], configTargets: configTargets, }); diff --git a/packages/cli/src/lib/urls.test.ts b/packages/cli/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/cli/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/cli/src/lib/urls.ts b/packages/cli/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/cli/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 3b61b67d7b..be4678aa0f 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -43,6 +43,7 @@ export function loadConfig(options: LoadConfigOptions): Promise; // @public export type LoadConfigOptions = { configRoot: string; + configPaths: string[]; configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 456bca90ad..58b0061996 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,7 +30,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/integration": "^0.6.5", "@backstage/cli-common": "^0.1.4", "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 32a0191cae..ca88b771ba 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readEnvConfig } from './env'; export * from './transform'; export * from './schema'; +export { isValidUrl } from './urls'; diff --git a/packages/config-loader/src/lib/urls.test.ts b/packages/config-loader/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/config-loader/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/config-loader/src/lib/urls.ts b/packages/config-loader/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/config-loader/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index e8b5a8828b..804254afa0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -67,6 +67,13 @@ describe('loadConfig', () => { $file: secrets/session-key.txt escaped: \$\${Escaped} `, + '/root/app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, '/root/app-config.development.yaml': ` app: sessionKey: development-key @@ -111,6 +118,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], env: 'production', }), @@ -136,6 +144,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], env: 'production', remote: { @@ -156,10 +165,43 @@ describe('loadConfig', () => { ]); }); - it('loads config with secrets', async () => { + it('loads config with secrets from two different files', async () => { await expect( loadConfig({ configRoot: '/root', + configPaths: ['/root/app-config2.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], + env: 'production', + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + { + context: 'app-config2.yaml', + data: { + app: { + title: 'Example App 2', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from single file', async () => { + await expect( + loadConfig({ + configRoot: '/root', + configPaths: ['/root/app-config.yaml'], configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), @@ -181,6 +223,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [ { path: '/root/app-config.yaml' }, { path: '/root/app-config.development.yaml' }, @@ -221,6 +264,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), @@ -246,6 +290,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: onChange.resolve, @@ -294,6 +339,7 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [{ url: configUrl }], watch: { onChange: onChange.resolve, @@ -339,6 +385,7 @@ describe('loadConfig', () => { await loadConfig({ configRoot: '/root', + configPaths: [], configTargets: [], watch: { onChange: () => { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 8c4bbae9d6..338f199132 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -25,10 +25,10 @@ import { createIncludeTransform, createSubstitutionTransform, EnvFunc, + isValidUrl, readEnvConfig, } from './lib'; import fetch from 'node-fetch'; -import { isValidUrl } from '@backstage/integration'; export type ConfigTarget = { path: string } | { url: string }; @@ -83,6 +83,11 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; + /** Absolute paths to load config files from. Configs from earlier paths have lower priority. + * @deprecated Use {@link configTargets} instead. + */ + configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. configTargets: ConfigTarget[]; @@ -122,6 +127,13 @@ export async function loadConfig( .filter((e): e is { path: string } => e.hasOwnProperty('path')) .map(configTarget => configTarget.path); + // Append deprecated configPaths to the absolute config paths received via configTargets. + options.configPaths.forEach(cp => { + if (!configPaths.includes(cp)) { + configPaths.push(cp); + } + }); + const configUrls: string[] = options.configTargets .slice() .filter((e): e is { url: string } => e.hasOwnProperty('url')) diff --git a/yarn.lock b/yarn.lock index 113b0d48c2..079cd96a5c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11555,11 +11555,6 @@ core-js@^2.4.0, core-js@^2.5.0, core-js@^2.6.10: resolved "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== -core-js@^3.0.0: - version "3.18.3" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.3.tgz#86a0bba2d8ec3df860fefcc07a8d119779f01509" - integrity sha512-tReEhtMReZaPFVw7dajMx0vlsz3oOb8ajgPoHVYGxr8ErnZ6PcYEvvmjGmXlfpnxpkYSdOQttjB+MvVbCGfvLw== - core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: version "3.15.0" resolved "https://registry.npmjs.org/core-js/-/core-js-3.15.0.tgz#db9554ebce0b6fd90dc9b1f2465c841d2d055044" @@ -14248,22 +14243,6 @@ fetch-blob@2.1.2: resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== -fetch-mock@^9.11.0: - version "9.11.0" - resolved "https://registry.npmjs.org/fetch-mock/-/fetch-mock-9.11.0.tgz#371c6fb7d45584d2ae4a18ee6824e7ad4b637a3f" - integrity sha512-PG1XUv+x7iag5p/iNHD4/jdpxL9FtVSqRMUQhPab4hVDt80T1MH5ehzVrL2IdXO9Q2iBggArFvPqjUbHFuI58Q== - dependencies: - "@babel/core" "^7.0.0" - "@babel/runtime" "^7.0.0" - core-js "^3.0.0" - debug "^4.1.1" - glob-to-regexp "^0.4.0" - is-subset "^0.1.1" - lodash.isequal "^4.5.0" - path-to-regexp "^2.2.1" - querystring "^0.2.0" - whatwg-url "^6.5.0" - fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -15088,7 +15067,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob-to-regexp@^0.4.0, glob-to-regexp@^0.4.1: +glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== @@ -17129,11 +17108,6 @@ is-subdir@^1.1.1: dependencies: better-path-resolve "1.0.0" -is-subset@^0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/is-subset/-/is-subset-0.1.1.tgz#8a59117d932de1de00f245fcdd39ce43f1e939a6" - integrity sha1-ilkRfZMt4d4A8kX83TnOQ/HpOaY= - is-svg@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz#9321dbd29c212e5ca99c4fa9794c714bcafa2f75" @@ -19002,7 +18976,7 @@ lodash.isempty@^4.4.0: resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= -lodash.isequal@^4.0.0, lodash.isequal@^4.5.0: +lodash.isequal@^4.0.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA= @@ -22191,11 +22165,6 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -path-to-regexp@^2.2.1: - version "2.4.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.4.0.tgz#35ce7f333d5616f1c1e1bfe266c3aba2e5b2e704" - integrity sha512-G6zHoVqC6GGTQkZwF4lkuEyMbVOjoBKAEybQUypI1WTkqinCOrq2x6U2+phkJ1XsEMTy4LjtwPI7HW+NVrRR2w== - path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -28732,7 +28701,7 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -whatwg-url@^6.4.1, whatwg-url@^6.5.0: +whatwg-url@^6.4.1: version "6.5.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== From 5b51907cbf2da15f03a9e9fccd4ec381b80fd9f9 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 14:56:31 +1100 Subject: [PATCH 12/17] resolved conflicts Signed-off-by: Matto --- packages/backend-common/src/config.ts | 16 +++++++++++++++ packages/config-loader/src/loader.ts | 29 ++++----------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 9b0eb8dcd0..5b09d66608 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -84,51 +84,66 @@ export class ObservableConfigProxy implements Config { has(key: string): boolean { return this.select(false)?.has(key) ?? false; } + keys(): string[] { return this.select(false)?.keys() ?? []; } + get(key?: string): T { return this.select(true).get(key); } + getOptional(key?: string): T | undefined { return this.select(false)?.getOptional(key); } + getConfig(key: string): Config { return new ObservableConfigProxy(this.logger, this, key); } + getOptionalConfig(key: string): Config | undefined { if (this.select(false)?.has(key)) { return new ObservableConfigProxy(this.logger, this, key); } return undefined; } + getConfigArray(key: string): Config[] { return this.select(true).getConfigArray(key); } + getOptionalConfigArray(key: string): Config[] | undefined { return this.select(false)?.getOptionalConfigArray(key); } + getNumber(key: string): number { return this.select(true).getNumber(key); } + getOptionalNumber(key: string): number | undefined { return this.select(false)?.getOptionalNumber(key); } + getBoolean(key: string): boolean { return this.select(true).getBoolean(key); } + getOptionalBoolean(key: string): boolean | undefined { return this.select(false)?.getOptionalBoolean(key); } + getString(key: string): string { return this.select(true).getString(key); } + getOptionalString(key: string): string | undefined { return this.select(false)?.getOptionalString(key); } + getStringArray(key: string): string[] { return this.select(true).getStringArray(key); } + getOptionalStringArray(key: string): string[] | undefined { return this.select(false)?.getOptionalStringArray(key); } @@ -184,6 +199,7 @@ export async function loadBackendConfig(options: { } }), }, + remote: { reloadIntervalSeconds: 10 }, }); options.logger.info( diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 338f199132..b8fab12917 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -24,7 +24,6 @@ import { applyConfigTransforms, createIncludeTransform, createSubstitutionTransform, - EnvFunc, isValidUrl, readEnvConfig, } from './lib'; @@ -51,29 +50,6 @@ export type LoadConfigOptionsRemote = { reloadIntervalSeconds: number; }; -/** @public */ -export type RemoteConfigProp = { - /** - * URL of the remote config - */ - url: string; - - /** - * Contents of the remote config - */ - content: string | null; - - /** - * An optional new ETag header value. Used when checking for updated config. - */ - newETag?: string; - - /** - * An optional old ETag header value. Used when checking for updated config - */ - oldETag?: string; -}; - /** * Options that control the loading of configuration files in the backend. * @@ -229,7 +205,10 @@ export async function loadConfig( try { remoteConfigs = await loadRemoteConfigFiles(); } catch (error) { - throw new ForwardedError(`Failed to read remote configuration file, ${error}`); + throw new ForwardedError( + `Failed to read remote configuration file`, + error, + ); } } From 5cce78f968099c3701303fbb035918c8914e50c0 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 15:11:54 +1100 Subject: [PATCH 13/17] Resolved conflicts Signed-off-by: Matto --- .changeset/giant-years-help.md | 1 - ADOPTERS.md | 122 +++++++++++++------------- packages/backend-common/src/config.ts | 15 ---- packages/cli/package.json | 4 - packages/config-loader/package.json | 1 - packages/integration/api-report.md | 5 -- packages/integration/src/index.ts | 1 - 7 files changed, 61 insertions(+), 88 deletions(-) diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md index f0ca12ef19..c6ca0def30 100644 --- a/.changeset/giant-years-help.md +++ b/.changeset/giant-years-help.md @@ -2,7 +2,6 @@ '@backstage/backend-common': patch '@backstage/cli': patch '@backstage/config-loader': patch -'@backstage/integration': patch --- Reading app config from a remote server diff --git a/ADOPTERS.md b/ADOPTERS.md index 2d2583be95..db751bfb9b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,61 +1,61 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 31869c685a..9b0eb8dcd0 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -84,66 +84,51 @@ export class ObservableConfigProxy implements Config { has(key: string): boolean { return this.select(false)?.has(key) ?? false; } - keys(): string[] { return this.select(false)?.keys() ?? []; } - get(key?: string): T { return this.select(true).get(key); } - getOptional(key?: string): T | undefined { return this.select(false)?.getOptional(key); } - getConfig(key: string): Config { return new ObservableConfigProxy(this.logger, this, key); } - getOptionalConfig(key: string): Config | undefined { if (this.select(false)?.has(key)) { return new ObservableConfigProxy(this.logger, this, key); } return undefined; } - getConfigArray(key: string): Config[] { return this.select(true).getConfigArray(key); } - getOptionalConfigArray(key: string): Config[] | undefined { return this.select(false)?.getOptionalConfigArray(key); } - getNumber(key: string): number { return this.select(true).getNumber(key); } - getOptionalNumber(key: string): number | undefined { return this.select(false)?.getOptionalNumber(key); } - getBoolean(key: string): boolean { return this.select(true).getBoolean(key); } - getOptionalBoolean(key: string): boolean | undefined { return this.select(false)?.getOptionalBoolean(key); } - getString(key: string): string { return this.select(true).getString(key); } - getOptionalString(key: string): string | undefined { return this.select(false)?.getOptionalString(key); } - getStringArray(key: string): string[] { return this.select(true).getStringArray(key); } - getOptionalStringArray(key: string): string[] | undefined { return this.select(false)?.getOptionalStringArray(key); } diff --git a/packages/cli/package.json b/packages/cli/package.json index b9ba9fe8a3..62a008087e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,14 +29,10 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.4", - "@babel/core": "^7.4.4", - "@babel/plugin-transform-modules-commonjs": "^7.4.4", - "@backstage/integration": "^0.6.5", "@backstage/config": "^0.1.10", "@backstage/config-loader": "^0.7.0", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", - "@backstage/config-loader": "^0.6.8", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 58b0061996..efcf753295 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,7 +34,6 @@ "@backstage/config": "^0.1.9", "@backstage/errors": "^0.1.3", "@backstage/types": "^0.1.1", - "@backstage/backend-common": "^0.9.4", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "chokidar": "^3.5.2", diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index e07f61eea6..01f84383fe 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -347,11 +347,6 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; -// Warning: (ae-missing-release-tag) "isValidUrl" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export function isValidUrl(url: string): boolean; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index b8992f52d2..3d5f336e8d 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -27,7 +27,6 @@ export * from './gitlab'; export * from './googleGcs'; export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; -export { isValidUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; export type { ScmIntegrationRegistry } from './registry'; From f7cd672c7f655427162a7ca0260e3438742537d8 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 29 Oct 2021 16:39:13 +1100 Subject: [PATCH 14/17] Updated error message Signed-off-by: Matto --- packages/config-loader/src/loader.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index b8fab12917..4a5660f46c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -116,9 +116,7 @@ export async function loadConfig( .map(configTarget => configTarget.url); if (remote === undefined && configUrls.length > 0) { - throw new Error( - `Remote config detected, however, this feature is turned off. Remote config will be ignored.`, - ); + throw new Error(`Remote config detected but this feature is turned off`); } // If no paths are provided, we default to reading From 98b8ef555fbeb8d41141360307b21ef7546d6092 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 11:03:35 +1100 Subject: [PATCH 15/17] Re-ran prettier on all files Signed-off-by: Matto --- ADOPTERS.md | 128 ++++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index a2015086c3..797d178270 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,64 +1,64 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | -| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | -| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation.| -| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc.| +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | +| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | +| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | From e37a84085624703018b48746fad5337883317a97 Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 11:17:01 +1100 Subject: [PATCH 16/17] Re-ran prettier on all files Signed-off-by: Matto --- ADOPTERS.md | 128 +++++++++++++------------- packages/backend-common/src/config.ts | 2 +- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index a2015086c3..797d178270 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,64 +1,64 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | -| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | -| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation.| -| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc.| +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | +| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | +| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 3c5a2d7b26..3fd5e1b44e 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -22,7 +22,7 @@ import { loadConfigSchema, loadConfig, ConfigSchema, - ConfigTarget + ConfigTarget, } from '@backstage/config-loader'; import { AppConfig, Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; From 89e9f486f1f925684741d0f81391f73c5c89977e Mon Sep 17 00:00:00 2001 From: Matto Date: Fri, 5 Nov 2021 12:45:14 +1100 Subject: [PATCH 17/17] api-report fix Signed-off-by: Matto --- packages/config-loader/api-report.md | 30 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index be4678aa0f..6f568df942 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -51,6 +51,21 @@ export type LoadConfigOptions = { watch?: LoadConfigOptionsWatch; }; +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsRemote = { + reloadIntervalSeconds: number; +}; + +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsWatch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; +}; + // @public export function loadConfigSchema( options: LoadConfigSchemaOptions, @@ -74,13 +89,6 @@ export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[]; -// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type LoadConfigOptionsRemote = { - reloadIntervalSeconds: number; -}; - // @public export type TransformFunc = ( value: T, @@ -89,11 +97,7 @@ export type TransformFunc = ( }, ) => T | undefined; -// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warnings were encountered during analysis: // -// @public (undocumented) -export type LoadConfigOptionsWatch = { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; -}; +// src/loader.d.ts:33:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/config-loader" does not have an export "configTargets" ```