paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Read config from remote config server

Browse Source 
Signed-off-by: Matto <muhamadto@gmail.com>
This commit is contained in:
Matto
2021-09-27 23:41:24 +10:00
parent 18d73ecc9d
commit 5d4053c827
12 changed files with 475 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/config-loader/api-report.md
+29 -5
View File
@@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = {
withFilteredKeys?: boolean;
};
// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type ConfigTarget =
| {
path: string;
}
| {
url: string;
};
// @public
export type ConfigVisibility = 'frontend' | 'backend' | 'secret';
@@ -32,13 +43,11 @@ export function loadConfig(options: LoadConfigOptions): Promise<AppConfig[]>;
// @public
export type LoadConfigOptions = {
configRoot: string;
configPaths: string[];
configTargets: ConfigTarget[];
env?: string;
experimentalEnvFunc?: (name: string) => Promise<string | undefined>;
watch?: {
onChange: (configs: AppConfig[]) => void;
stopSignal?: Promise<void>;
};
remote?: Remote;
watch?: Watch;
};
// @public
@@ -64,6 +73,13 @@ export function readEnvConfig(env: {
[name: string]: string | undefined;
}): AppConfig[];
// Warning: (ae-missing-release-tag) "Remote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type Remote = {
reloadIntervalSeconds: number;
};
// @public
export type TransformFunc<T extends number | string | boolean> = (
value: T,
@@ -71,4 +87,12 @@ export type TransformFunc<T extends number | string | boolean> = (
visibility: ConfigVisibility;
},
) => T | undefined;
// Warning: (ae-missing-release-tag) "Watch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type Watch = {
onChange: (configs: AppConfig[]) => void;
stopSignal?: Promise<void>;
};
```
packages/config-loader/package.json
+8 -2
View File
@@ -30,10 +30,12 @@
"clean": "backstage-cli clean"
},
"dependencies": {
"@backstage/integration": "^0.6.5",
"@backstage/cli-common": "^0.1.4",
"@backstage/config": "^0.1.9",
"@backstage/errors": "^0.1.3",
"@backstage/types": "^0.1.1",
"@backstage/backend-common": "^0.9.4",
"@types/json-schema": "^7.0.6",
"ajv": "^7.0.3",
"chokidar": "^3.5.2",
@@ -42,8 +44,10 @@
"json-schema-merge-allof": "^0.8.1",
"json-schema-traverse": "^1.0.0",
"typescript-json-schema": "^0.50.1",
"uuid": "^8.3.2",
"yaml": "^1.9.2",
"yup": "^0.32.9"
"yup": "^0.32.9",
"node-fetch": "2.6.5"
},
"devDependencies": {
"@types/jest": "^26.0.7",
@@ -51,7 +55,9 @@
"@types/mock-fs": "^4.10.0",
"@types/node": "^14.14.32",
"@types/yup": "^0.29.13",
"mock-fs": "^5.1.0"
"mock-fs": "^5.1.0",
"fetch-mock-jest": "1.5.1",
"fetch-mock": "^9.11.0"
},
"files": [
"dist"
packages/config-loader/src/index.ts
+1 -1
View File
@@ -29,4 +29,4 @@ export type {
TransformFunc,
} from './lib';
export { loadConfig } from './loader';
export type { LoadConfigOptions } from './loader';
export type { ConfigTarget, LoadConfigOptions, Watch, Remote } from './loader';
packages/config-loader/src/loader.test.ts
+117 -8
View File
@@ -18,12 +18,33 @@ import { AppConfig } from '@backstage/config';
import { loadConfig } from './loader';
import mockFs from 'mock-fs';
import fs from 'fs-extra';
import { v4 as uuidv4 } from 'uuid';
const fetchMock = require('fetch-mock').sandbox();
const nodeFetch = require('node-fetch');
nodeFetch.default = fetchMock;
describe('loadConfig', () => {
beforeEach(() => {
process.env.MY_SECRET = 'is-secret';
process.env.SUBSTITUTE_ME = 'substituted';
fetchMock.mock(
{
url: 'https://some.domain.io/app-config.yaml',
method: 'GET',
},
{
body: `app:
title: Remote Example App
sessionKey: 'abc123'
escaped: \$\${Escaped}
`,
headers: { ETag: uuidv4().toString() },
},
);
mockFs({
'/root/app-config.yaml': `
app:
@@ -66,6 +87,7 @@ describe('loadConfig', () => {
});
afterEach(() => {
fetchMock.restore();
mockFs.restore();
});
@@ -73,7 +95,7 @@ describe('loadConfig', () => {
await expect(
loadConfig({
configRoot: '/root',
configPaths: [],
configTargets: [],
env: 'production',
}),
).resolves.toEqual([
@@ -90,11 +112,37 @@ describe('loadConfig', () => {
]);
});
it('load config from remote path', async () => {
const configUrl = 'https://some.domain.io/app-config.yaml';
await expect(
loadConfig({
configRoot: '/root',
configTargets: [{ url: configUrl }],
env: 'production',
remote: {
reloadIntervalSeconds: 30,
},
}),
).resolves.toEqual([
{
context: configUrl,
data: {
app: {
title: 'Remote Example App',
sessionKey: 'abc123',
escaped: '${Escaped}',
},
},
},
]);
});
it('loads config with secrets', async () => {
await expect(
loadConfig({
configRoot: '/root',
configPaths: ['/root/app-config.yaml'],
configTargets: [{ path: '/root/app-config.yaml' }],
env: 'production',
}),
).resolves.toEqual([
@@ -115,9 +163,9 @@ describe('loadConfig', () => {
await expect(
loadConfig({
configRoot: '/root',
configPaths: [
'/root/app-config.yaml',
'/root/app-config.development.yaml',
configTargets: [
{ path: '/root/app-config.yaml' },
{ path: '/root/app-config.development.yaml' },
],
env: 'development',
}),
@@ -155,7 +203,7 @@ describe('loadConfig', () => {
await expect(
loadConfig({
configRoot: '/root',
configPaths: ['/root/app-config.substitute.yaml'],
configTargets: [{ path: '/root/app-config.substitute.yaml' }],
env: 'development',
}),
).resolves.toEqual([
@@ -180,7 +228,7 @@ describe('loadConfig', () => {
await expect(
loadConfig({
configRoot: '/root',
configPaths: [],
configTargets: [],
watch: {
onChange: onChange.resolve,
stopSignal: stopSignal.promise,
@@ -218,12 +266,73 @@ describe('loadConfig', () => {
stopSignal.resolve();
});
it('watches remote config urls', async () => {
const onChange = defer<AppConfig[]>();
const stopSignal = defer<void>();
const configUrl = 'https://some.domain.io/app-config.yaml';
await expect(
loadConfig({
configRoot: '/root',
configTargets: [{ url: configUrl }],
watch: {
onChange: onChange.resolve,
stopSignal: stopSignal.promise,
},
remote: {
reloadIntervalSeconds: 1,
},
}),
).resolves.toEqual([
{
context: configUrl,
data: {
app: {
title: 'Remote Example App',
sessionKey: 'abc123',
escaped: '${Escaped}',
},
},
},
]);
fetchMock.mock(
{
url: 'https://some.domain.io/app-config.yaml',
},
{
body: `app:
title: NEW ReMOTe ExaMPLe App
sessionKey: 'abc123'
escaped: \$\${Escaped}
`,
headers: { ETag: uuidv4().toString() },
},
{ overwriteRoutes: true },
);
await expect(onChange.promise).resolves.toEqual([
{
context: configUrl,
data: {
app: {
title: 'NEW ReMOTe ExaMPLe App',
sessionKey: 'abc123',
escaped: '${Escaped}',
},
},
},
]);
stopSignal.resolve();
});
it('stops watching config files', async () => {
const stopSignal = defer<void>();
await loadConfig({
configRoot: '/root',
configPaths: [],
configTargets: [],
watch: {
onChange: () => {
expect('not').toBe('called');
packages/config-loader/src/loader.ts
+209 -27
View File
@@ -17,15 +17,66 @@
import fs from 'fs-extra';
import yaml from 'yaml';
import chokidar from 'chokidar';
import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path';
import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path';
import { AppConfig } from '@backstage/config';
import { ForwardedError } from '@backstage/errors';
import {
applyConfigTransforms,
readEnvConfig,
createIncludeTransform,
createSubstitutionTransform,
EnvFunc,
readEnvConfig,
} from './lib';
import fetch from 'node-fetch';
import { isValidUrl } from '@backstage/integration';
export type ConfigTarget = { path: string } | { url: string };
export type Watch = {
/**
* A listener that is called when a config file is changed.
*/
onChange: (configs: AppConfig[]) => void;
/**
* An optional signal that stops the watcher once the promise resolves.
*/
stopSignal?: Promise<void>;
};
/**
* Options that control the loading of configuration files in the backend.
*
* @public
*/
export type Remote = {
/**
* An optional remote config reloading period, in seconds
*/
reloadIntervalSeconds: number;
};
export type RemoteConfigProp = {
/**
* URL of the remote config
*/
url: string;
/**
* Contents of the remote config
*/
content: string | null;
/**
* An optional new ETag header value. Used when checking for updated config.
*/
newETag?: string;
/**
* An optional old ETag header value. Used when checking for updated config
*/
oldETag?: string;
};
/**
* Options that control the loading of configuration files in the backend.
@@ -36,8 +87,8 @@ export type LoadConfigOptions = {
// The root directory of the config loading context. Used to find default configs.
configRoot: string;
// Absolute paths to load config files from. Configs from earlier paths have lower priority.
configPaths: string[];
// Paths to load config files from. Configs from earlier paths have lower priority.
configTargets: ConfigTarget[];
/** @deprecated This option has been removed */
env?: string;
@@ -49,22 +100,19 @@ export type LoadConfigOptions = {
*/
experimentalEnvFunc?: (name: string) => Promise<string | undefined>;
/**
* An optional remote config
*/
remote?: Remote;
/**
* An optional configuration that enables watching of config files.
*/
watch?: {
/**
* A listener that is called when a config file is changed.
*/
onChange: (configs: AppConfig[]) => void;
/**
* An optional signal that stops the watcher once the promise resolves.
*/
stopSignal?: Promise<void>;
};
watch?: Watch;
};
const HTTP_RESPONSE_HEADER_ETAG = 'ETag';
/**
* Load configuration data.
*
@@ -73,12 +121,30 @@ export type LoadConfigOptions = {
export async function loadConfig(
options: LoadConfigOptions,
): Promise<AppConfig[]> {
const { configRoot, experimentalEnvFunc: envFunc, watch } = options;
const configPaths = options.configPaths.slice();
const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options;
const configPaths: string[] = options.configTargets
.slice()
.filter((e): e is { path: string } => e.hasOwnProperty('path'))
.map(configTarget => configTarget.path);
let configUrls: string[] = options.configTargets
.slice()
.filter((e): e is { url: string } => e.hasOwnProperty('url'))
.map(configTarget => configTarget.url);
const remoteConfigProps: RemoteConfigProp[] = [];
if (remote === undefined && configUrls.length > 0) {
console.warn(
`Remote config detected, however, this feature is turned off. Remote config will be ignored.`,
);
configUrls = [];
}
// If no paths are provided, we default to reading
// `app-config.yaml` and, if it exists, `app-config.local.yaml`
if (configPaths.length === 0) {
if (configPaths.length === 0 && configUrls.length === 0) {
configPaths.push(resolvePath(configRoot, 'app-config.yaml'));
const localConfig = resolvePath(configRoot, 'app-config.local.yaml');
@@ -114,22 +180,79 @@ export async function loadConfig(
return configs;
};
let fileConfigs;
const loadRemoteConfigFiles = async () => {
const configs: AppConfig[] = [];
const readConfigFromUrl = async (remoteConfigProp: RemoteConfigProp) => {
const response = await fetch(remoteConfigProp.url);
if (!response.ok) {
throw new Error(
`Could not read config file at ${remoteConfigProp.url}`,
);
}
remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined;
remoteConfigProp.newETag =
response.headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined;
remoteConfigProp.content = await response.text();
return remoteConfigProp;
};
for (let i = 0; i < configUrls.length; i++) {
const remoteConfigProp = await readConfigFromUrl({
url: configUrls[i],
content: null,
});
if (!isValidUrl(remoteConfigProp.url)) {
throw new Error(
`Config load path is not valid: '${remoteConfigProp.url}'`,
);
}
const dir = configRoot;
if (!remoteConfigProp.content) {
throw new Error(`Config is not valid`);
}
const input = yaml.parse(remoteConfigProp.content);
const substitutionTransform = createSubstitutionTransform(env);
const data = await applyConfigTransforms(dir, input, [
substitutionTransform,
]);
configs.push({ data, context: remoteConfigProp.url });
remoteConfigProps.push(remoteConfigProp);
}
return configs;
};
let fileConfigs: AppConfig[];
try {
fileConfigs = await loadConfigFiles();
} catch (error) {
throw new ForwardedError('Failed to read static configuration file', error);
}
let remoteConfigs: AppConfig[] = [];
if (remote) {
try {
remoteConfigs = await loadRemoteConfigFiles();
} catch (error) {
throw new Error(`Failed to read remote configuration file, ${error}`);
}
}
const envConfigs = await readEnvConfig(process.env);
// Set up config file watching if requested by the caller
if (watch) {
let currentSerializedConfig = JSON.stringify(fileConfigs);
const watchConfigFile = (watchProp: Watch) => {
const watcher = chokidar.watch(configPaths, {
usePolling: process.env.NODE_ENV === 'test',
});
let currentSerializedConfig = JSON.stringify(fileConfigs);
watcher.on('change', async () => {
try {
const newConfigs = await loadConfigFiles();
@@ -140,18 +263,77 @@ export async function loadConfig(
}
currentSerializedConfig = newSerializedConfig;
watch.onChange([...newConfigs, ...envConfigs]);
watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]);
} catch (error) {
console.error(`Failed to reload configuration files, ${error}`);
}
});
if (watch.stopSignal) {
watch.stopSignal.then(() => {
if (watchProp.stopSignal) {
watchProp.stopSignal.then(() => {
watcher.close();
});
}
};
const watchRemoteConfig = (watchProp: Watch, remoteProp: Remote) => {
const hasConfigChanged = async (remoteConfigProp: RemoteConfigProp) => {
const requestProps = { method: 'HEAD' };
const { headers } = await fetch(remoteConfigProp.url, requestProps);
remoteConfigProp.oldETag = remoteConfigProp.newETag ?? undefined;
remoteConfigProp.newETag =
headers.get(HTTP_RESPONSE_HEADER_ETAG) ?? undefined;
return (
remoteConfigProp.oldETag !== undefined &&
remoteConfigProp.newETag !== undefined &&
remoteConfigProp.oldETag !== remoteConfigProp.newETag
);
};
let handle: NodeJS.Timeout | undefined;
try {
handle = setInterval(async () => {
console.info(`Checking for config update`);
for (const remoteConfigProp of remoteConfigProps) {
if (await hasConfigChanged(remoteConfigProp)) {
console.info(`Remote config change, reloading config ...`);
const newRemoteConfigs = await loadRemoteConfigFiles();
watchProp.onChange([
...newRemoteConfigs,
...fileConfigs,
...envConfigs,
]);
console.info(`Remote config reloaded`);
break;
}
}
}, remoteProp.reloadIntervalSeconds * 1000);
} catch (error) {
console.error(`Failed to reload configuration files, ${error}`);
}
if (watchProp.stopSignal) {
watchProp.stopSignal.then(() => {
if (handle !== undefined) {
console.info(`Stopping remote config watch`);
clearInterval(handle);
handle = undefined;
}
});
}
};
// Set up config file watching if requested by the caller
if (watch) {
watchConfigFile(watch);
}
return [...fileConfigs, ...envConfigs];
if (watch && remote) {
watchRemoteConfig(watch, remote);
}
return remote
? [...remoteConfigs, ...fileConfigs, ...envConfigs]
: [...fileConfigs, ...envConfigs];
}