Merge pull request #2490 from spotify/freben/ghe
feat(catalog-backend): support GHE and other GitHub providers
This commit is contained in:
Fredrik Adelöw
+11
-3
@@ -62,9 +62,17 @@ catalog:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
githubApi:
|
||||
privateToken:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
providers:
|
||||
- target: https://github.com
|
||||
token:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
# Example for how to add your GitHub Enterprise instance:
|
||||
# - target: https://ghe.example.net
|
||||
# apiBaseUrl: https://ghe.example.net/api/v3
|
||||
# token:
|
||||
# $secret:
|
||||
# env: GHE_PRIVATE_TOKEN
|
||||
bitbucketApi:
|
||||
username:
|
||||
$secret:
|
||||
|
||||
@@ -61,6 +61,25 @@ scaffolder:
|
||||
visibility: public # or 'internal' or 'private'
|
||||
|
||||
catalog:
|
||||
rules:
|
||||
- allow: [Component, API, Group, Template, Location]
|
||||
processors:
|
||||
github:
|
||||
privateToken:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
githubApi:
|
||||
providers:
|
||||
- target: https://github.com
|
||||
token:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
# Example for how to add your GitHub Enterprise instance:
|
||||
# - target: https://ghe.example.net
|
||||
# apiBaseUrl: https://ghe.example.net/api/v3
|
||||
# token:
|
||||
# $secret:
|
||||
# env: GHE_PRIVATE_TOKEN
|
||||
locations:
|
||||
# Backstage example components
|
||||
- type: github
|
||||
|
||||
@@ -78,7 +78,7 @@ export class LocationReaders implements LocationReader {
|
||||
StaticLocationProcessor.fromConfig(config),
|
||||
new FileReaderProcessor(),
|
||||
new GithubReaderProcessor(config),
|
||||
new GithubApiReaderProcessor(config),
|
||||
GithubApiReaderProcessor.fromConfig(config),
|
||||
new GitlabApiReaderProcessor(config),
|
||||
new GitlabReaderProcessor(),
|
||||
new BitbucketApiReaderProcessor(config),
|
||||
|
||||
+136
-103
@@ -14,116 +14,149 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { GithubApiReaderProcessor } from './GithubApiReaderProcessor';
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
getRawUrl,
|
||||
getRequestOptions,
|
||||
GithubApiReaderProcessor,
|
||||
ProviderConfig,
|
||||
readConfig,
|
||||
} from './GithubApiReaderProcessor';
|
||||
|
||||
describe('GithubApiReaderProcessor', () => {
|
||||
const createConfig = (token: string | undefined) =>
|
||||
ConfigReader.fromConfigs([
|
||||
{
|
||||
context: '',
|
||||
data: {
|
||||
catalog: {
|
||||
processors: {
|
||||
githubApi: {
|
||||
privateToken: token,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
describe('getRequestOptions', () => {
|
||||
it('sets the correct API version', () => {
|
||||
const config: ProviderConfig = { target: '', apiBaseUrl: '' };
|
||||
expect((getRequestOptions(config).headers as any).Accept).toEqual(
|
||||
'application/vnd.github.v3.raw',
|
||||
);
|
||||
});
|
||||
|
||||
it('should build raw api', () => {
|
||||
const processor = new GithubApiReaderProcessor(createConfig(undefined));
|
||||
|
||||
const tests = [
|
||||
{
|
||||
target: 'https://github.com/a/b/blob/master/path/to/c.yaml',
|
||||
url: new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master',
|
||||
),
|
||||
err: undefined,
|
||||
},
|
||||
{
|
||||
target: 'https://api.com/a/b/blob/master/path/to/c.yaml',
|
||||
url: null,
|
||||
err:
|
||||
'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL or Invalid file path',
|
||||
},
|
||||
{
|
||||
target: 'com/a/b/blob/master/path/to/c.yaml',
|
||||
url: null,
|
||||
err:
|
||||
'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
|
||||
},
|
||||
{
|
||||
target:
|
||||
'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml',
|
||||
url: new URL(
|
||||
'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master',
|
||||
),
|
||||
err: undefined,
|
||||
},
|
||||
];
|
||||
|
||||
for (const test of tests) {
|
||||
if (test.err) {
|
||||
expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
|
||||
} else if (test.url) {
|
||||
expect(processor.buildRawUrl(test.target).toString()).toEqual(
|
||||
test.url.toString(),
|
||||
);
|
||||
} else {
|
||||
throw new Error(
|
||||
'This should not have happened. Either err or url should have matched.',
|
||||
);
|
||||
}
|
||||
}
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: ProviderConfig = {
|
||||
target: '',
|
||||
apiBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: ProviderConfig = {
|
||||
target: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('should return request options', () => {
|
||||
const tests = [
|
||||
{
|
||||
token: '0123456789',
|
||||
expect: {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
Authorization: 'token 0123456789',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
token: '',
|
||||
err:
|
||||
"Invalid type in config for key 'catalog.processors.githubApi.privateToken' in '', got empty-string, wanted string",
|
||||
expect: {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
token: undefined,
|
||||
expect: {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
describe('getRawUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: ProviderConfig = { target: '', apiBaseUrl: '' };
|
||||
expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
|
||||
for (const test of tests) {
|
||||
if (test.err) {
|
||||
expect(
|
||||
() => new GithubApiReaderProcessor(createConfig(test.token)),
|
||||
).toThrowError(test.err);
|
||||
} else {
|
||||
const processor = new GithubApiReaderProcessor(
|
||||
createConfig(test.token),
|
||||
);
|
||||
expect(processor.getRequestOptions()).toEqual(test.expect);
|
||||
}
|
||||
it('passes through the happy path', () => {
|
||||
const config: ProviderConfig = {
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
};
|
||||
expect(
|
||||
getRawUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readConfig', () => {
|
||||
function config(
|
||||
providers: { target: string; apiBaseUrl?: string; token?: string }[],
|
||||
) {
|
||||
return ConfigReader.fromConfigs([
|
||||
{
|
||||
context: '',
|
||||
data: {
|
||||
catalog: { processors: { githubApi: { providers } } },
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
it('adds a default GitHub entry when missing', () => {
|
||||
const output = readConfig(config([]));
|
||||
expect(output).toEqual([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('injects the correct GitHub API base URL when missing', () => {
|
||||
const output = readConfig(config([{ target: 'https://github.com' }]));
|
||||
expect(output).toEqual([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects custom targets with no API base URL', () => {
|
||||
expect(() =>
|
||||
readConfig(config([{ target: 'https://ghe.company.com' }])),
|
||||
).toThrow(
|
||||
'Provider at https://ghe.company.com must configure an explicit apiBaseUrl',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects funky configs', () => {
|
||||
expect(() => readConfig(config([{ target: 7 } as any]))).toThrow(
|
||||
/target/,
|
||||
);
|
||||
expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow(
|
||||
/target/,
|
||||
);
|
||||
expect(() =>
|
||||
readConfig(
|
||||
config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]),
|
||||
),
|
||||
).toThrow(/apiBaseUrl/);
|
||||
expect(() =>
|
||||
readConfig(config([{ target: 'https://github.com', token: 7 } as any])),
|
||||
).toThrow(/token/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation', () => {
|
||||
it('rejects unknown types', async () => {
|
||||
const processor = new GithubApiReaderProcessor([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
const location: LocationSpec = {
|
||||
type: 'not-github/api',
|
||||
target: 'https://github.com',
|
||||
};
|
||||
await expect(
|
||||
processor.readLocation(location, false, () => {}),
|
||||
).resolves.toBeFalsy();
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
const processor = new GithubApiReaderProcessor([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
const location: LocationSpec = {
|
||||
type: 'github/api',
|
||||
target: 'https://not.github.com/apa',
|
||||
};
|
||||
await expect(
|
||||
processor.readLocation(location, false, () => {}),
|
||||
).rejects.toThrow(
|
||||
/There is no GitHub provider that matches https:\/\/not.github.com\/apa/,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,34 +15,135 @@
|
||||
*/
|
||||
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
|
||||
import { Config } from '@backstage/config';
|
||||
import fetch, { HeadersInit, RequestInit } from 'node-fetch';
|
||||
import * as result from './results';
|
||||
import { LocationProcessor, LocationProcessorEmit } from './types';
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
export class GithubApiReaderProcessor implements LocationProcessor {
|
||||
private privateToken: string;
|
||||
/**
|
||||
* The configuration parameters for a single GitHub API provider.
|
||||
*/
|
||||
export type ProviderConfig = {
|
||||
/**
|
||||
* The prefix of the target that this matches on, e.g. "https://github.com",
|
||||
* with no trailing slash.
|
||||
*/
|
||||
target: string;
|
||||
|
||||
constructor(config: Config) {
|
||||
this.privateToken =
|
||||
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
|
||||
'';
|
||||
/**
|
||||
* The base URL of the API of this provider, e.g. "https://api.github.com",
|
||||
* with no trailing slash.
|
||||
*/
|
||||
apiBaseUrl: string;
|
||||
|
||||
/**
|
||||
* The authorization token to use for requests to this provider.
|
||||
*
|
||||
* If no token is specified, anonymous API access is used.
|
||||
*/
|
||||
token?: string;
|
||||
};
|
||||
|
||||
export function getRequestOptions(provider: ProviderConfig): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
getRequestOptions(): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.privateToken !== '') {
|
||||
headers.Authorization = `token ${this.privateToken}`;
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/path/to/c.yaml
|
||||
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
|
||||
export function getRawUrl(target: string, provider: ProviderConfig): URL {
|
||||
try {
|
||||
const oldPath = new URL(target).pathname.split('/');
|
||||
const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath;
|
||||
|
||||
if (
|
||||
!userOrOrg ||
|
||||
!repoName ||
|
||||
(blobOrRaw !== 'blob' && blobOrRaw !== 'raw') ||
|
||||
!restOfPath.join('/').match(/\.ya?ml$/)
|
||||
) {
|
||||
throw new Error('Wrong URL or Invalid file path');
|
||||
}
|
||||
|
||||
const requestOptions: RequestInit = {
|
||||
headers,
|
||||
};
|
||||
// Transform to API path
|
||||
const newPath = [
|
||||
'repos',
|
||||
userOrOrg,
|
||||
repoName,
|
||||
'contents',
|
||||
...restOfPath,
|
||||
].join('/');
|
||||
return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
return requestOptions;
|
||||
export function readConfig(configRoot: Config): ProviderConfig[] {
|
||||
const providers: ProviderConfig[] = [];
|
||||
|
||||
// In a previous version of the configuration, we only supported github,
|
||||
// and the "privateToken" key held the token to use for it. The new
|
||||
// configuration method is to use the "providers" key instead.
|
||||
const config = configRoot.getOptionalConfig('catalog.processors.githubApi');
|
||||
const providerConfigs = config?.getOptionalConfigArray('providers') ?? [];
|
||||
const legacyToken = config?.getOptionalString('privateToken');
|
||||
|
||||
// First read all the explicit providers
|
||||
for (const providerConfig of providerConfigs) {
|
||||
const target = providerConfig.getString('target').replace(/\/+$/, '');
|
||||
let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
|
||||
const token = providerConfig.getOptionalString('token');
|
||||
|
||||
if (apiBaseUrl) {
|
||||
apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
|
||||
} else if (target === 'https://github.com') {
|
||||
apiBaseUrl = 'https://api.github.com';
|
||||
} else {
|
||||
throw new Error(
|
||||
`Provider at ${target} must configure an explicit apiBaseUrl`,
|
||||
);
|
||||
}
|
||||
|
||||
providers.push({ target, apiBaseUrl, token });
|
||||
}
|
||||
|
||||
// If no explicit github.com provider was added, put one in the list as
|
||||
// a convenience
|
||||
if (!providers.some(p => p.target === 'https://github.com')) {
|
||||
providers.push({
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
token: legacyToken,
|
||||
});
|
||||
}
|
||||
|
||||
return providers;
|
||||
}
|
||||
|
||||
/**
|
||||
* A processor that adds the ability to read files from GitHub v3 APIs, such as
|
||||
* the one exposed by GitHub itself.
|
||||
*/
|
||||
export class GithubApiReaderProcessor implements LocationProcessor {
|
||||
private providers: ProviderConfig[];
|
||||
|
||||
static fromConfig(config: Config) {
|
||||
return new GithubApiReaderProcessor(readConfig(config));
|
||||
}
|
||||
|
||||
constructor(providers: ProviderConfig[]) {
|
||||
this.providers = providers;
|
||||
}
|
||||
|
||||
async readLocation(
|
||||
@@ -54,10 +155,19 @@ export class GithubApiReaderProcessor implements LocationProcessor {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const url = this.buildRawUrl(location.target);
|
||||
const provider = this.providers.find(p =>
|
||||
location.target.startsWith(`${p.target}/`),
|
||||
);
|
||||
if (!provider) {
|
||||
throw new Error(
|
||||
`There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`,
|
||||
);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), this.getRequestOptions());
|
||||
try {
|
||||
const url = getRawUrl(location.target, provider);
|
||||
const options = getRequestOptions(provider);
|
||||
const response = await fetch(url.toString(), options);
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.buffer();
|
||||
@@ -79,50 +189,4 @@ export class GithubApiReaderProcessor implements LocationProcessor {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://github.com/a/b/blob/master/path/to/c.yaml
|
||||
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master
|
||||
buildRawUrl(target: string): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
repoName,
|
||||
blobKeyword,
|
||||
ref,
|
||||
...restOfPath
|
||||
] = url.pathname.split('/');
|
||||
|
||||
if (
|
||||
url.hostname !== 'github.com' ||
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
repoName === '' ||
|
||||
blobKeyword !== 'blob' ||
|
||||
!restOfPath.join('/').match(/\.yaml$/)
|
||||
) {
|
||||
throw new Error('Wrong GitHub URL or Invalid file path');
|
||||
}
|
||||
|
||||
// transform to api
|
||||
url.pathname = [
|
||||
empty,
|
||||
'repos',
|
||||
userOrOrg,
|
||||
repoName,
|
||||
'contents',
|
||||
...restOfPath,
|
||||
].join('/');
|
||||
url.hostname = 'api.github.com';
|
||||
url.protocol = 'https';
|
||||
url.search = `ref=${ref}`;
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user