fix(catalog-backend): unify github and github/api, and use git-url-parse
This commit is contained in:
+7
-5
@@ -58,21 +58,23 @@ catalog:
|
||||
- allow: [Component, API, Group, Template, Location]
|
||||
processors:
|
||||
github:
|
||||
privateToken:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
githubApi:
|
||||
providers:
|
||||
- target: https://github.com
|
||||
token:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
# Example for how to add your GitHub Enterprise instance:
|
||||
#### Example for how to add your GitHub Enterprise instance using the API:
|
||||
# - target: https://ghe.example.net
|
||||
# apiBaseUrl: https://ghe.example.net/api/v3
|
||||
# token:
|
||||
# $secret:
|
||||
# env: GHE_PRIVATE_TOKEN
|
||||
#### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional):
|
||||
# - target: https://ghe.example.net
|
||||
# rawBaseUrl: https://ghe.example.net/raw
|
||||
# token:
|
||||
# $secret:
|
||||
# env: GHE_PRIVATE_TOKEN
|
||||
bitbucketApi:
|
||||
username:
|
||||
$secret:
|
||||
|
||||
@@ -65,10 +65,6 @@ catalog:
|
||||
- allow: [Component, API, Group, Template, Location]
|
||||
processors:
|
||||
github:
|
||||
privateToken:
|
||||
$secret:
|
||||
env: GITHUB_PRIVATE_TOKEN
|
||||
githubApi:
|
||||
providers:
|
||||
- target: https://github.com
|
||||
token:
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^3.0.3",
|
||||
"fs-extra": "^9.0.0",
|
||||
"git-url-parse": "^11.2.0",
|
||||
"knex": "^0.21.1",
|
||||
"lodash": "^4.17.15",
|
||||
"morgan": "^1.10.0",
|
||||
@@ -40,6 +41,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "^0.1.1-alpha.21",
|
||||
"@types/git-url-parse": "^9.0.0",
|
||||
"@types/lodash": "^4.14.151",
|
||||
"@types/node-fetch": "^2.5.7",
|
||||
"@types/supertest": "^2.0.8",
|
||||
|
||||
@@ -27,7 +27,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn
|
||||
import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
|
||||
import { FileReaderProcessor } from './processors/FileReaderProcessor';
|
||||
import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
|
||||
import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor';
|
||||
import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
|
||||
import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
|
||||
import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
|
||||
@@ -77,8 +76,7 @@ export class LocationReaders implements LocationReader {
|
||||
return [
|
||||
StaticLocationProcessor.fromConfig(config),
|
||||
new FileReaderProcessor(),
|
||||
new GithubReaderProcessor(config),
|
||||
GithubApiReaderProcessor.fromConfig(config),
|
||||
GithubReaderProcessor.fromConfig(config),
|
||||
new GitlabApiReaderProcessor(config),
|
||||
new GitlabReaderProcessor(),
|
||||
new BitbucketApiReaderProcessor(config),
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import fetch, { HeadersInit, RequestInit } from 'node-fetch';
|
||||
import * as result from './results';
|
||||
import { LocationProcessor, LocationProcessorEmit } from './types';
|
||||
|
||||
/**
|
||||
* The configuration parameters for a single GitHub API provider.
|
||||
*/
|
||||
export type ProviderConfig = {
|
||||
/**
|
||||
* The prefix of the target that this matches on, e.g. "https://github.com",
|
||||
* with no trailing slash.
|
||||
*/
|
||||
target: string;
|
||||
|
||||
/**
|
||||
* The base URL of the API of this provider, e.g. "https://api.github.com",
|
||||
* with no trailing slash.
|
||||
*/
|
||||
apiBaseUrl: string;
|
||||
|
||||
/**
|
||||
* The authorization token to use for requests to this provider.
|
||||
*
|
||||
* If no token is specified, anonymous API access is used.
|
||||
*/
|
||||
token?: string;
|
||||
};
|
||||
|
||||
export function getRequestOptions(provider: ProviderConfig): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/path/to/c.yaml
|
||||
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
|
||||
export function getRawUrl(target: string, provider: ProviderConfig): URL {
|
||||
try {
|
||||
const oldPath = new URL(target).pathname.split('/');
|
||||
const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath;
|
||||
|
||||
if (
|
||||
!userOrOrg ||
|
||||
!repoName ||
|
||||
(blobOrRaw !== 'blob' && blobOrRaw !== 'raw') ||
|
||||
!restOfPath.join('/').match(/\.ya?ml$/)
|
||||
) {
|
||||
throw new Error('Wrong URL or Invalid file path');
|
||||
}
|
||||
|
||||
// Transform to API path
|
||||
const newPath = [
|
||||
'repos',
|
||||
userOrOrg,
|
||||
repoName,
|
||||
'contents',
|
||||
...restOfPath,
|
||||
].join('/');
|
||||
return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function readConfig(configRoot: Config): ProviderConfig[] {
|
||||
const providers: ProviderConfig[] = [];
|
||||
|
||||
// In a previous version of the configuration, we only supported github,
|
||||
// and the "privateToken" key held the token to use for it. The new
|
||||
// configuration method is to use the "providers" key instead.
|
||||
const config = configRoot.getOptionalConfig('catalog.processors.githubApi');
|
||||
const providerConfigs = config?.getOptionalConfigArray('providers') ?? [];
|
||||
const legacyToken = config?.getOptionalString('privateToken');
|
||||
|
||||
// First read all the explicit providers
|
||||
for (const providerConfig of providerConfigs) {
|
||||
const target = providerConfig.getString('target').replace(/\/+$/, '');
|
||||
let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
|
||||
const token = providerConfig.getOptionalString('token');
|
||||
|
||||
if (apiBaseUrl) {
|
||||
apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
|
||||
} else if (target === 'https://github.com') {
|
||||
apiBaseUrl = 'https://api.github.com';
|
||||
} else {
|
||||
throw new Error(
|
||||
`Provider at ${target} must configure an explicit apiBaseUrl`,
|
||||
);
|
||||
}
|
||||
|
||||
providers.push({ target, apiBaseUrl, token });
|
||||
}
|
||||
|
||||
// If no explicit github.com provider was added, put one in the list as
|
||||
// a convenience
|
||||
if (!providers.some(p => p.target === 'https://github.com')) {
|
||||
providers.push({
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
token: legacyToken,
|
||||
});
|
||||
}
|
||||
|
||||
return providers;
|
||||
}
|
||||
|
||||
/**
|
||||
* A processor that adds the ability to read files from GitHub v3 APIs, such as
|
||||
* the one exposed by GitHub itself.
|
||||
*/
|
||||
export class GithubApiReaderProcessor implements LocationProcessor {
|
||||
private providers: ProviderConfig[];
|
||||
|
||||
static fromConfig(config: Config) {
|
||||
return new GithubApiReaderProcessor(readConfig(config));
|
||||
}
|
||||
|
||||
constructor(providers: ProviderConfig[]) {
|
||||
this.providers = providers;
|
||||
}
|
||||
|
||||
async readLocation(
|
||||
location: LocationSpec,
|
||||
optional: boolean,
|
||||
emit: LocationProcessorEmit,
|
||||
): Promise<boolean> {
|
||||
if (location.type !== 'github/api') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const provider = this.providers.find(p =>
|
||||
location.target.startsWith(`${p.target}/`),
|
||||
);
|
||||
if (!provider) {
|
||||
throw new Error(
|
||||
`There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const url = getRawUrl(location.target, provider);
|
||||
const options = getRequestOptions(provider);
|
||||
const response = await fetch(url.toString(), options);
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.buffer();
|
||||
emit(result.data(location, data));
|
||||
} else {
|
||||
const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
|
||||
if (response.status === 404) {
|
||||
if (!optional) {
|
||||
emit(result.notFoundError(location, message));
|
||||
}
|
||||
} else {
|
||||
emit(result.generalError(location, message));
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
const message = `Unable to read ${location.type} ${location.target}, ${e}`;
|
||||
emit(result.generalError(location, message));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+114
-18
@@ -17,18 +17,20 @@
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import {
|
||||
getApiUrl,
|
||||
getApiRequestOptions,
|
||||
getRawUrl,
|
||||
getRequestOptions,
|
||||
GithubApiReaderProcessor,
|
||||
getRawRequestOptions,
|
||||
GithubReaderProcessor,
|
||||
ProviderConfig,
|
||||
readConfig,
|
||||
} from './GithubApiReaderProcessor';
|
||||
} from './GithubReaderProcessor';
|
||||
|
||||
describe('GithubApiReaderProcessor', () => {
|
||||
describe('getRequestOptions', () => {
|
||||
describe('GithubReaderProcessor', () => {
|
||||
describe('getApiRequestOptions', () => {
|
||||
it('sets the correct API version', () => {
|
||||
const config: ProviderConfig = { target: '', apiBaseUrl: '' };
|
||||
expect((getRequestOptions(config).headers as any).Accept).toEqual(
|
||||
expect((getApiRequestOptions(config).headers as any).Accept).toEqual(
|
||||
'application/vnd.github.v3.raw',
|
||||
);
|
||||
});
|
||||
@@ -44,24 +46,95 @@ describe('GithubApiReaderProcessor', () => {
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getRequestOptions(withToken).headers as any).Authorization,
|
||||
(getApiRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getRequestOptions(withoutToken).headers as any).Authorization,
|
||||
(getApiRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRawRequestOptions', () => {
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: ProviderConfig = {
|
||||
target: '',
|
||||
rawBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: ProviderConfig = {
|
||||
target: '',
|
||||
rawBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getRawRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getRawRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getApiUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: ProviderConfig = { target: '', apiBaseUrl: '' };
|
||||
expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
|
||||
it('happy path for github', () => {
|
||||
const config: ProviderConfig = {
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe', () => {
|
||||
const config: ProviderConfig = {
|
||||
target: 'https://ghe.mycompany.net',
|
||||
apiBaseUrl: 'https://ghe.mycompany.net/api/v3',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRawUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: ProviderConfig = { target: '', apiBaseUrl: '' };
|
||||
expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
|
||||
it('passes through the happy path', () => {
|
||||
it('happy path for github', () => {
|
||||
const config: ProviderConfig = {
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
};
|
||||
expect(
|
||||
getRawUrl(
|
||||
@@ -70,10 +143,25 @@ describe('GithubApiReaderProcessor', () => {
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe', () => {
|
||||
const config: ProviderConfig = {
|
||||
target: 'https://ghe.mycompany.net',
|
||||
rawBaseUrl: 'https://ghe.mycompany.net/raw',
|
||||
};
|
||||
expect(
|
||||
getRawUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readConfig', () => {
|
||||
@@ -84,7 +172,7 @@ describe('GithubApiReaderProcessor', () => {
|
||||
{
|
||||
context: '',
|
||||
data: {
|
||||
catalog: { processors: { githubApi: { providers } } },
|
||||
catalog: { processors: { github: { providers } } },
|
||||
},
|
||||
},
|
||||
]);
|
||||
@@ -93,22 +181,30 @@ describe('GithubApiReaderProcessor', () => {
|
||||
it('adds a default GitHub entry when missing', () => {
|
||||
const output = readConfig(config([]));
|
||||
expect(output).toEqual([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
{
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('injects the correct GitHub API base URL when missing', () => {
|
||||
const output = readConfig(config([{ target: 'https://github.com' }]));
|
||||
expect(output).toEqual([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
{
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects custom targets with no API base URL', () => {
|
||||
it('rejects custom targets with no base URLs', () => {
|
||||
expect(() =>
|
||||
readConfig(config([{ target: 'https://ghe.company.com' }])),
|
||||
).toThrow(
|
||||
'Provider at https://ghe.company.com must configure an explicit apiBaseUrl',
|
||||
'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -132,7 +228,7 @@ describe('GithubApiReaderProcessor', () => {
|
||||
|
||||
describe('implementation', () => {
|
||||
it('rejects unknown types', async () => {
|
||||
const processor = new GithubApiReaderProcessor([
|
||||
const processor = new GithubReaderProcessor([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
const location: LocationSpec = {
|
||||
@@ -145,7 +241,7 @@ describe('GithubApiReaderProcessor', () => {
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
const processor = new GithubApiReaderProcessor([
|
||||
const processor = new GithubReaderProcessor([
|
||||
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
|
||||
]);
|
||||
const location: LocationSpec = {
|
||||
@@ -15,33 +15,199 @@
|
||||
*/
|
||||
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
|
||||
import { Config } from '@backstage/config';
|
||||
import parseGitUri from 'git-url-parse';
|
||||
import fetch, { HeadersInit, RequestInit } from 'node-fetch';
|
||||
import * as result from './results';
|
||||
import { LocationProcessor, LocationProcessorEmit } from './types';
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
export class GithubReaderProcessor implements LocationProcessor {
|
||||
private privateToken: string;
|
||||
/**
|
||||
* The configuration parameters for a single GitHub API provider.
|
||||
*/
|
||||
export type ProviderConfig = {
|
||||
/**
|
||||
* The prefix of the target that this matches on, e.g. "https://github.com",
|
||||
* with no trailing slash.
|
||||
*/
|
||||
target: string;
|
||||
|
||||
constructor(config?: Config) {
|
||||
this.privateToken =
|
||||
config?.getOptionalString('catalog.processors.github.privateToken') ?? '';
|
||||
/**
|
||||
* The base URL of the API of this provider, e.g. "https://api.github.com",
|
||||
* with no trailing slash.
|
||||
*
|
||||
* May be omitted specifically for GitHub; then it will be deduced.
|
||||
*
|
||||
* The API will always be preferred if both its base URL and a token are
|
||||
* present.
|
||||
*/
|
||||
apiBaseUrl?: string;
|
||||
|
||||
/**
|
||||
* The base URL of the raw fetch endpoint of this provider, e.g.
|
||||
* "https://raw.githubusercontent.com", with no trailing slash.
|
||||
*
|
||||
* May be omitted specifically for GitHub; then it will be deduced.
|
||||
*
|
||||
* The API will always be preferred if both its base URL and a token are
|
||||
* present.
|
||||
*/
|
||||
rawBaseUrl?: string;
|
||||
|
||||
/**
|
||||
* The authorization token to use for requests to this provider.
|
||||
*
|
||||
* If no token is specified, anonymous access is used.
|
||||
*/
|
||||
token?: string;
|
||||
};
|
||||
|
||||
export function getApiRequestOptions(provider: ProviderConfig): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
getRequestOptions(): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.privateToken !== '') {
|
||||
headers.Authorization = `token ${this.privateToken}`;
|
||||
export function getRawRequestOptions(provider: ProviderConfig): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/path/to/c.yaml
|
||||
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
|
||||
export function getApiUrl(target: string, provider: ProviderConfig): URL {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
|
||||
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
!ref ||
|
||||
(filepathtype !== 'blob' && filepathtype !== 'raw') ||
|
||||
!filepath?.match(/\.ya?ml$/)
|
||||
) {
|
||||
throw new Error('Wrong URL or invalid file path');
|
||||
}
|
||||
|
||||
const requestOptions: RequestInit = {
|
||||
headers,
|
||||
};
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
return new URL(
|
||||
`${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`,
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
return requestOptions;
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/c.yaml
|
||||
// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml
|
||||
export function getRawUrl(target: string, provider: ProviderConfig): URL {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
|
||||
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
!ref ||
|
||||
(filepathtype !== 'blob' && filepathtype !== 'raw') ||
|
||||
!filepath?.match(/\.ya?ml$/)
|
||||
) {
|
||||
throw new Error('Wrong URL or invalid file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
return new URL(
|
||||
`${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`,
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function readConfig(config: Config): ProviderConfig[] {
|
||||
const providers: ProviderConfig[] = [];
|
||||
|
||||
// In a previous version of the configuration, we only supported github,
|
||||
// and the "privateToken" key held the token to use for it. The new
|
||||
// configuration method is to use the "providers" key instead.
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('catalog.processors.github.providers') ??
|
||||
config.getOptionalConfigArray('catalog.processors.githubApi.providers') ??
|
||||
[];
|
||||
const legacyToken =
|
||||
config.getOptionalString('catalog.processors.github.privateToken') ??
|
||||
config.getOptionalString('catalog.processors.githubApi.privateToken');
|
||||
|
||||
// First read all the explicit providers
|
||||
for (const providerConfig of providerConfigs) {
|
||||
const target = providerConfig.getString('target').replace(/\/+$/, '');
|
||||
let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
|
||||
let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl');
|
||||
const token = providerConfig.getOptionalString('token');
|
||||
|
||||
if (apiBaseUrl) {
|
||||
apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
|
||||
} else if (target === 'https://github.com') {
|
||||
apiBaseUrl = 'https://api.github.com';
|
||||
}
|
||||
|
||||
if (rawBaseUrl) {
|
||||
rawBaseUrl = rawBaseUrl.replace(/\/+$/, '');
|
||||
} else if (target === 'https://github.com') {
|
||||
rawBaseUrl = 'https://raw.githubusercontent.com';
|
||||
}
|
||||
|
||||
if (!apiBaseUrl && !rawBaseUrl) {
|
||||
throw new Error(
|
||||
`Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`,
|
||||
);
|
||||
}
|
||||
|
||||
providers.push({ target, apiBaseUrl, rawBaseUrl, token });
|
||||
}
|
||||
|
||||
// If no explicit github.com provider was added, put one in the list as
|
||||
// a convenience
|
||||
if (!providers.some(p => p.target === 'https://github.com')) {
|
||||
providers.push({
|
||||
target: 'https://github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
token: legacyToken,
|
||||
});
|
||||
}
|
||||
|
||||
return providers;
|
||||
}
|
||||
|
||||
/**
|
||||
* A processor that adds the ability to read files from GitHub v3 APIs, such as
|
||||
* the one exposed by GitHub itself.
|
||||
*/
|
||||
export class GithubReaderProcessor implements LocationProcessor {
|
||||
private providers: ProviderConfig[];
|
||||
|
||||
static fromConfig(config: Config) {
|
||||
return new GithubReaderProcessor(readConfig(config));
|
||||
}
|
||||
|
||||
constructor(providers: ProviderConfig[]) {
|
||||
this.providers = providers;
|
||||
}
|
||||
|
||||
async readLocation(
|
||||
@@ -49,16 +215,31 @@ export class GithubReaderProcessor implements LocationProcessor {
|
||||
optional: boolean,
|
||||
emit: LocationProcessorEmit,
|
||||
): Promise<boolean> {
|
||||
if (location.type !== 'github') {
|
||||
// The github/api type is for backward compatibility
|
||||
if (location.type !== 'github' && location.type !== 'github/api') {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const url = this.buildRawUrl(location.target);
|
||||
const provider = this.providers.find(p =>
|
||||
location.target.startsWith(`${p.target}/`),
|
||||
);
|
||||
if (!provider) {
|
||||
throw new Error(
|
||||
`There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`,
|
||||
);
|
||||
}
|
||||
|
||||
// TODO(freben): Should "hard" errors thrown by this line be treated as
|
||||
// notFound instead of fatal?
|
||||
const response = await fetch(url.toString(), this.getRequestOptions());
|
||||
try {
|
||||
const useApi =
|
||||
provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl);
|
||||
const url = useApi
|
||||
? getApiUrl(location.target, provider)
|
||||
: getRawUrl(location.target, provider);
|
||||
const options = useApi
|
||||
? getApiRequestOptions(provider)
|
||||
: getRawRequestOptions(provider);
|
||||
console.log(url.toString());
|
||||
const response = await fetch(url.toString(), options);
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.buffer();
|
||||
@@ -80,41 +261,4 @@ export class GithubReaderProcessor implements LocationProcessor {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://github.com/a/b/blob/master/c.yaml
|
||||
// to: https://raw.githubusercontent.com/a/b/master/c.yaml
|
||||
private buildRawUrl(target: string): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
repoName,
|
||||
blobKeyword,
|
||||
...restOfPath
|
||||
] = url.pathname.split('/');
|
||||
|
||||
if (
|
||||
url.hostname !== 'github.com' ||
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
repoName === '' ||
|
||||
blobKeyword !== 'blob' ||
|
||||
!restOfPath.join('/').match(/\.yaml$/)
|
||||
) {
|
||||
throw new Error('Wrong GitHub URL');
|
||||
}
|
||||
|
||||
// Removing the "blob" part
|
||||
url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
|
||||
url.hostname = 'raw.githubusercontent.com';
|
||||
url.protocol = 'https';
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^3.0.3",
|
||||
"fs-extra": "^9.0.0",
|
||||
"git-url-parse": "^11.1.2",
|
||||
"git-url-parse": "^11.2.0",
|
||||
"globby": "^11.0.0",
|
||||
"helmet": "^4.0.0",
|
||||
"jsonschema": "^1.2.6",
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^3.0.3",
|
||||
"fs-extra": "^9.0.1",
|
||||
"git-url-parse": "^11.1.3",
|
||||
"git-url-parse": "^11.2.0",
|
||||
"knex": "^0.21.1",
|
||||
"node-fetch": "^2.6.0",
|
||||
"nodegit": "^0.27.0",
|
||||
|
||||
@@ -10695,13 +10695,20 @@ git-up@^4.0.0:
|
||||
is-ssh "^1.3.0"
|
||||
parse-url "^5.0.0"
|
||||
|
||||
git-url-parse@^11.1.2, git-url-parse@^11.1.3:
|
||||
git-url-parse@^11.1.2:
|
||||
version "11.1.3"
|
||||
resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.1.3.tgz#03625b6fc09905e9ad1da7bb2b84be1bf9123143"
|
||||
integrity sha512-GPsfwticcu52WQ+eHp0IYkAyaOASgYdtsQDIt4rUp6GbiNt1P9ddrh3O0kQB0eD4UJZszVqNT3+9Zwcg40fywA==
|
||||
dependencies:
|
||||
git-up "^4.0.0"
|
||||
|
||||
git-url-parse@^11.2.0:
|
||||
version "11.2.0"
|
||||
resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.2.0.tgz#2955fd51befd6d96ea1389bbe2ef57e8e6042b04"
|
||||
integrity sha512-KPoHZg8v+plarZvto4ruIzzJLFQoRx+sUs5DQSr07By9IBKguVd+e6jwrFR6/TP6xrCJlNV1tPqLO1aREc7O2g==
|
||||
dependencies:
|
||||
git-up "^4.0.0"
|
||||
|
||||
gitconfiglocal@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b"
|
||||
|
||||
Reference in New Issue
Block a user