paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

catalog(github): merge github v3 to github

Browse Source
This commit is contained in:
danztran
2020-07-08 12:14:28 +07:00
parent e9773a0bdb
commit c295d8ce06
5 changed files with 100 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/LocationReaders.ts
-1
View File
@@ -58,7 +58,6 @@ export class LocationReaders implements LocationReader {
return [
new FileReaderProcessor(),
new GithubReaderProcessor(),
new GithubV3ReaderProcessor(),
new GitlabReaderProcessor(),
new YamlProcessor(),
new EntityPolicyProcessor(entityPolicy),
plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts
+65
View File
@@ -0,0 +1,65 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GithubReaderProcessor } from './GithubReaderProcessor';
describe('GithubReaderProcessor', () => {
it('should build raw api', () => {
const processor = new GithubReaderProcessor();
const tests = [
{
target: 'https://github.com/a/b/blob/master/path/to/c.yaml',
url: new URL(
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master',
),
err: undefined,
},
{
target: 'https://api.com/a/b/blob/master/path/to/c.yaml',
url: null,
err:
'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL',
},
{
target: 'com/a/b/blob/master/path/to/c.yaml',
url: null,
err:
'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
},
{
target:
'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml',
url: new URL(
'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master',
),
err: undefined,
},
];
for (const test of tests) {
if (test.err) {
expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
} else {
expect(processor.buildRawUrl(test.target)).toEqual(test.url);
}
}
});
it('should return request options', () => {
// todo
});
});
plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts
+35 -9
View File
@@ -15,11 +15,28 @@
*/
import { LocationSpec } from '@backstage/catalog-model';
import fetch from 'node-fetch';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
const privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || '';
export class GithubReaderProcessor implements LocationProcessor {
getRequestOptions(): RequestInit {
const headers: HeadersInit = {
Accept: 'application/vnd.github.v3.raw',
};
if (privateToken) {
headers.Authorization = `token ${privateToken}`;
}
const requestOptions: RequestInit = {
headers,
};
return requestOptions;
}
async readLocation(
location: LocationSpec,
optional: boolean,
@@ -34,7 +51,7 @@ export class GithubReaderProcessor implements LocationProcessor {
// TODO(freben): Should "hard" errors thrown by this line be treated as
// notFound instead of fatal?
const response = await fetch(url.toString());
const response = await fetch(url.toString(), this.getRequestOptions());
if (response.ok) {
const data = await response.buffer();
@@ -58,9 +75,9 @@ export class GithubReaderProcessor implements LocationProcessor {
}
// Converts
// from: https://github.com/a/b/blob/master/c.yaml
// to: https://raw.githubusercontent.com/a/b/master/c.yaml
private buildRawUrl(target: string): URL {
// from: https://github.com/a/b/blob/master/path/to/c.yaml
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master
buildRawUrl(target: string): URL {
try {
const url = new URL(target);
@@ -69,6 +86,7 @@ export class GithubReaderProcessor implements LocationProcessor {
userOrOrg,
repoName,
blobKeyword,
ref,
...restOfPath
] = url.pathname.split('/');
@@ -80,13 +98,21 @@ export class GithubReaderProcessor implements LocationProcessor {
blobKeyword !== 'blob' ||
!restOfPath.join('/').match(/\.yaml$/)
) {
throw new Error('Wrong GitHub URL');
throw new Error('Wrong GitHub URL or Invalid file path');
}
// Removing the "blob" part
url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
url.hostname = 'raw.githubusercontent.com';
// transform to api
url.pathname = [
empty,
'repos',
userOrOrg,
repoName,
'contents',
...restOfPath,
].join('/');
url.hostname = 'api.github.com';
url.protocol = 'https';
url.search = `ref=${ref}`;
return url;
} catch (e) {
plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts
-21
View File
@@ -1,21 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor';
describe('GithubV3ReaderProcessor', () => {
// it('should build raw url')
});
plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts
-119
View File
@@ -1,119 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec } from '@backstage/catalog-model';
import fetch from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
const privateToken = process.env.GITHUB_PRIVATE_TOKEN;
export class GithubV3ReaderProcessor implements LocationProcessor {
getRequestOptions(): RequestInit {
const requestOptions: RequestInit = {
headers: {
Accept: 'application/vnd.github.v3.raw',
},
};
if (privateToken) {
requestOptions.headers.Authorization = `token ${privateToken}`;
}
return requestOptions;
}
async readLocation(
location: LocationSpec,
optional: boolean,
emit: LocationProcessorEmit,
): Promise<boolean> {
if (location.type !== 'github/v3') {
return false;
}
try {
const url = this.buildRawUrl(location.target);
const response = await fetch(url.toString(), this.getRequestOptions());
if (response.ok) {
const buffer = await response.buffer();
emit(result.data(location, buffer));
} else {
const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!optional) {
emit(result.notFoundError(location, message));
}
} else {
emit(result.generalError(location, message));
}
}
} catch (e) {
const message = `Unable to read ${location.type} ${location.target}, ${e}`;
emit(result.generalError(location, message));
}
return true;
}
// Converts
// from: https://github.com/a/b/blob/master/path/to/c.yaml
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master
buildRawUrl(target: string): URL {
try {
const url = new URL(target);
const [
empty,
userOrOrg,
repoName,
blobKeyword,
ref,
...restOfPath
] = url.pathname.split('/');
if (
url.hostname !== 'github.com' ||
empty !== '' ||
userOrOrg === '' ||
repoName === '' ||
blobKeyword !== 'blob' ||
!restOfPath.join('/').match(/\.yaml$/)
) {
throw new Error('Wrong GitHub URL');
}
// transform to api
url.pathname = [
empty,
'repos',
userOrOrg,
repoName,
'contents',
...restOfPath,
].join('/');
url.hostname = 'api.github.com';
url.protocol = 'https';
url.search = `ref=${ref}`;
return url;
} catch (e) {
throw new Error(`Incorrect url: ${target}, ${e}`);
}
}
}