paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 47 Packages Projects Releases Wiki Activity

Add ingestor for private bitbucket cloud repositories (#1938)

Browse Source 
* Add ingestor for private bitbucket cloud repositories

* Fix test to compare URL string value instead of object

* Fix tests for Github and Gitlab ingestor to compare URL string value instead of object

* Fix bitbucket password env variable for clarity
This commit is contained in:
msamad
2020-08-17 18:01:36 +12:00
committed by GitHub
parent deb108d986
commit 42a885d07c
5 changed files with 245 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+2
View File
@@ -30,6 +30,7 @@ import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor';
import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
import { LocationRefProcessor } from './processors/LocationEntityProcessor';
import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
@@ -77,6 +78,7 @@ export class LocationReaders implements LocationReader {
new GithubApiReaderProcessor(),
new GitlabApiReaderProcessor(),
new GitlabReaderProcessor(),
new BitbucketApiReaderProcessor(),
new UrlReaderProcessor(),
new YamlProcessor(),
new EntityPolicyProcessor(entityPolicy),
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+102
View File
@@ -0,0 +1,102 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
describe('BitbucketApiReaderProcessor', () => {
it('should build raw api', () => {
const processor = new BitbucketApiReaderProcessor();
const tests = [
{
target:
'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
url: new URL(
'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml',
),
err: undefined,
},
{
target: 'https://api.com/a/b/blob/master/path/to/c.yaml',
url: null,
err:
'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path',
},
{
target: 'com/a/b/blob/master/path/to/c.yaml',
url: null,
err:
'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
},
];
for (const test of tests) {
if (test.err) {
expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
} else if (test.url) {
expect(processor.buildRawUrl(test.target).toString()).toEqual(
test.url.toString(),
);
} else {
throw new Error(
'This should not have happened. Either err or url should have matched.',
);
}
}
});
it('should return request options', () => {
const tests = [
{
username: '',
password: '',
expect: {
headers: {},
},
},
{
username: 'only-user-provided',
password: '',
expect: {
headers: {},
},
},
{
username: '',
password: 'only-password-provided',
expect: {
headers: {},
},
},
{
username: 'some-user',
password: 'my-secret',
expect: {
headers: {
Authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==',
},
},
},
];
for (const test of tests) {
process.env.BITBUCKET_USERNAME = test.username;
process.env.BITBUCKET_APP_PASSWORD = test.password;
const processor = new BitbucketApiReaderProcessor();
expect(processor.getRequestOptions()).toEqual(test.expect);
}
});
});
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+124
View File
@@ -0,0 +1,124 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
export class BitbucketApiReaderProcessor implements LocationProcessor {
private username: string = process.env.BITBUCKET_USERNAME || '';
private password: string = process.env.BITBUCKET_APP_PASSWORD || '';
getRequestOptions(): RequestInit {
const headers: HeadersInit = {};
if (this.username !== '' && this.password !== '') {
headers.Authorization = `Basic ${Buffer.from(
`${this.username}:${this.password}`,
'utf8',
).toString('base64')}`;
}
const requestOptions: RequestInit = {
headers,
};
return requestOptions;
}
async readLocation(
location: LocationSpec,
optional: boolean,
emit: LocationProcessorEmit,
): Promise<boolean> {
if (location.type !== 'bitbucket/api') {
return false;
}
try {
const url = this.buildRawUrl(location.target);
const response = await fetch(url.toString(), this.getRequestOptions());
if (response.ok) {
const data = await response.buffer();
emit(result.data(location, data));
} else {
const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
if (!optional) {
emit(result.notFoundError(location, message));
}
} else {
emit(result.generalError(location, message));
}
}
} catch (e) {
const message = `Unable to read ${location.type} ${location.target}, ${e}`;
emit(result.generalError(location, message));
}
return true;
}
// Converts
// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml
// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml
buildRawUrl(target: string): URL {
try {
const url = new URL(target);
const [
empty,
userOrOrg,
repoName,
srcKeyword,
ref,
...restOfPath
] = url.pathname.split('/');
if (
url.hostname !== 'bitbucket.org' ||
empty !== '' ||
userOrOrg === '' ||
repoName === '' ||
srcKeyword !== 'src' ||
!restOfPath.join('/').match(/\.yaml$/)
) {
throw new Error('Wrong Bitbucket URL or Invalid file path');
}
// transform to api
url.pathname = [
empty,
'2.0',
'repositories',
userOrOrg,
repoName,
'src',
ref,
...restOfPath,
].join('/');
url.hostname = 'api.bitbucket.org';
url.protocol = 'https';
return url;
} catch (e) {
throw new Error(`Incorrect url: ${target}, ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+7 -1
View File
@@ -53,8 +53,14 @@ describe('GithubApiReaderProcessor', () => {
for (const test of tests) {
if (test.err) {
expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
} else if (test.url) {
expect(processor.buildRawUrl(test.target).toString()).toEqual(
test.url.toString(),
);
} else {
expect(processor.buildRawUrl(test.target)).toEqual(test.url);
throw new Error(
'This should not have happened. Either err or url should have matched.',
);
}
}
});
plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+10 -4
View File
@@ -25,7 +25,7 @@ describe('GitlabApiReaderProcessor', () => {
target:
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
url: new URL(
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch',
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
),
err: undefined,
},
@@ -33,7 +33,7 @@ describe('GitlabApiReaderProcessor', () => {
target:
'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
url: new URL(
'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch',
'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
),
err: undefined,
},
@@ -41,7 +41,7 @@ describe('GitlabApiReaderProcessor', () => {
target:
'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
url: new URL(
'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch',
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
),
err: undefined,
},
@@ -59,8 +59,14 @@ describe('GitlabApiReaderProcessor', () => {
expect(() => processor.buildRawUrl(test.target, 12345)).toThrowError(
test.err,
);
} else if (test.url) {
expect(processor.buildRawUrl(test.target, 12345).toString()).toEqual(
test.url.toString(),
);
} else {
expect(processor.buildRawUrl(test.target, 12345)).toEqual(test.url);
throw new Error(
'This should not have happened. Either err or url should have matched.',
);
}
}
});