From 42a885d07c747947ea6c62f335d51a855fe9b087 Mon Sep 17 00:00:00 2001 From: msamad Date: Mon, 17 Aug 2020 18:01:36 +1200 Subject: [PATCH] Add ingestor for private bitbucket cloud repositories (#1938) * Add ingestor for private bitbucket cloud repositories * Fix test to compare URL string value instead of object * Fix tests for Github and Gitlab ingestor to compare URL string value instead of object * Fix bitbucket password env variable for clarity --- .../src/ingestion/LocationReaders.ts | 2 + .../BitbucketApiReaderProcessor.test.ts | 102 ++++++++++++++ .../processors/BitbucketApiReaderProcessor.ts | 124 ++++++++++++++++++ .../GithubApiReaderProcessor.test.ts | 8 +- .../GitlabApiReaderProcessor.test.ts | 14 +- 5 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 7e91af7c59..05f93815b8 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -30,6 +30,7 @@ import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; +import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; @@ -77,6 +78,7 @@ export class LocationReaders implements LocationReader { new GithubApiReaderProcessor(), new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), + new BitbucketApiReaderProcessor(), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts new file mode 100644 index 0000000000..fa4731a40e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; + +describe('BitbucketApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new BitbucketApiReaderProcessor(); + + const tests = [ + { + target: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + url: new URL( + 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + username: '', + password: '', + expect: { + headers: {}, + }, + }, + { + username: 'only-user-provided', + password: '', + expect: { + headers: {}, + }, + }, + { + username: '', + password: 'only-password-provided', + expect: { + headers: {}, + }, + }, + { + username: 'some-user', + password: 'my-secret', + expect: { + headers: { + Authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', + }, + }, + }, + ]; + + for (const test of tests) { + process.env.BITBUCKET_USERNAME = test.username; + process.env.BITBUCKET_APP_PASSWORD = test.password; + const processor = new BitbucketApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts new file mode 100644 index 0000000000..8162e505d6 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class BitbucketApiReaderProcessor implements LocationProcessor { + private username: string = process.env.BITBUCKET_USERNAME || ''; + private password: string = process.env.BITBUCKET_APP_PASSWORD || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.username !== '' && this.password !== '') { + headers.Authorization = `Basic ${Buffer.from( + `${this.username}:${this.password}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + srcKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'bitbucket.org' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + srcKeyword !== 'src' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong Bitbucket URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + '2.0', + 'repositories', + userOrOrg, + repoName, + 'src', + ref, + ...restOfPath, + ].join('/'); + url.hostname = 'api.bitbucket.org'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index 4dd3fd359f..63ddb071c3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -53,8 +53,14 @@ describe('GithubApiReaderProcessor', () => { for (const test of tests) { if (test.err) { expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); } else { - expect(processor.buildRawUrl(test.target)).toEqual(test.url); + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); } } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts index a429a0e6fb..cd46d97d13 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts @@ -25,7 +25,7 @@ describe('GitlabApiReaderProcessor', () => { target: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', url: new URL( - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', ), err: undefined, }, @@ -33,7 +33,7 @@ describe('GitlabApiReaderProcessor', () => { target: 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', url: new URL( - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', ), err: undefined, }, @@ -41,7 +41,7 @@ describe('GitlabApiReaderProcessor', () => { target: 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup url: new URL( - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', ), err: undefined, }, @@ -59,8 +59,14 @@ describe('GitlabApiReaderProcessor', () => { expect(() => processor.buildRawUrl(test.target, 12345)).toThrowError( test.err, ); + } else if (test.url) { + expect(processor.buildRawUrl(test.target, 12345).toString()).toEqual( + test.url.toString(), + ); } else { - expect(processor.buildRawUrl(test.target, 12345)).toEqual(test.url); + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); } } });