From 257a3b52ed64d82230b270a2c04a9543fc1f42ec Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:17:57 +1200 Subject: [PATCH 1/6] Add Azure ingestion processor --- .../src/ingestion/LocationReaders.ts | 2 + .../AzureApiReaderProcessor.test.ts | 84 +++++++++++ .../processors/AzureApiReaderProcessor.ts | 134 ++++++++++++++++++ .../RegisterComponentForm.tsx | 2 +- .../RegisterComponentPage.tsx | 13 +- 5 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 05f93815b8..29892bb2dc 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor' import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; @@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader { new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(), + new AzureApiReaderProcessor(), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts new file mode 100644 index 0000000000..35101b2607 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; + +describe('BitbucketApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new AzureApiReaderProcessor(); + const tests = [ + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }, + }, + }, + { + token: '', + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + process.env.AZURE_PRIVATE_TOKEN = test.token; + const processor = new AzureApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..41b4336b4f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' || + !path.match(/\.yaml$/) + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'sourceProviders', + 'TfsGit', + 'filecontents', + ].join('/'); + + url.search = [ + `repository=${repoName}`, + `commitOrBranch=${ref}`, + `path=${path}`, + 'api-version=6.0-preview.1', + ].join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index c1ea9c454d..a61469e969 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo." + helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 85d63fe2cb..be497e9aa7 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,7 +79,18 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - const data = await catalogApi.addLocation('github', target); + var typeMapping = [ + { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, + { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, + { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, + { url: /.*/, type: 'github' }, + ]; + + var type = typeMapping.filter(function (item) { + return new RegExp(item.url).test(target); + })[0].type; + + const data = await catalogApi.addLocation(type, target); if (!isMounted()) return; From c33fc6c356227a1f5b034350630a879c413e85be Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:41:58 +1200 Subject: [PATCH 2/6] Fix linting errors --- .../RegisterComponentPage/RegisterComponentPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index be497e9aa7..1f0eb4fead 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,14 +79,14 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - var typeMapping = [ + const typeMapping = [ { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, { url: /.*/, type: 'github' }, ]; - var type = typeMapping.filter(function (item) { + const type = typeMapping.filter(item => { return new RegExp(item.url).test(target); })[0].type; From 7e454952f7e623fde4ea0437ed98e08c9a6ac0b8 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:59:03 +1200 Subject: [PATCH 3/6] fix broken UI test --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index 01402ca19b..fd7a1cc0f8 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,14 +30,14 @@ const setup = (props?: Partial) => { ), }; }; -describe('RegisterComponentForm', () => { +fdescribe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo.', + 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); From 54b6a9cc06d17e3365096c3314c4e78e4bb87cca Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 01:02:32 +1200 Subject: [PATCH 4/6] undo test focus --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index fd7a1cc0f8..d4d2cf4789 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,7 +30,7 @@ const setup = (props?: Partial) => { ), }; }; -fdescribe('RegisterComponentForm', () => { +describe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { From f57f6a7402965d7c95eb0b73af2608f779770e99 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 14:26:39 +1200 Subject: [PATCH 5/6] modify ends with .yaml check to contains .yaml --- .../processors/AzureApiReaderProcessor.test.ts | 2 +- .../ingestion/processors/YamlProcessor.test.ts | 16 ++++++++++++++++ .../src/ingestion/processors/YamlProcessor.ts | 2 +- .../RegisterComponentPage.tsx | 4 +--- .../register-component/src/util/validate.test.ts | 3 ++- plugins/register-component/src/util/validate.ts | 4 ++-- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts index 35101b2607..0ef111fcb1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -16,7 +16,7 @@ import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; -describe('BitbucketApiReaderProcessor', () => { +describe('AzureApiReaderProcessor', () => { it('should build raw api', () => { const processor = new AzureApiReaderProcessor(); const tests = [ diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts index 587793a6de..9f1ede3f9e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts @@ -51,6 +51,22 @@ describe('YamlProcessor', () => { expect(never).not.toBeCalled(); }); + it('should process url that contains yaml', async () => { + const containsYamlLocationSpec = { + type: 'url', + target: 'http://example.com/component?path=test.yaml&c=1&d=2', + }; + + const buffer = Buffer.from([]); + const emit = jest.fn(); + + expect( + await processor.parseData(buffer, containsYamlLocationSpec, emit), + ).toBe(true); + + expect(emit).toBeCalled(); + }); + it('should process entity with yaml', async () => { const entity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 6a2b5cf419..79ae55fae1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor { location: LocationSpec, emit: LocationProcessorEmit, ): Promise { - if (!location.target.match(/\.ya?ml$/)) { + if (!location.target.match(/\.ya?ml/)) { return false; } diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 1f0eb4fead..e61f904347 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -86,9 +86,7 @@ const RegisterComponentPage: FC<{}> = () => { { url: /.*/, type: 'github' }, ]; - const type = typeMapping.filter(item => { - return new RegExp(item.url).test(target); - })[0].type; + const type = typeMapping.filter(item => item.url.test(target))[0].type; const data = await catalogApi.addLocation(type, target); diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index d062655bd2..e4fc90699b 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -31,11 +31,12 @@ describe('ComponentIdValidators', () => { }); }); describe('yamlValidator', () => { - const errorMessage = "Must end with '.yaml'."; + const errorMessage = "Must contain '.yaml'."; test.each([ [true, '.yaml'], [true, 'http://example.com/blob/master/service.yaml'], [true, 'https://example.yaml'], + [true, 'https://example.com?path=abc.yaml&c=1'], [errorMessage, '.yml'], [errorMessage, 'http://example.com/blob/master/service'], [errorMessage, undefined], diff --git a/plugins/register-component/src/util/validate.ts b/plugins/register-component/src/util/validate.ts index 78d20995f5..8552872015 100644 --- a/plugins/register-component/src/util/validate.ts +++ b/plugins/register-component/src/util/validate.ts @@ -19,6 +19,6 @@ export const ComponentIdValidators = { (typeof value === 'string' && value.match(/^https:\/\//) !== null) || 'Must start with https://.', yamlValidator: (value: any) => - (typeof value === 'string' && value.match(/.yaml$/) !== null) || - "Must end with '.yaml'.", + (typeof value === 'string' && value.match(/.yaml/) !== null) || + "Must contain '.yaml'.", }; From ba8bef38a637a3544b0aaae56a839fefcb393e56 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Tue, 25 Aug 2020 18:30:12 +1200 Subject: [PATCH 6/6] update casing --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- .../components/RegisterComponentForm/RegisterComponentForm.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index d4d2cf4789..e1226bbf32 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', + 'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index a61469e969..de3c54610d 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." + helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators,