From 257a3b52ed64d82230b270a2c04a9543fc1f42ec Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:17:57 +1200 Subject: [PATCH] Add Azure ingestion processor --- .../src/ingestion/LocationReaders.ts | 2 + .../AzureApiReaderProcessor.test.ts | 84 +++++++++++ .../processors/AzureApiReaderProcessor.ts | 134 ++++++++++++++++++ .../RegisterComponentForm.tsx | 2 +- .../RegisterComponentPage.tsx | 13 +- 5 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 05f93815b8..29892bb2dc 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor' import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; @@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader { new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(), + new AzureApiReaderProcessor(), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts new file mode 100644 index 0000000000..35101b2607 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; + +describe('BitbucketApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new AzureApiReaderProcessor(); + const tests = [ + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }, + }, + }, + { + token: '', + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + process.env.AZURE_PRIVATE_TOKEN = test.token; + const processor = new AzureApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..41b4336b4f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' || + !path.match(/\.yaml$/) + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'sourceProviders', + 'TfsGit', + 'filecontents', + ].join('/'); + + url.search = [ + `repository=${repoName}`, + `commitOrBranch=${ref}`, + `path=${path}`, + 'api-version=6.0-preview.1', + ].join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index c1ea9c454d..a61469e969 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo." + helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 85d63fe2cb..be497e9aa7 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,7 +79,18 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - const data = await catalogApi.addLocation('github', target); + var typeMapping = [ + { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, + { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, + { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, + { url: /.*/, type: 'github' }, + ]; + + var type = typeMapping.filter(function (item) { + return new RegExp(item.url).test(target); + })[0].type; + + const data = await catalogApi.addLocation(type, target); if (!isMounted()) return;