Merge pull request #2081 from o-farooq/master

Add Azure private repos ingestion processor

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
 import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
+import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
 import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
 import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
@@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader {
       new GitlabApiReaderProcessor(),
       new GitlabReaderProcessor(),
       new BitbucketApiReaderProcessor(),
+      new AzureApiReaderProcessor(),
       new UrlReaderProcessor(),
       new YamlProcessor(),
       new EntityPolicyProcessor(entityPolicy),

plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
+
+describe('AzureApiReaderProcessor', () => {
+  it('should build raw api', () => {
+    const processor = new AzureApiReaderProcessor();
+    const tests = [
+      {
+        target:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+        url: new URL(
+          'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1',
+        ),
+        err: undefined,
+      },
+      {
+        target: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+        url: null,
+        err:
+          'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
+      },
+      {
+        target: 'com/a/b/blob/master/path/to/c.yaml',
+        url: null,
+        err:
+          'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+      },
+    ];
+
+    for (const test of tests) {
+      if (test.err) {
+        expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
+      } else if (test.url) {
+        expect(processor.buildRawUrl(test.target).toString()).toEqual(
+          test.url.toString(),
+        );
+      } else {
+        throw new Error(
+          'This should not have happened. Either err or url should have matched.',
+        );
+      }
+    }
+  });
+
+  it('should return request options', () => {
+    const tests = [
+      {
+        token: '0123456789',
+        expect: {
+          headers: {
+            Authorization: 'Basic OjAxMjM0NTY3ODk=',
+          },
+        },
+      },
+      {
+        token: '',
+        expect: {
+          headers: {},
+        },
+      },
+    ];
+
+    for (const test of tests) {
+      process.env.AZURE_PRIVATE_TOKEN = test.token;
+      const processor = new AzureApiReaderProcessor();
+      expect(processor.getRequestOptions()).toEqual(test.expect);
+    }
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import fetch, { RequestInit, HeadersInit } from 'node-fetch';
+import * as result from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+
+export class AzureApiReaderProcessor implements LocationProcessor {
+  private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || '';
+
+  getRequestOptions(): RequestInit {
+    const headers: HeadersInit = {};
+
+    if (this.privateToken !== '') {
+      headers.Authorization = `Basic ${Buffer.from(
+        `:${this.privateToken}`,
+        'utf8',
+      ).toString('base64')}`;
+    }
+
+    const requestOptions: RequestInit = {
+      headers,
+    };
+
+    return requestOptions;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'azure/api') {
+      return false;
+    }
+
+    try {
+      const url = this.buildRawUrl(location.target);
+
+      const response = await fetch(url.toString(), this.getRequestOptions());
+
+      if (response.ok) {
+        const data = await response.buffer();
+        emit(result.data(location, data));
+      } else {
+        const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
+        if (response.status === 404) {
+          if (!optional) {
+            emit(result.notFoundError(location, message));
+          }
+        } else {
+          emit(result.generalError(location, message));
+        }
+      }
+    } catch (e) {
+      const message = `Unable to read ${location.type} ${location.target}, ${e}`;
+      emit(result.generalError(location, message));
+    }
+    return true;
+  }
+
+  // Converts
+  // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
+  // to:   https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1
+
+  buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        project,
+        srcKeyword,
+        repoName,
+      ] = url.pathname.split('/');
+
+      const path = url.searchParams.get('path') || '';
+      const ref = url.searchParams.get('version')?.substr(2);
+
+      if (
+        url.hostname !== 'dev.azure.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        project === '' ||
+        srcKeyword !== '_git' ||
+        repoName === '' ||
+        path === '' ||
+        ref === '' ||
+        !path.match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong Azure Devops URL or Invalid file path');
+      }
+
+      // transform to api
+      url.pathname = [
+        empty,
+        userOrOrg,
+        project,
+        '_apis',
+        'sourceProviders',
+        'TfsGit',
+        'filecontents',
+      ].join('/');
+
+      url.search = [
+        `repository=${repoName}`,
+        `commitOrBranch=${ref}`,
+        `path=${path}`,
+        'api-version=6.0-preview.1',
+      ].join('&');
+
+      url.protocol = 'https';
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+}

plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts

@@ -51,6 +51,22 @@ describe('YamlProcessor', () => {
     expect(never).not.toBeCalled();
   });
 
+  it('should process url that contains yaml', async () => {
+    const containsYamlLocationSpec = {
+      type: 'url',
+      target: 'http://example.com/component?path=test.yaml&c=1&d=2',
+    };
+
+    const buffer = Buffer.from([]);
+    const emit = jest.fn();
+
+    expect(
+      await processor.parseData(buffer, containsYamlLocationSpec, emit),
+    ).toBe(true);
+
+    expect(emit).toBeCalled();
+  });
+
   it('should process entity with yaml', async () => {
     const entity = {
       apiVersion: 'backstage.io/v1alpha1',

plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts

@@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor {
     location: LocationSpec,
     emit: LocationProcessorEmit,
   ): Promise<boolean> {
-    if (!location.target.match(/\.ya?ml$/)) {
+    if (!location.target.match(/\.ya?ml/)) {
       return false;
     }
 

plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx

@@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => {
     const { rendered } = setup();
     expect(
       await rendered.findByText(
-        'Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo.',
+        'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.',
       ),
     ).toBeInTheDocument();
 

plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx

@@ -71,7 +71,7 @@ const RegisterComponentForm: FC<Props> = ({ onSubmit, submitting }) => {
           name="componentLocation"
           required
           margin="normal"
-          helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo."
+          helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config."
           inputRef={register({
             required: true,
             validate: ComponentIdValidators,

plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx

@@ -79,7 +79,16 @@ const RegisterComponentPage: FC<{}> = () => {
     setFormState(FormStates.Submitting);
     const { componentLocation: target } = formData;
     try {
-      const data = await catalogApi.addLocation('github', target);
+      const typeMapping = [
+        { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' },
+        { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' },
+        { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' },
+        { url: /.*/, type: 'github' },
+      ];
+
+      const type = typeMapping.filter(item => item.url.test(target))[0].type;
+
+      const data = await catalogApi.addLocation(type, target);
 
       if (!isMounted()) return;
 

plugins/register-component/src/util/validate.test.ts

@@ -31,11 +31,12 @@ describe('ComponentIdValidators', () => {
     });
   });
   describe('yamlValidator', () => {
-    const errorMessage = "Must end with '.yaml'.";
+    const errorMessage = "Must contain '.yaml'.";
     test.each([
       [true, '.yaml'],
       [true, 'http://example.com/blob/master/service.yaml'],
       [true, 'https://example.yaml'],
+      [true, 'https://example.com?path=abc.yaml&c=1'],
       [errorMessage, '.yml'],
       [errorMessage, 'http://example.com/blob/master/service'],
       [errorMessage, undefined],

plugins/register-component/src/util/validate.ts

@@ -19,6 +19,6 @@ export const ComponentIdValidators = {
     (typeof value === 'string' && value.match(/^https:\/\//) !== null) ||
     'Must start with https://.',
   yamlValidator: (value: any) =>
-    (typeof value === 'string' && value.match(/.yaml$/) !== null) ||
-    "Must end with '.yaml'.",
+    (typeof value === 'string' && value.match(/.yaml/) !== null) ||
+    "Must contain '.yaml'.",
 };