Merge branch 'master' into ebarriosjr/Creating-gcp-projects-plugin

2020-08-28 15:12:51 +02:00
parent cfeda75a4e b75c4495de
commit 47c353f04f
16 changed files with 319 additions and 53 deletions
@@ -61,7 +61,9 @@
      "prettier --write"
    ]
  },
-  "dependencies": {
-    "ts-node": "^8.10.2"
+  "jest": {
+    "transformModules": [
+      "@kyma-project/asyncapi-react"
+    ]
  }
 }
@@ -25,43 +25,6 @@ async function getConfig() {
    return require(path.resolve('jest.config.ts'));
  }

-  const options = {
-    rootDir: path.resolve('src'),
-    coverageDirectory: path.resolve('coverage'),
-    collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'],
-    moduleNameMapper: {
-      '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
-    },
-
-    // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed
-    // TODO: jest is working on module support, it's possible that we can remove this in the future
-    transform: {
-      '\\.esm\\.js$': require.resolve('jest-esm-transformer'),
-      '\\.(js|jsx|ts|tsx)$': [
-        require.resolve('ts-jest'),
-        { isolatedModules: true },
-      ],
-      '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve(
-        './jestFileTransform.js',
-      ),
-    },
-
-    // A bit more opinionated
-    testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'],
-
-    // Default behaviour is to not apply transforms for node_modules, but we still want
-    // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages.
-    // The @kyma-project/asyncapi-react library needs to be transformed.
-    transformIgnorePatterns: [
-      '/node_modules/(?!@kyma-project/asyncapi-react/)(?!.*\\.(?:esm\\.js|bmp|gif|jpg|jpeg|png|frag|xml|svg)$)',
-    ],
-  };
-
-  // Use src/setupTests.ts as the default location for configuring test env
-  if (fs.existsSync('src/setupTests.ts')) {
-    options.setupFilesAfterEnv = ['<rootDir>/setupTests.ts'];
-  }
-
  // We read all "jest" config fields in package.json files all the way to the filesystem root.
  // All configs are merged together to create the final config, with longer paths taking precedence.
  // The merging of the configs is shallow, meaning e.g. all transforms are replaced if new ones are defined.
@@ -92,6 +55,56 @@ async function getConfig() {
    currentPath = newPath;
  }

+  // We add an additional Jest config parameter only known by the Backstage CLI
+  // called `transformModules`. It's a list of modules that we want to apply
+  // our configured jest transformations for.
+  // This is useful when packages are published in untranspiled ESM or TS form.
+  const transformModules = pkgJsonConfigs
+    .flatMap(conf => {
+      const modules = conf.transformModules || [];
+      delete conf.transformModules;
+      return modules;
+    })
+    .map(name => `${name}/`)
+    .join('|');
+  const transformModulePattern = transformModules && `(?!${transformModules})`;
+
+  const options = {
+    rootDir: path.resolve('src'),
+    coverageDirectory: path.resolve('coverage'),
+    collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'],
+    moduleNameMapper: {
+      '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
+    },
+
+    // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed
+    // TODO: jest is working on module support, it's possible that we can remove this in the future
+    transform: {
+      '\\.esm\\.js$': require.resolve('jest-esm-transformer'),
+      '\\.(js|jsx|ts|tsx)$': [
+        require.resolve('ts-jest'),
+        { isolatedModules: true },
+      ],
+      '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve(
+        './jestFileTransform.js',
+      ),
+    },
+
+    // A bit more opinionated
+    testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'],
+
+    // Default behaviour is to not apply transforms for node_modules, but we still want
+    // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages.
+    transformIgnorePatterns: [
+      `/node_modules/${transformModulePattern}(?:(?!\\.esm).)*\\.(?:js|json)$`,
+    ],
+  };
+
+  // Use src/setupTests.ts as the default location for configuring test env
+  if (fs.existsSync('src/setupTests.ts')) {
+    options.setupFilesAfterEnv = ['<rootDir>/setupTests.ts'];
+  }
+
  return Object.assign(options, ...pkgJsonConfigs);
 }

@@ -42,5 +42,10 @@
    "*.{json,md}": [
      "prettier --write"
    ]
+  },
+  "jest": {
+    "transformModules": [
+      "@kyma-project/asyncapi-react"
+    ]
  }
 }
@@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
 import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
+import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
 import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
 import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
@@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader {
      new GitlabApiReaderProcessor(),
      new GitlabReaderProcessor(),
      new BitbucketApiReaderProcessor(),
+      new AzureApiReaderProcessor(),
      new UrlReaderProcessor(),
      new YamlProcessor(),
      new EntityPolicyProcessor(entityPolicy),
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
+
+describe('AzureApiReaderProcessor', () => {
+  it('should build raw api', () => {
+    const processor = new AzureApiReaderProcessor();
+    const tests = [
+      {
+        target:
+          'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+        url: new URL(
+          'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1',
+        ),
+        err: undefined,
+      },
+      {
+        target: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+        url: null,
+        err:
+          'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
+      },
+      {
+        target: 'com/a/b/blob/master/path/to/c.yaml',
+        url: null,
+        err:
+          'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+      },
+    ];
+
+    for (const test of tests) {
+      if (test.err) {
+        expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err);
+      } else if (test.url) {
+        expect(processor.buildRawUrl(test.target).toString()).toEqual(
+          test.url.toString(),
+        );
+      } else {
+        throw new Error(
+          'This should not have happened. Either err or url should have matched.',
+        );
+      }
+    }
+  });
+
+  it('should return request options', () => {
+    const tests = [
+      {
+        token: '0123456789',
+        expect: {
+          headers: {
+            Authorization: 'Basic OjAxMjM0NTY3ODk=',
+          },
+        },
+      },
+      {
+        token: '',
+        expect: {
+          headers: {},
+        },
+      },
+    ];
+
+    for (const test of tests) {
+      process.env.AZURE_PRIVATE_TOKEN = test.token;
+      const processor = new AzureApiReaderProcessor();
+      expect(processor.getRequestOptions()).toEqual(test.expect);
+    }
+  });
+});
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import fetch, { RequestInit, HeadersInit } from 'node-fetch';
+import * as result from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+
+export class AzureApiReaderProcessor implements LocationProcessor {
+  private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || '';
+
+  getRequestOptions(): RequestInit {
+    const headers: HeadersInit = {};
+
+    if (this.privateToken !== '') {
+      headers.Authorization = `Basic ${Buffer.from(
+        `:${this.privateToken}`,
+        'utf8',
+      ).toString('base64')}`;
+    }
+
+    const requestOptions: RequestInit = {
+      headers,
+    };
+
+    return requestOptions;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'azure/api') {
+      return false;
+    }
+
+    try {
+      const url = this.buildRawUrl(location.target);
+
+      const response = await fetch(url.toString(), this.getRequestOptions());
+
+      if (response.ok) {
+        const data = await response.buffer();
+        emit(result.data(location, data));
+      } else {
+        const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
+        if (response.status === 404) {
+          if (!optional) {
+            emit(result.notFoundError(location, message));
+          }
+        } else {
+          emit(result.generalError(location, message));
+        }
+      }
+    } catch (e) {
+      const message = `Unable to read ${location.type} ${location.target}, ${e}`;
+      emit(result.generalError(location, message));
+    }
+    return true;
+  }
+
+  // Converts
+  // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
+  // to:   https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1
+
+  buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        project,
+        srcKeyword,
+        repoName,
+      ] = url.pathname.split('/');
+
+      const path = url.searchParams.get('path') || '';
+      const ref = url.searchParams.get('version')?.substr(2);
+
+      if (
+        url.hostname !== 'dev.azure.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        project === '' ||
+        srcKeyword !== '_git' ||
+        repoName === '' ||
+        path === '' ||
+        ref === '' ||
+        !path.match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong Azure Devops URL or Invalid file path');
+      }
+
+      // transform to api
+      url.pathname = [
+        empty,
+        userOrOrg,
+        project,
+        '_apis',
+        'sourceProviders',
+        'TfsGit',
+        'filecontents',
+      ].join('/');
+
+      url.search = [
+        `repository=${repoName}`,
+        `commitOrBranch=${ref}`,
+        `path=${path}`,
+        'api-version=6.0-preview.1',
+      ].join('&');
+
+      url.protocol = 'https';
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+}
@@ -51,6 +51,22 @@ describe('YamlProcessor', () => {
    expect(never).not.toBeCalled();
  });

+  it('should process url that contains yaml', async () => {
+    const containsYamlLocationSpec = {
+      type: 'url',
+      target: 'http://example.com/component?path=test.yaml&c=1&d=2',
+    };
+
+    const buffer = Buffer.from([]);
+    const emit = jest.fn();
+
+    expect(
+      await processor.parseData(buffer, containsYamlLocationSpec, emit),
+    ).toBe(true);
+
+    expect(emit).toBeCalled();
+  });
+
  it('should process entity with yaml', async () => {
    const entity = {
      apiVersion: 'backstage.io/v1alpha1',
@@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor {
    location: LocationSpec,
    emit: LocationProcessorEmit,
  ): Promise<boolean> {
-    if (!location.target.match(/\.ya?ml$/)) {
+    if (!location.target.match(/\.ya?ml/)) {
      return false;
    }

@@ -54,7 +54,7 @@
    "jest-fetch-mock": "^3.0.3",
    "msw": "^0.20.5",
    "react-test-renderer": "^16.13.1",
-    "whatwg-fetch": "^2.0.0"
+    "whatwg-fetch": "^3.4.0"
  },
  "files": [
    "dist"
@@ -27,7 +27,7 @@
    "express": "^4.17.1",
    "express-promise-router": "^3.0.3",
    "graphql": "^15.3.0",
-    "whatwg-fetch": "^2.0.0",
+    "whatwg-fetch": "^3.4.0",
    "winston": "^3.2.1",
    "yn": "^4.0.0"
  },
@@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => {
    const { rendered } = setup();
    expect(
      await rendered.findByText(
-        'Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo.',
+        'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.',
      ),
    ).toBeInTheDocument();

@@ -71,7 +71,7 @@ const RegisterComponentForm: FC<Props> = ({ onSubmit, submitting }) => {
          name="componentLocation"
          required
          margin="normal"
-          helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo."
+          helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config."
          inputRef={register({
            required: true,
            validate: ComponentIdValidators,
@@ -79,7 +79,16 @@ const RegisterComponentPage: FC<{}> = () => {
    setFormState(FormStates.Submitting);
    const { componentLocation: target } = formData;
    try {
-      const data = await catalogApi.addLocation('github', target);
+      const typeMapping = [
+        { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' },
+        { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' },
+        { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' },
+        { url: /.*/, type: 'github' },
+      ];
+
+      const type = typeMapping.filter(item => item.url.test(target))[0].type;
+
+      const data = await catalogApi.addLocation(type, target);

      if (!isMounted()) return;

@@ -31,11 +31,12 @@ describe('ComponentIdValidators', () => {
    });
  });
  describe('yamlValidator', () => {
-    const errorMessage = "Must end with '.yaml'.";
+    const errorMessage = "Must contain '.yaml'.";
    test.each([
      [true, '.yaml'],
      [true, 'http://example.com/blob/master/service.yaml'],
      [true, 'https://example.yaml'],
+      [true, 'https://example.com?path=abc.yaml&c=1'],
      [errorMessage, '.yml'],
      [errorMessage, 'http://example.com/blob/master/service'],
      [errorMessage, undefined],
@@ -19,6 +19,6 @@ export const ComponentIdValidators = {
    (typeof value === 'string' && value.match(/^https:\/\//) !== null) ||
    'Must start with https://.',
  yamlValidator: (value: any) =>
-    (typeof value === 'string' && value.match(/.yaml$/) !== null) ||
-    "Must end with '.yaml'.",
+    (typeof value === 'string' && value.match(/.yaml/) !== null) ||
+    "Must contain '.yaml'.",
 };
@@ -22726,15 +22726,15 @@ whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3, whatwg-encoding@^1.0.5:
  dependencies:
    iconv-lite "0.4.24"

-whatwg-fetch@^2.0.0, whatwg-fetch@^2.0.4:
+whatwg-fetch@^2.0.4:
  version "2.0.4"
  resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f"
  integrity sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==

-whatwg-fetch@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb"
-  integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==
+whatwg-fetch@^3.0.0, whatwg-fetch@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.0.tgz#e11de14f4878f773fbebcde8871b2c0699af8b30"
+  integrity sha512-rsum2ulz2iuZH08mJkT0Yi6JnKhwdw4oeyMjokgxd+mmqYSd9cPpOQf01TIWgjxG/U4+QR+AwKq6lSbXVxkyoQ==

 whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0:
  version "2.3.0"