feat: github location reader

2020-05-20 15:21:24 +02:00
parent 986b230016
commit ca83d97591
4 changed files with 188 additions and 1 deletions
@@ -16,6 +16,7 @@
  },
  "dependencies": {
    "@backstage/backend-common": "^0.1.1-alpha.6",
+    "@types/node-fetch": "^2.5.7",
    "compression": "^1.7.4",
    "cors": "^2.8.5",
    "express": "^4.17.1",
@@ -25,6 +26,7 @@
    "knex": "^0.21.1",
    "lodash": "^4.17.15",
    "morgan": "^1.10.0",
+    "node-fetch": "^2.6.0",
    "sqlite3": "^4.2.0",
    "uuid": "^8.0.0",
    "winston": "^3.2.1",
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import { ReaderOutput } from '../types';
+import { LocationSource } from './types';
+import { readDescriptorYaml } from './util';
+import { URL } from 'url';
+
+// Pointing to raw.githubusercontent.com for now
+// to be changed in the future, after auth and tokens are done
+export class GitHubLocationSource implements LocationSource {
+  async read(target: string): Promise<ReaderOutput[]> {
+    let url: URL;
+
+    try {
+      url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        blobKeyword,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        url.hostname !== 'github.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        blobKeyword !== 'blob' ||
+        !restOfPath.join('/').match(/\.yaml$/)
+      ) {
+        throw new Error();
+      }
+
+      // Removing the "blob" part
+      url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
+      url.hostname = 'raw.githubusercontent.com';
+      url.protocol = 'https';
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+
+    let rawYaml;
+    try {
+      rawYaml = await fetch(url.toString()).then(x => {
+        return x.text();
+      });
+    } catch (e) {
+      throw new Error(`Unable to read "${target}", ${e}`);
+    }
+
+    try {
+      return readDescriptorYaml(rawYaml);
+    } catch (e) {
+      throw new Error(`Malformed descriptor at "${target}", ${e}`);
+    }
+  }
+}
@@ -0,0 +1,94 @@
+jest.mock('node-fetch');
+
+import fs from 'fs-extra';
+import fetch from 'node-fetch';
+import path from 'path';
+import { GitHubLocationSource } from '../GitHubLocationSource';
+
+const { Response } = jest.requireActual('node-fetch');
+
+const fixtures_dir = path.resolve(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  '..',
+  'fixtures',
+);
+const fixtures = fs.readdirSync(fixtures_dir).reduce((acc, filename) => {
+  acc[filename] = fs.readFileSync(path.resolve(fixtures_dir, filename), 'utf8');
+  return acc;
+}, {} as Record<string, string>);
+
+describe('Unit: GitHubLocationSource', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('fetches the file and parses it correctly', async () => {
+    (fetch as any).mockReturnValueOnce(
+      Promise.resolve(new Response(fixtures['one_component.yaml'])),
+    );
+    const reader = new GitHubLocationSource();
+
+    const result = await reader.read(
+      'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
+    );
+
+    expect(result[0].type).toBe('data');
+    expect((result[0] as any).data.metadata.name).toBe('component3');
+  });
+
+  it('changes the url to point to https://raw.githubusercontent.com', async () => {
+    const gitHubUrl = `https://github.com`;
+    const project = `spotify/backstage`;
+    const path = `master/plugins/catalog-backend/fixtures`;
+    const componentFilename = `one_component.yaml`;
+    const rawGitHubUrl = `https://raw.githubusercontent.com`;
+    const reader = new GitHubLocationSource();
+    (fetch as any).mockReturnValueOnce(
+      Promise.resolve(new Response(fixtures[componentFilename])),
+    );
+
+    await reader.read(
+      `${gitHubUrl}/${project}/blob/${path}/${componentFilename}`,
+    );
+
+    expect(fetch).toHaveBeenCalledWith(
+      `${rawGitHubUrl}/${project}/${path}/${componentFilename}`,
+    );
+  });
+
+  describe('rejects wrong urls', () => {
+    const reader = new GitHubLocationSource();
+
+    it.each([
+      ['http://example.com/one_component.yaml'],
+      ['http://github.com/one_component.yaml'],
+      ['http://github.com/PROJECT/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.json'],
+    ])(
+      '%p',
+      async (url: string) =>
+        await expect(reader.read(url)).rejects.toThrow(/url/),
+    );
+  });
+});
+
+describe('Integration: GitHubLocationSource', () => {
+  beforeAll(() => {
+    (fetch as any).mockImplementation(jest.requireActual('node-fetch'));
+  });
+
+  it('fetches the fixture from backstage repo', async () => {
+    const PERMANENT_LINK =
+      'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
+    const reader = new GitHubLocationSource();
+
+    const result = await reader.read(PERMANENT_LINK);
+
+    expect(result[0].type).toBe('data');
+    expect((result[0] as any).data.metadata.name).toBe('component3');
+  });
+});
@@ -4093,6 +4093,14 @@
  dependencies:
    "@types/express" "*"

+"@types/node-fetch@^2.5.7":
+  version "2.5.7"
+  resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c"
+  integrity sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2":
  version "13.9.2"
  resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"
@@ -6823,7 +6831,7 @@ columnify@^1.5.4, columnify@~1.5.4:
    strip-ansi "^3.0.0"
    wcwidth "^1.0.0"

-combined-stream@^1.0.6, combined-stream@~1.0.6:
+combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
  version "1.0.8"
  resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
  integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
@@ -9717,6 +9725,15 @@ form-data@^2.3.1:
    combined-stream "^1.0.6"
    mime-types "^2.1.12"

+form-data@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682"
+  integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 form-data@~2.3.2:
  version "2.3.3"
  resolved "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"