Merge pull request #939 from spotify/shmidt-i/github-reader

GitHub location reader
2020-05-25 11:21:31 +02:00
parent acade31849 2f01a5be26
commit 7e2b314458
5 changed files with 196 additions and 0 deletions
@@ -16,6 +16,7 @@
  },
  "dependencies": {
    "@backstage/backend-common": "^0.1.1-alpha.6",
+    "@types/node-fetch": "^2.5.7",
    "compression": "^1.7.4",
    "cors": "^2.8.5",
    "express": "^4.17.1",
@@ -25,6 +26,7 @@
    "knex": "^0.21.1",
    "lodash": "^4.17.15",
    "morgan": "^1.10.0",
+    "node-fetch": "^2.6.0",
    "sqlite3": "^4.2.0",
    "uuid": "^8.0.0",
    "winston": "^3.2.1",
@@ -15,6 +15,7 @@
 */

 import { FileLocationSource } from './sources/FileLocationSource';
+import { GitHubLocationSource } from './sources/GitHubLocationSource';
 import { LocationSource } from './sources/types';
 import { LocationReader, ReaderOutput } from './types';

@@ -22,6 +23,7 @@ export class LocationReaders implements LocationReader {
  static create(): LocationReader {
    return new LocationReaders({
      file: new FileLocationSource(),
+      github: new GitHubLocationSource(),
    });
  }

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import { ReaderOutput } from '../types';
+import { LocationSource } from './types';
+import { readDescriptorYaml } from './util';
+import { URL } from 'url';
+
+// Pointing to raw.githubusercontent.com for now
+// to be changed in the future, after auth and tokens are done
+export class GitHubLocationSource implements LocationSource {
+  async read(target: string): Promise<ReaderOutput[]> {
+    let url: URL;
+
+    try {
+      url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        blobKeyword,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        url.hostname !== 'github.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        blobKeyword !== 'blob' ||
+        !restOfPath.join('/').match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong GitHub URL');
+      }
+
+      // Removing the "blob" part
+      url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
+      url.hostname = 'raw.githubusercontent.com';
+      url.protocol = 'https';
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+
+    let rawYaml;
+    try {
+      rawYaml = await fetch(url.toString()).then((x) => {
+        return x.text();
+      });
+    } catch (e) {
+      throw new Error(`Unable to read "${target}", ${e}`);
+    }
+
+    try {
+      return readDescriptorYaml(rawYaml);
+    } catch (e) {
+      throw new Error(`Malformed descriptor at "${target}", ${e}`);
+    }
+  }
+}
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('node-fetch');
+
+import fs from 'fs-extra';
+import fetch from 'node-fetch';
+import path from 'path';
+import { GitHubLocationSource } from '../GitHubLocationSource';
+
+const { Response } = jest.requireActual('node-fetch');
+
+const FIXTURES_DIR = path.resolve(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  '..',
+  'fixtures',
+);
+const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => {
+  acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8');
+  return acc;
+}, {} as Record<string, string>);
+
+describe('Unit: GitHubLocationSource', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('fetches the file and parses it correctly', async () => {
+    (fetch as any).mockReturnValueOnce(
+      Promise.resolve(new Response(fixtures['one_component.yaml'])),
+    );
+    const reader = new GitHubLocationSource();
+
+    const result = await reader.read(
+      'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
+    );
+
+    expect(result[0].type).toBe('data');
+    expect((result[0] as any).data.metadata.name).toBe('component3');
+  });
+
+  it('changes the url to point to https://raw.githubusercontent.com', async () => {
+    const gitHubUrl = `https://github.com`;
+    const project = `spotify/backstage`;
+    const folderPath = `master/plugins/catalog-backend/fixtures`;
+    const componentFilename = `one_component.yaml`;
+    const rawGitHubUrl = `https://raw.githubusercontent.com`;
+    const reader = new GitHubLocationSource();
+    (fetch as any).mockReturnValueOnce(
+      Promise.resolve(new Response(fixtures[componentFilename])),
+    );
+
+    await reader.read(
+      `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
+    );
+
+    expect(fetch).toHaveBeenCalledWith(
+      `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`,
+    );
+  });
+
+  describe('rejects wrong urls', () => {
+    const reader = new GitHubLocationSource();
+
+    it.each([
+      ['http://example.com/one_component.yaml'],
+      ['http://github.com/one_component.yaml'],
+      ['http://github.com/PROJECT/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.json'],
+    ])(
+      '%p',
+      async (url: string) =>
+        await expect(reader.read(url)).rejects.toThrow(/url/),
+    );
+  });
+});
+
+describe('Integration: GitHubLocationSource', () => {
+  beforeAll(() => {
+    (fetch as any).mockImplementation(jest.requireActual('node-fetch'));
+  });
+
+  it('fetches the fixture from backstage repo', async () => {
+    const PERMANENT_LINK =
+      'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
+    const reader = new GitHubLocationSource();
+
+    const result = await reader.read(PERMANENT_LINK);
+
+    expect(result[0].type).toBe('data');
+    expect((result[0] as any).data.metadata.name).toBe('component3');
+  });
+});
@@ -4129,6 +4129,14 @@
  dependencies:
    "@types/express" "*"

+"@types/node-fetch@^2.5.7":
+  version "2.5.7"
+  resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c"
+  integrity sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2":
  version "13.9.2"
  resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"