Create the packages/catalog-model package

plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts

@@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise<any> {
   return knex.schema.createTable('location_update_log', table => {
     table.uuid('id').primary();
     table.enum('status', ['success', 'fail']).notNullable();
-    table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
+    table
+      .dateTime('created_at')
+      .defaultTo(knex.fn.now())
+      .notNullable();
     table.string('message');
     table
       .uuid('location_id')

plugins/catalog-backend/src/ingestion/IngestionModels.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model';
+import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types';
+import { LocationReader, LocationReaders } from './source';
+import { IngestionModel } from './types';
+import { DescriptorParsers } from './descriptor';
+
+export class IngestionModels implements IngestionModel {
+  private readonly reader: LocationReader;
+  private readonly parser: DescriptorParser;
+  private readonly entityPolicy: EntityPolicy;
+
+  static default(): IngestionModel {
+    return new IngestionModels(
+      new LocationReaders(),
+      new DescriptorParsers(),
+      new EntityPolicies(),
+    );
+  }
+
+  constructor(
+    reader: LocationReader,
+    parser: DescriptorParser,
+    entityPolicy: EntityPolicy,
+  ) {
+    this.reader = reader;
+    this.parser = parser;
+    this.entityPolicy = entityPolicy;
+  }
+
+  async readLocation(type: string, target: string): Promise<ReaderOutput[]> {
+    const buffer = await this.reader.tryRead(type, target);
+    if (!buffer) {
+      throw new Error(`No reader could handle location ${type} ${target}`);
+    }
+
+    const items = await this.parser.tryParse(buffer);
+    if (!items) {
+      throw new Error(`No parser could handle location ${type} ${target}`);
+    }
+
+    const result: ReaderOutput[] = [];
+    for (const item of items) {
+      if (item.type === 'error') {
+        result.push(item);
+      } else {
+        try {
+          const output = await this.entityPolicy.apply(item.data);
+          result.push({ type: 'data', data: output });
+        } catch (e) {
+          result.push({ type: 'error', error: e });
+        }
+      }
+    }
+
+    return result;
+  }
+}

plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DescriptorParser, ReaderOutput } from './parsers/types';
+import { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
+
+/**
+ * Parses raw descriptor data (e.g. from a file or stream) into entities.
+ */
+export class DescriptorParsers implements DescriptorParser {
+  private readonly parsers: DescriptorParser[];
+
+  static defaultParsers(): DescriptorParser[] {
+    return [new YamlDescriptorParser()];
+  }
+
+  constructor(
+    parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(),
+  ) {
+    this.parsers = parsers;
+  }
+
+  async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
+    for (const parser of this.parsers) {
+      const result = await parser.tryParse(data);
+      if (result) {
+        return result;
+      }
+    }
+    throw new Error(`Unsupported descriptor format`);
+  }
+}

plugins/catalog-backend/src/ingestion/descriptor/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { DescriptorParsers } from './DescriptorParsers';
+export { YamlDescriptorParser } from './parsers/YamlDescriptorParser';

plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import yaml from 'yaml';
+import { DescriptorParser, ReaderOutput } from './types';
+
+/**
+ * Parses descriptors on YAML format
+ */
+export class YamlDescriptorParser implements DescriptorParser {
+  async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
+    // TODO(freben): Should perhaps first do format detection, so the parse
+    // failure can be emitted as a proper error instead of just as if we
+    // weren't handling the format at all.
+    let documents;
+    try {
+      documents = yaml.parseAllDocuments(data.toString('utf8'));
+    } catch (e) {
+      return undefined;
+    }
+
+    const result: ReaderOutput[] = [];
+
+    for (const document of documents) {
+      if (document.contents) {
+        if (document.errors?.length) {
+          result.push({
+            type: 'error',
+            error: new Error(`Malformed YAML document, ${document.errors[0]}`),
+          });
+        } else {
+          const json = document.toJSON();
+          if (typeof json !== 'object' || Array.isArray(json)) {
+            result.push({
+              type: 'error',
+              error: new Error(`Malformed descriptor, expected object at root`),
+            });
+          } else {
+            result.push({
+              type: 'data',
+              data: json as Entity,
+            });
+          }
+        }
+      }
+    }
+
+    return result;
+  }
+}

plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+
+export type ReaderOutput =
+  | { type: 'error'; error: Error }
+  | { type: 'data'; data: Entity };
+
+/**
+ * Parses raw descriptor data (e.g. from a file) into entities.
+ */
+export type DescriptorParser = {
+  /**
+   * Try to parse some raw data into an entity.
+   *
+   * Note that this is only the low level operation of parsing the raw file
+   * format, e.g. reading JSON or YAML or similar and emitting as structured
+   * but unvalidated data. The actual validation is performed by EntityPolicy
+   * and KindParser.
+   *
+   * @param data Raw descriptor data
+   * @returns A list of raw unvalidated entities / errors, or undefined if the
+   *          given data is not meant to be handled by this parser
+   * @throws An Error if the format was handled and found to not be properly
+   *         formed
+   */
+  tryParse(data: Buffer): Promise<ReaderOutput[] | undefined>;
+};

plugins/catalog-backend/src/ingestion/source/LocationReaders.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FileLocationReader } from './readers/FileLocationReader';
+import { GitHubLocationReader } from './readers/GitHubLocationReader';
+import { LocationReader } from './readers/types';
+
+export class LocationReaders implements LocationReader {
+  private readonly readers: LocationReader[];
+
+  static defaultReaders(): LocationReader[] {
+    return [new FileLocationReader(), new GitHubLocationReader()];
+  }
+
+  constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) {
+    this.readers = readers;
+  }
+
+  async tryRead(type: string, target: string): Promise<Buffer | undefined> {
+    for (const reader of this.readers) {
+      const result = await reader.tryRead(type, target);
+      if (result) {
+        return result;
+      }
+    }
+    throw new Error(`Could not read unknown location "${type}", "${target}"`);
+  }
+}

plugins/catalog-backend/src/ingestion/source/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { LocationReaders } from './LocationReaders';
+export { FileLocationReader } from './readers/FileLocationReader';
+export { GitHubLocationReader } from './readers/GitHubLocationReader';
+export { LocationReader } from './readers/types';

plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { LocationReader } from './types';
+
+/**
+ * Reads a file from the local file system.
+ */
+export class FileLocationReader implements LocationReader {
+  async tryRead(type: string, target: string): Promise<Buffer | undefined> {
+    if (type !== 'file') {
+      return undefined;
+    }
+
+    try {
+      return await fs.readFile(target);
+    } catch (e) {
+      throw new Error(`Unable to read "${target}", ${e}`);
+    }
+  }
+}

plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('node-fetch');
+
+import fetch from 'node-fetch';
+import { GitHubLocationReader } from './GitHubLocationReader';
+
+const { Response } = jest.requireActual('node-fetch');
+
+describe('Unit: GitHubLocationReader', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('fetches the file and parses it correctly', async () => {
+    (fetch as any).mockResolvedValueOnce(new Response('hello'));
+
+    const reader = new GitHubLocationReader();
+    const buffer = await reader.tryRead(
+      'github',
+      'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
+    );
+
+    expect(buffer?.toString('utf8')).toBe('hello');
+  });
+
+  it('changes the url to point to https://raw.githubusercontent.com', async () => {
+    const gitHubUrl = `https://github.com`;
+    const project = `spotify/backstage`;
+    const folderPath = `master/plugins/catalog-backend/fixtures`;
+    const componentFilename = `one_component.yaml`;
+    const rawGitHubUrl = `https://raw.githubusercontent.com`;
+
+    const reader = new GitHubLocationReader();
+    (fetch as any).mockResolvedValueOnce(new Response('hello'));
+
+    await reader.tryRead(
+      'github',
+      `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
+    );
+
+    expect(fetch).toHaveBeenCalledWith(
+      `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`,
+    );
+  });
+
+  describe('rejects wrong urls', () => {
+    const reader = new GitHubLocationReader();
+
+    it.each([
+      ['http://example.com/one_component.yaml'],
+      ['http://github.com/one_component.yaml'],
+      ['http://github.com/PROJECT/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.yaml'],
+      ['http://github.com/PROJECT/REPO/one_component.json'],
+    ])(
+      '%p',
+      async (url: string) =>
+        await expect(reader.tryRead('github', url)).rejects.toThrow(/url/),
+    );
+  });
+});
+
+describe('Integration: GitHubLocationSource', () => {
+  beforeAll(() => {
+    (fetch as any).mockImplementation(jest.requireActual('node-fetch'));
+  });
+
+  it('fetches the fixture from backstage repo', async () => {
+    const PERMANENT_LINK =
+      'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
+
+    const reader = new GitHubLocationReader();
+    const result = await reader.tryRead('github', PERMANENT_LINK);
+
+    expect(result?.toString('utf8')).toContain('component3');
+  });
+});

plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import { URL } from 'url';
+import { LocationReader } from './types';
+
+/**
+ * Reads a file whose target is a GitHub URL.
+ *
+ * Uses raw.githubusercontent.com for now, but this will probably change in the
+ * future when token auth is implemented.
+ */
+export class GitHubLocationReader implements LocationReader {
+  async tryRead(type: string, target: string): Promise<Buffer | undefined> {
+    if (type !== 'github') {
+      return undefined;
+    }
+
+    const url = this.buildRawUrl(target);
+    try {
+      return await fetch(url.toString()).then(x => x.buffer());
+    } catch (e) {
+      throw new Error(`Unable to read "${target}", ${e}`);
+    }
+  }
+
+  private buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        blobKeyword,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        url.hostname !== 'github.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        blobKeyword !== 'blob' ||
+        !restOfPath.join('/').match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong GitHub URL');
+      }
+
+      // Removing the "blob" part
+      url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
+      url.hostname = 'raw.githubusercontent.com';
+      url.protocol = 'https';
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+}

plugins/catalog-backend/src/ingestion/source/readers/types.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type LocationReader = {
+  /**
+   * Reads the contents of a single location.
+   *
+   * @param type The type of location to read
+   * @param target The location target (type-specific)
+   * @returns The target contents, as a raw Buffer, or undefined if this type
+   *          was not meant to be consumed by this reader
+   * @throws An error if the type was meant for this reader, but could not be
+   *         read
+   */
+  tryRead(type: string, target: string): Promise<Buffer | undefined>;
+};