Rearrange ingestion and descriptor parsing

plugins/catalog-backend/fixtures/component1.yaml

@@ -1 +1,6 @@
-name: Component1
+apiVersion: catalog.backstage.io/v1
+kind: Component
+metadata:
+  name: component1
+spec:
+  type: service

plugins/catalog-backend/src/catalog/DatabaseCatalog.ts

@@ -17,42 +17,10 @@
 import { NotFoundError } from '@backstage/backend-common';
 import Knex from 'knex';
 import path from 'path';
-import fs from 'fs-extra';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
-import {
-  AddLocationRequest,
-  Catalog,
-  Component,
-  Location,
-  componentShape,
-} from './types';
-import YAML from 'yaml';
-
-async function readLocation(location: Location): Promise<Component[]> {
-  switch (location.type) {
-    case 'file': {
-      let parsed;
-      try {
-        const raw = await fs.readFile(location.target, 'utf8');
-        parsed = YAML.parse(raw);
-      } catch (e) {
-        throw new Error(`Unable to read "${location.target}", ${e}`);
-      }
-
-      try {
-        return [await componentShape.validate(parsed, { strict: true })];
-      } catch (e) {
-        throw new Error(
-          `Malformed file contents at "${location.target}", ${e}`,
-        );
-      }
-    }
-
-    default:
-      throw new Error(`Unknown type "${location.type}"`);
-  }
-}
+import { readLocation } from '../ingestion';
+import { AddLocationRequest, Catalog, Component, Location } from './types';
 
 export class DatabaseCatalog implements Catalog {
   static async create(

plugins/catalog-backend/src/catalog/types.ts

@@ -20,12 +20,6 @@ export type Component = {
   name: string;
 };
 
-export const componentShape: yup.Schema<Component> = yup
-  .object({
-    name: yup.string().required(),
-  })
-  .unknown();
-
 export type Location = {
   id: string;
   type: string;

plugins/catalog-backend/src/descriptors/component.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as yup from 'yup';
+import { DescriptorEnvelope } from './envelope';
+import { Component } from '../catalog/types';
+
+export type ComponentDescriptor = {
+  metadata: {
+    name: string;
+  };
+  spec: {
+    type: string;
+  };
+};
+
+const componentDescriptorSchema: yup.Schema<ComponentDescriptor> = yup.object({
+  metadata: yup.object({
+    name: yup.string().required(),
+  }),
+  spec: yup.object({
+    type: yup.string().required(),
+  }),
+});
+
+export async function parseComponentDescriptor(
+  envelope: DescriptorEnvelope,
+): Promise<Component[]> {
+  let componentDescriptor;
+  try {
+    componentDescriptor = await componentDescriptorSchema.validate(envelope, {
+      strict: true,
+    });
+  } catch (e) {
+    throw new Error(`Malformed component, ${e}`);
+  }
+
+  const component: Component = {
+    name: componentDescriptor.metadata.name,
+  };
+
+  return [component];
+}

plugins/catalog-backend/src/descriptors/envelope.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yaml from 'yaml';
+import * as yup from 'yup';
+
+export type DescriptorEnvelope = {
+  apiVersion: string;
+  kind: string;
+  metadata?: object;
+  spec?: object;
+};
+
+const descriptorEnvelopeSchema: yup.Schema<DescriptorEnvelope> = yup
+  .object({
+    apiVersion: yup.string().required(),
+    kind: yup.string().required(),
+    metadata: yup.object(),
+    spec: yup.object(),
+  })
+  .noUnknown();
+
+export async function parseDescriptorEnvelope(
+  rawYaml: string,
+): Promise<DescriptorEnvelope> {
+  let descriptor;
+  try {
+    descriptor = yaml.parse(rawYaml);
+  } catch (e) {
+    throw new Error(`Malformed YAML, ${e}`);
+  }
+
+  try {
+    return await descriptorEnvelopeSchema.validate(descriptor, {
+      strict: true,
+    });
+  } catch (e) {
+    throw new Error(`Malformed envelope, ${e}`);
+  }
+}

plugins/catalog-backend/src/descriptors/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './component';
+export * from './envelope';
+export * from './util';

plugins/catalog-backend/src/descriptors/util.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Component } from '../catalog/types';
+import { parseComponentDescriptor } from './component';
+import { parseDescriptorEnvelope } from './envelope';
+
+// TODO(freben): Temporary helper that ignores the kind
+export async function parseDescriptor(rawYaml: string): Promise<Component[]> {
+  const env = await parseDescriptorEnvelope(rawYaml);
+  return await parseComponentDescriptor(env);
+}

plugins/catalog-backend/src/ingestion/fileLocation.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { parseDescriptor } from '../descriptors';
+import { Component } from '../catalog/types';
+
+export async function readFileLocation(target: string): Promise<Component[]> {
+  let rawYaml;
+  try {
+    rawYaml = await fs.readFile(target, 'utf8');
+  } catch (e) {
+    throw new Error(`Unable to read "${target}", ${e}`);
+  }
+
+  try {
+    return parseDescriptor(rawYaml);
+  } catch (e) {
+    throw new Error(`Malformed descriptor at "${target}", ${e}`);
+  }
+}

plugins/catalog-backend/src/ingestion/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './fileLocation';
+export * from './util';

plugins/catalog-backend/src/ingestion/util.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Component, Location } from '../catalog/types';
+import { readFileLocation } from './fileLocation';
+
+export async function readLocation(location: Location): Promise<Component[]> {
+  switch (location.type) {
+    case 'file':
+      return await readFileLocation(location.target);
+    default:
+      throw new Error(`Unknown type "${location.type}"`);
+  }
+}