diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 617c29105b..7a8886adc3 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -34,6 +34,7 @@ "@types/http-errors": "^1.6.3", "@types/morgan": "^1.9.0", "@types/supertest": "^2.0.8", + "@types/yaml": "^1.9.7", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^25.1.0", diff --git a/plugins/catalog-backend/fixtures/component1.yaml b/plugins/catalog-backend/fixtures/component1.yaml index 8db5cfc851..b0b54e0902 100644 --- a/plugins/catalog-backend/fixtures/component1.yaml +++ b/plugins/catalog-backend/fixtures/component1.yaml @@ -1 +1,6 @@ -name: Component1 +apiVersion: catalog.backstage.io/v1 +kind: Component +metadata: + name: component1 +spec: + type: service diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts index d28278c496..51cccf2286 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts @@ -17,42 +17,10 @@ import { NotFoundError } from '@backstage/backend-common'; import Knex from 'knex'; import path from 'path'; -import fs from 'fs-extra'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; -import { - AddLocationRequest, - Catalog, - Component, - Location, - componentShape, -} from './types'; -import YAML from 'yaml'; - -async function readLocation(location: Location): Promise { - switch (location.type) { - case 'file': { - let parsed; - try { - const raw = await fs.readFile(location.target, 'utf8'); - parsed = YAML.parse(raw); - } catch (e) { - throw new Error(`Unable to read "${location.target}", ${e}`); - } - - try { - return [await componentShape.validate(parsed, { strict: true })]; - } catch (e) { - throw new Error( - `Malformed file contents at "${location.target}", ${e}`, - ); - } - } - - default: - throw new Error(`Unknown type "${location.type}"`); - } -} +import { readLocation } from '../ingestion'; +import { AddLocationRequest, Catalog, Component, Location } from './types'; export class DatabaseCatalog implements Catalog { static async create( diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index abac94345d..e91c5f8267 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -20,12 +20,6 @@ export type Component = { name: string; }; -export const componentShape: yup.Schema = yup - .object({ - name: yup.string().required(), - }) - .unknown(); - export type Location = { id: string; type: string; diff --git a/plugins/catalog-backend/src/descriptors/component.ts b/plugins/catalog-backend/src/descriptors/component.ts new file mode 100644 index 0000000000..cb0c30aa61 --- /dev/null +++ b/plugins/catalog-backend/src/descriptors/component.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { DescriptorEnvelope } from './envelope'; +import { Component } from '../catalog/types'; + +export type ComponentDescriptor = { + metadata: { + name: string; + }; + spec: { + type: string; + }; +}; + +const componentDescriptorSchema: yup.Schema = yup.object({ + metadata: yup.object({ + name: yup.string().required(), + }), + spec: yup.object({ + type: yup.string().required(), + }), +}); + +export async function parseComponentDescriptor( + envelope: DescriptorEnvelope, +): Promise { + let componentDescriptor; + try { + componentDescriptor = await componentDescriptorSchema.validate(envelope, { + strict: true, + }); + } catch (e) { + throw new Error(`Malformed component, ${e}`); + } + + const component: Component = { + name: componentDescriptor.metadata.name, + }; + + return [component]; +} diff --git a/plugins/catalog-backend/src/descriptors/envelope.ts b/plugins/catalog-backend/src/descriptors/envelope.ts new file mode 100644 index 0000000000..dd78d860ee --- /dev/null +++ b/plugins/catalog-backend/src/descriptors/envelope.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import * as yup from 'yup'; + +export type DescriptorEnvelope = { + apiVersion: string; + kind: string; + metadata?: object; + spec?: object; +}; + +const descriptorEnvelopeSchema: yup.Schema = yup + .object({ + apiVersion: yup.string().required(), + kind: yup.string().required(), + metadata: yup.object(), + spec: yup.object(), + }) + .noUnknown(); + +export async function parseDescriptorEnvelope( + rawYaml: string, +): Promise { + let descriptor; + try { + descriptor = yaml.parse(rawYaml); + } catch (e) { + throw new Error(`Malformed YAML, ${e}`); + } + + try { + return await descriptorEnvelopeSchema.validate(descriptor, { + strict: true, + }); + } catch (e) { + throw new Error(`Malformed envelope, ${e}`); + } +} diff --git a/plugins/catalog-backend/src/descriptors/index.ts b/plugins/catalog-backend/src/descriptors/index.ts new file mode 100644 index 0000000000..db3fd4c938 --- /dev/null +++ b/plugins/catalog-backend/src/descriptors/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './component'; +export * from './envelope'; +export * from './util'; diff --git a/plugins/catalog-backend/src/descriptors/util.ts b/plugins/catalog-backend/src/descriptors/util.ts new file mode 100644 index 0000000000..e7b4944215 --- /dev/null +++ b/plugins/catalog-backend/src/descriptors/util.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component } from '../catalog/types'; +import { parseComponentDescriptor } from './component'; +import { parseDescriptorEnvelope } from './envelope'; + +// TODO(freben): Temporary helper that ignores the kind +export async function parseDescriptor(rawYaml: string): Promise { + const env = await parseDescriptorEnvelope(rawYaml); + return await parseComponentDescriptor(env); +} diff --git a/plugins/catalog-backend/src/ingestion/fileLocation.ts b/plugins/catalog-backend/src/ingestion/fileLocation.ts new file mode 100644 index 0000000000..524721035e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/fileLocation.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { parseDescriptor } from '../descriptors'; +import { Component } from '../catalog/types'; + +export async function readFileLocation(target: string): Promise { + let rawYaml; + try { + rawYaml = await fs.readFile(target, 'utf8'); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + + try { + return parseDescriptor(rawYaml); + } catch (e) { + throw new Error(`Malformed descriptor at "${target}", ${e}`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts new file mode 100644 index 0000000000..775fe29f84 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './fileLocation'; +export * from './util'; diff --git a/plugins/catalog-backend/src/ingestion/util.ts b/plugins/catalog-backend/src/ingestion/util.ts new file mode 100644 index 0000000000..91b7ad2678 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/util.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Location } from '../catalog/types'; +import { readFileLocation } from './fileLocation'; + +export async function readLocation(location: Location): Promise { + switch (location.type) { + case 'file': + return await readFileLocation(location.target); + default: + throw new Error(`Unknown type "${location.type}"`); + } +} diff --git a/yarn.lock b/yarn.lock index 4168fce334..d312e5b03b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4602,6 +4602,13 @@ "@types/webpack-sources" "*" source-map "^0.6.0" +"@types/yaml@^1.9.7": + version "1.9.7" + resolved "https://registry.npmjs.org/@types/yaml/-/yaml-1.9.7.tgz#2331f36e0aac91311a63d33eb026c21687729679" + integrity sha512-8WMXRDD1D+wCohjfslHDgICd2JtMATZU8CkhH8LVJqcJs6dyYj5TGptzP8wApbmEullGBSsCEzzap73DQ1HJaA== + dependencies: + yaml "*" + "@types/yargs-parser@*": version "15.0.0" resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" @@ -22129,6 +22136,13 @@ yallist@^4.0.0: resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@*, yaml@^1.9.2: + version "1.9.2" + resolved "https://registry.npmjs.org/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed" + integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg== + dependencies: + "@babel/runtime" "^7.9.2" + yaml@^1.7.2: version "1.8.3" resolved "https://registry.npmjs.org/yaml/-/yaml-1.8.3.tgz#2f420fca58b68ce3a332d0ca64be1d191dd3f87a" @@ -22136,13 +22150,6 @@ yaml@^1.7.2: dependencies: "@babel/runtime" "^7.8.7" -yaml@^1.9.2: - version "1.9.2" - resolved "https://registry.npmjs.org/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed" - integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg== - dependencies: - "@babel/runtime" "^7.9.2" - yargs-parser@^10.0.0: version "10.1.0" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"