diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 51d7ca98d7..5c2c79d427 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -15,7 +15,7 @@ */ import { Database } from '../database'; -import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; +import { DescriptorEnvelope } from '../ingestion/types'; import { EntitiesCatalog } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 8c7ccb4bd1..d74487ab2c 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -15,8 +15,8 @@ */ import { NotFoundError } from '@backstage/backend-common'; +import { DescriptorEnvelope } from '../ingestion'; import { EntitiesCatalog } from './types'; -import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; export class StaticEntitiesCatalog implements EntitiesCatalog { private _entities: DescriptorEnvelope[]; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 3c248f59e4..bb50a00783 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,7 +15,7 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; +import { DescriptorEnvelope } from '../ingestion'; // // Items diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index f0419df6d0..71530c3965 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -17,7 +17,7 @@ import { InputError, NotFoundError } from '@backstage/backend-common'; import Knex from 'knex'; import { v4 as uuidv4 } from 'uuid'; -import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; +import { DescriptorEnvelope } from '../ingestion'; import { AddDatabaseLocation, DatabaseLocationUpdateLogEvent, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 33221a3fa6..0a199fa7d6 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -15,7 +15,7 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; +import { DescriptorEnvelope } from '../ingestion'; export type DbEntitiesRow = { id: string; diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts index e60a9a54c0..86283df666 100644 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts @@ -16,12 +16,13 @@ import { makeValidator } from '../validation'; import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; +import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; import { DescriptorEnvelope, - DescriptorEnvelopeParser, -} from './descriptors/DescriptorEnvelopeParser'; -import { KindParser } from './descriptors/types'; -import { DescriptorParser, ParserError } from './types'; + DescriptorParser, + KindParser, + ParserError, +} from './types'; export class DescriptorParsers implements DescriptorParser { static create(): DescriptorParser { diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts index 39b294f9de..8d756ac010 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts @@ -15,14 +15,10 @@ */ import * as yup from 'yup'; -import { ParserError } from '../types'; -import { DescriptorEnvelope } from './DescriptorEnvelopeParser'; +import { DescriptorEnvelope, ParserError } from '../types'; import { KindParser } from './types'; export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope { - metadata: { - name: string; - }; spec: { type: string; }; diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts index 1398e08ce2..7c96fb7cf5 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts @@ -27,6 +27,9 @@ describe('DescriptorEnvelopeParser', () => { apiVersion: backstage.io/v1beta1 kind: Component metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 name: my-component-yay namespace: the-namespace labels: @@ -77,6 +80,36 @@ describe('DescriptorEnvelopeParser', () => { await expect(parser.parse(data)).rejects.toThrow(/metadata/); }); + it('accepts missing uid', async () => { + delete data.metadata.uid; + await expect(parser.parse(data)).resolves.toBe(data); + }); + + it('rejects bad uid', async () => { + data.metadata.uid = 7; + await expect(parser.parse(data)).rejects.toThrow(/uid/); + }); + + it('accepts missing etag', async () => { + delete data.metadata.etag; + await expect(parser.parse(data)).resolves.toBe(data); + }); + + it('rejects bad etag', async () => { + data.metadata.etag = 7; + await expect(parser.parse(data)).rejects.toThrow(/etag/); + }); + + it('accepts missing generation', async () => { + delete data.metadata.generation; + await expect(parser.parse(data)).resolves.toBe(data); + }); + + it('rejects bad generation', async () => { + data.metadata.generation = 'a'; + await expect(parser.parse(data)).rejects.toThrow(/generation/); + }); + it('accepts missing spec', async () => { delete data.spec; await expect(parser.parse(data)).resolves.toBe(data); diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts index 443b20a02a..ccf5fc3051 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts @@ -16,21 +16,7 @@ import * as yup from 'yup'; import { Validators } from '../../validation'; - -/** - * The format envelope that's common to all versions/kinds - */ -export type DescriptorEnvelope = { - apiVersion: string; - kind: string; - metadata?: { - name?: string; - namespace?: string; - labels?: object; - annotations?: object; - }; - spec?: object; -}; +import { DescriptorEnvelope } from '../types'; /** * Parses some raw structured data as a descriptor envelope @@ -57,6 +43,33 @@ export class DescriptorEnvelopeParser { validators.isValidKind, ); + const uidSchema = yup + .string() + .notRequired() + .test( + 'metadata.uid', + 'The uid is not formatted according to schema', + value => value === undefined || value.length > 0, + ); + + const etagSchema = yup + .string() + .notRequired() + .test( + 'metadata.etag', + 'The etag value is not according to schema', + value => value === undefined || value.length > 0, + ); + + const generationSchema = yup + .number() + .notRequired() + .test( + 'metadata.generation', + 'The generation value is not according to schema', + value => value === undefined || value > 0, + ); + const nameSchema = yup .string() .notRequired() @@ -125,6 +138,9 @@ export class DescriptorEnvelopeParser { const metadataSchema = yup .object({ + uid: uidSchema, + etag: etagSchema, + generation: generationSchema, name: nameSchema, namespace: namespaceSchema, labels: labelsSchema, @@ -158,6 +174,9 @@ export class DescriptorEnvelopeParser { const reservedKeys = [ 'apiVersion', 'kind', + 'uid', + 'etag', + 'generation', 'name', 'namespace', 'labels', diff --git a/plugins/catalog-backend/src/ingestion/descriptors/types.ts b/plugins/catalog-backend/src/ingestion/descriptors/types.ts index 3249d21c27..92b5fdab11 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/types.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/types.ts @@ -14,18 +14,21 @@ * limitations under the License. */ -import { DescriptorEnvelope } from './DescriptorEnvelopeParser'; +import { DescriptorEnvelope } from '../types'; +/** + * Parses and validates a single envelope into its materialized kind. + * + * These parsers may assume that the envelope is already validated and well + * formed. + */ export type KindParser = { /** - * Parses and validates a single envelope into its materialized kind. - * - * These parsers may assume that the envelope is already validated and - * well formed. + * Try to parse an envelope into a materialized kind. * * @param envelope A valid descriptor envelope * @returns A materialized type, or undefined if the given version/kind is - * not handled by this parser + * not meant to be handled by this parser * @throws An Error if the type was handled and found to not be properly * formatted */ diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index acd9d1d5fb..52817737ac 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -15,10 +15,101 @@ */ import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser'; -import { DescriptorEnvelope } from './descriptors/DescriptorEnvelopeParser'; export type ComponentDescriptor = ComponentDescriptorV1beta1; +/** + * The format envelope that's common to all versions/kinds. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type DescriptorEnvelope = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Optional metadata related to the entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + */ + metadata?: { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace, for any given kind. + */ + name?: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: object; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: object; + }; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Parses and validates descriptors. + * + * The output must be validated and well formed. + */ export type DescriptorParser = { /** * Parses and validates a single raw descriptor. @@ -30,6 +121,27 @@ export type DescriptorParser = { parse(descriptor: object): Promise; }; +/** + * Parses and validates a single envelope into its materialized kind. + * + * These parsers may assume that the envelope is already validated and well + * formed. + */ +export type KindParser = { + /** + * Try to parse an envelope into a materialized kind. + * + * @param envelope A valid descriptor envelope + * @returns A materialized type, or undefined if the given version/kind is + * not meant to be handled by this parser + * @throws An Error if the type was handled and found to not be properly + * formatted + */ + tryParse( + envelope: DescriptorEnvelope, + ): Promise; +}; + export class ParserError extends Error { constructor(message?: string, private _entityName?: string | undefined) { super(message); @@ -55,3 +167,14 @@ export type LocationReader = { */ read(type: string, target: string): Promise; }; + +export type LocationSource = { + /** + * Reads the contents of a single location. + * + * @param target The location target to read + * @returns The parsed contents, as an array of unverified descriptors + * @throws An error if the location target could not be read + */ + read(target: string): Promise; +}; diff --git a/yarn.lock b/yarn.lock index 73a7b17bb4..8a2c895638 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5370,7 +5370,7 @@ aws4@^1.8.0: resolved "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz#7e33d8f7d449b3f673cd72deb9abdc552dbe528e" integrity sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug== -axios@^0.19.0: +axios@^0.19.0, axios@^0.19.2: version "0.19.2" resolved "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==