From 68fdbf014ebc768f686606d5b0ffe606ca19b002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 11 May 2021 18:31:37 +0200 Subject: [PATCH] Add the `status` field to the Entity envelope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/mean-taxis-leave.md | 5 ++ .../software-catalog/descriptor-format.md | 49 ++++++++++++++++++ .../software-catalog/extending-the-model.md | 25 +++++++-- .../software-catalog/well-known-statuses.md | 51 +++++++++++++++++++ microsite/sidebars.json | 1 + mkdocs.yml | 1 + packages/catalog-model/api-report.md | 1 + packages/catalog-model/src/entity/Entity.ts | 8 +++ packages/catalog-model/src/entity/util.ts | 8 +-- .../src/schema/Entity.schema.json | 9 ++++ .../src/schema/shared/common.schema.json | 11 ++-- .../catalog-backend/src/next/Stitcher.test.ts | 10 ++++ plugins/catalog-backend/src/next/Stitcher.ts | 17 ++++++- 13 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 .changeset/mean-taxis-leave.md create mode 100644 docs/features/software-catalog/well-known-statuses.md diff --git a/.changeset/mean-taxis-leave.md b/.changeset/mean-taxis-leave.md new file mode 100644 index 0000000000..8c1e5ea3c3 --- /dev/null +++ b/.changeset/mean-taxis-leave.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Add the `status` field to the Entity envelope diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index eb087a9c5f..665c729191 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -23,6 +23,7 @@ we recommend that you name them `catalog-info.yaml`. - [Common to All Kinds: The Envelope](#common-to-all-kinds-the-envelope) - [Common to All Kinds: The Metadata](#common-to-all-kinds-the-metadata) - [Common to All Kinds: Relations](#common-to-all-kinds-relations) +- [Common to All Kinds: Status](#common-to-all-kinds-status) - [Kind: Component](#kind-component) - [Kind: Template](#kind-template) - [Kind: API](#kind-api) @@ -396,6 +397,54 @@ with it (such as the default kind being `Group` if not specified). See the [well-known relations section](well-known-relations.md) for a list of well-known / common relations and their semantics. +## Common to All Kinds: Status + +The `status` root field is a read-only set of statuses, pertaining to the +current state or health of the entity, described in the +[well-known statuses section](well-known-statuses.md). Each status field +contains a specific blob of data that describes some aspect of the state of the +entity, as seen from the point of view of some specific system. Different +systems may contribute to this status object, under their own respective keys. + +The current main use case for this field is for the ingestion processes of the +catalog itself to convey information about failures and warnings back to the +user. + +A status field as part of a single entity that's read out of the API may look as +follows. + +```js +{ + // ... + "status": { + "backstage.io/processing-status": { + "errors": [] + } + }, + "spec": { + // ... + } +} +``` + +The keys of the `status` object are arbitrary strings. We recommend that any +statuses, that are not strictly private within the organization, be namespaced +to avoid collisions. Statuses emitted by Backstage core processes will for +example be prefixed with `backstage.io/` as in the example above. + +The values of the `status` object are currently left unrestricted, except that +they must be objects. We reserve the right to extend this model in the future, +such that some fields of those value objects gain standardized meaning. We may +for example want to add a standard concept of "severity" or "level" to these. + +Entity descriptor YAML files are not supposed to contain this field. Instead, +catalog processors analyze the entity descriptor data and its surroundings, and +deduce status entries that are then attached onto the entity as read from the +catalog. + +See the [well-known statuses section](well-known-statuses.md) for a list of +well-known / common relations and their semantics. + ## Kind: Component Describes the following entity kind: diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index d4dca0c059..4cc1bc1110 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -1,6 +1,7 @@ --- id: extending-the-model title: Extending the model +# prettier-ignore description: Documentation on extending the catalog model --- @@ -301,9 +302,9 @@ Example intents: > "We have this concept of service maintainership, separate from ownership, that > we would like to make relations to individual users for." -> We feel that we want to explicitly model the team-to-global-department mapping -> as a relation, because it is core to our org setup and we frequently query for -> it. +> "We feel that we want to explicitly model the team-to-global-department +> mapping as a relation, because it is core to our org setup and we frequently +> query for it." Any processor can emit relations for entities as they are being processed, and new processors can be added when building the backend catalog using the @@ -349,3 +350,21 @@ If you want to extend the use of an established relation type in a way that has an effect outside of your organization, reach out to the Backstage maintainers or a support partner to discuss risk/impact. It may even be that one end of the relation could be considered for addition to the core. + +## Adding a New Status field + +Example intents: + +> "We would like to convey entity statuses through the catalog in a generic way, +> as an integration layer. Our monitoring and alerting system has a plugin with +> Backstage, and it would be useful if the entity's status field contained the +> current alert state close to the actual entity data for anyone to consume. + +While we are considering a mechanism for contributing generic statuses to +entities, no such mechanism has yet been built. If you are interested in that +topic, [this issue](https://github.com/backstage/backstage/issues/2292) contains +more context. + +But in general, errors emitted (and exceptions thrown) by any processor +including custom ones, end up in the [well known key](well-known-statuses.md) +for ingestion status. diff --git a/docs/features/software-catalog/well-known-statuses.md b/docs/features/software-catalog/well-known-statuses.md new file mode 100644 index 0000000000..a516c61daf --- /dev/null +++ b/docs/features/software-catalog/well-known-statuses.md @@ -0,0 +1,51 @@ +--- +id: well-known-statuses +title: Well-known Status fields of Catalog Entities +sidebar_label: Well-known Statuses +# prettier-ignore +description: Lists a number of well known entity statuses, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed. +--- + +This section lists a number of well known +[entity status fields](descriptor-format.md#common-to-all-kinds-status), that +have defined semantics. They can be attached to catalog entities and consumed by +plugins as needed. + +If you are looking to extend the set of statuses, see +[Extending the model](extending-the-model.md). + +## Common Fields + +The values of statuses are currently left unrestricted, except that they must be +objects. They therefore currently formally have no common fields. + +We reserve the right to extend this model in the future, such that some fields +of those value objects gain standardized meaning. We may for example want to add +a standard concept of "severity" or "level" to these. + +## Statuses + +This is a (non-exhaustive) list of statuses that are known to be in active use. + +### `backstage.io/processing-status` + +Contains the current status of the catalog's ingestion of this entity. Errors +that may appear here include inability to read from the remote SCM provider, +syntax errors in the YAML file, and similar. + +Note that the entity data itself may be of an older version, when errors are +present. The ingestion system keeps the old, valid entity data untouched when +possible, so the errors described in this state may not seem to align with the +rest of the entity, because they pertain to a remote that could not be +successfully ingested. This is normal. + +```yaml +# Example: +status: + backstage.io/processing-status: + errors: [] +``` + +This status is in active development and its format will change unexpectedly. Do +not consume it in your own code until such a time that this documentation has +been updated. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c7b049a53e..3544ebdca3 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -40,6 +40,7 @@ "features/software-catalog/references", "features/software-catalog/well-known-annotations", "features/software-catalog/well-known-relations", + "features/software-catalog/well-known-statuses", "features/software-catalog/extending-the-model", "features/software-catalog/external-integrations", "features/software-catalog/software-catalog-api" diff --git a/mkdocs.yml b/mkdocs.yml index 7826b3221d..0e2e78be26 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,7 @@ nav: - Entity References: 'features/software-catalog/references.md' - Well-known Annotations: 'features/software-catalog/well-known-annotations.md' - Well-known Relations: 'features/software-catalog/well-known-relations.md' + - Well-known Statuses: 'features/software-catalog/well-known-statuses.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 2b56879f48..00eaf8410e 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -112,6 +112,7 @@ export type Entity = { metadata: EntityMeta; spec?: JsonObject; relations?: EntityRelation[]; + status?: Record; }; // @public diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index 7abe1f9c0c..1b5d38dee9 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -48,6 +48,14 @@ export type Entity = { * The relations that this entity has with other entities. */ relations?: EntityRelation[]; + + /** + * The current status of the entity, as claimed by various sources. + * + * The keys are implementation defined and the values can be any JSON object + * with semantics that match that implementation. + */ + status?: Record; }; /** diff --git a/packages/catalog-model/src/entity/util.ts b/packages/catalog-model/src/entity/util.ts index 2c63cc4c49..116913c85c 100644 --- a/packages/catalog-model/src/entity/util.ts +++ b/packages/catalog-model/src/entity/util.ts @@ -42,11 +42,9 @@ export function generateEntityEtag(): string { * the next version of this entity. * * Significance, in this case, means that we do not compare generated fields - * such as uid, etag and generation, and we only check that no new annotations - * are added or existing annotations were changed (since they are effectively - * merged when doing updates). + * such as uid, etag and generation. * - * Note that this comparison does NOT take state, relations or similar into + * Note that this comparison does NOT take status, relations or similar into * account. It only compares the actual input entity data, i.e. metadata and * spec. * @@ -86,7 +84,9 @@ export function entityHasChanges(previous: Entity, next: Entity): boolean { // Remove things that we explicitly do not compare delete e1.relations; + delete e1.status; delete e2.relations; + delete e2.status; return !lodash.isEqual(e1, e2); } diff --git a/packages/catalog-model/src/schema/Entity.schema.json b/packages/catalog-model/src/schema/Entity.schema.json index 803e025796..e6b004dfd0 100644 --- a/packages/catalog-model/src/schema/Entity.schema.json +++ b/packages/catalog-model/src/schema/Entity.schema.json @@ -62,6 +62,15 @@ "items": { "$ref": "common#relation" } + }, + "status": { + "type": "object", + "description": "The current status of the entity, as claimed by various sources.", + "patternProperties": { + "^.+$": { + "$ref": "common#status" + } + } } } } diff --git a/packages/catalog-model/src/schema/shared/common.schema.json b/packages/catalog-model/src/schema/shared/common.schema.json index 84ae8a4147..cb4d82cdb9 100644 --- a/packages/catalog-model/src/schema/shared/common.schema.json +++ b/packages/catalog-model/src/schema/shared/common.schema.json @@ -29,7 +29,7 @@ "$id": "#relation", "type": "object", "description": "A directed relation from one entity to another.", - "required": ["type", "source", "target"], + "required": ["type", "target"], "additionalProperties": false, "properties": { "type": { @@ -38,13 +38,16 @@ "pattern": "^\\w+$", "description": "The type of relation." }, - "source": { - "$ref": "#reference" - }, "target": { "$ref": "#reference" } } + }, + "status": { + "$id": "#status", + "type": "object", + "description": "A specific status of an entity.", + "additionalProperties": true } } } diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts index f38af384fa..da1b79600a 100644 --- a/plugins/catalog-backend/src/next/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -91,6 +91,11 @@ describe('Stitcher', () => { }, }, ], + status: { + 'backstage.io/processing-status': { + errors: [], + }, + }, apiVersion: 'a', kind: 'k', metadata: { @@ -171,6 +176,11 @@ describe('Stitcher', () => { }, }, ]), + status: { + 'backstage.io/processing-status': { + errors: [], + }, + }, apiVersion: 'a', kind: 'k', metadata: { diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 2bddf41ac9..8cf96c6b7f 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -41,6 +41,11 @@ function generateStableHash(entity: Entity) { .digest('hex'); } +/** + * Performs the act of stitching - to take all of the various outputs from the + * ingestion process, and stitching them together into the final entity JSON + * shape. + */ export class Stitcher { constructor( private readonly database: Knex, @@ -109,7 +114,7 @@ export class Stitcher { const { entityId, processedEntity, - // errors, + errors, incomingReferenceCount, previousHash, } = result[0]; @@ -137,7 +142,15 @@ export class Stitcher { ['backstage.io/orphan']: 'true', }; } - + if (errors !== '') { + const parsedErrors = JSON.parse(errors); + entity.status = { + ...entity.status, + 'backstage.io/processing-status': { + errors: parsedErrors, + }, + }; + } // TODO: entityRef is lower case and should be uppercase in the final // result entity.relations = result