diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js new file mode 100644 index 0000000000..fdabf72fce --- /dev/null +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('data') + .nullable() + .comment('The entire JSON data blob of the entity'); + }); + + await knex('entities').update({ + // apiVersion and kind should not contain any JSON unsafe chars, and both + // metadata and spec are already valid serialized JSON + data: knex.raw( + `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, + ), + }); + + await knex.schema.alterTable('entities', table => { + table.dropColumn('metadata'); + table.dropColumn('spec'); + }); + + // SQLite does not support ALTER COLUMN. Note that we do not use the try/ + // catch method as in other migrations, because if the transaction is + // partially failed, it will further mess up the already messed-up + // statement below this. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.text('data').notNullable().alter(); + }); + } + + // NOTE(freben): For some reason, specifically sqlite3 in-mem just drops some + // subset of constraints sometimes, when a table column is dropped - even if + // the column had no relation at all to the constraint. We therefore recreate + // the constraint here as a stupid fix. + if (knex.client.config.client === 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.unique(['full_name'], 'entities_unique_full_name'); + }); + } +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + table.dropColumn('data'); + }); +}; diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index 5327384593..628b9dfb97 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -21,7 +21,6 @@ import { } from '@backstage/backend-common'; import { Entity, - EntityMeta, EntityName, ENTITY_DEFAULT_NAMESPACE, ENTITY_META_GENERATED_FIELDS, @@ -389,6 +388,11 @@ export class CommonDatabase implements Database { ).toLowerCase(); const lowerName = entity.metadata.name.toLowerCase(); + const data = { + ...entity, + metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS), + }; + return { id: entity.metadata.uid!, location_id: locationId || null, @@ -399,29 +403,15 @@ export class CommonDatabase implements Database { kind: entity.kind, name: entity.metadata.name, namespace: entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE, - metadata: JSON.stringify( - lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS), - ), - spec: entity.spec ? JSON.stringify(entity.spec) : null, + data: JSON.stringify(data), }; } private toEntityResponse(row: DbEntitiesRow): DbEntityResponse { - const entity: Entity = { - apiVersion: row.api_version, - kind: row.kind, - metadata: { - ...(JSON.parse(row.metadata) as EntityMeta), - uid: row.id, - etag: row.etag, - generation: Number(row.generation), // cast because of sqlite - }, - }; - - if (row.spec) { - const spec = JSON.parse(row.spec); - entity.spec = spec; - } + const entity = JSON.parse(row.data) as Entity; + entity.metadata.uid = row.id; + entity.metadata.etag = row.etag; + entity.metadata.generation = Number(row.generation); // cast due to sqlite return { locationId: row.location_id || undefined, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index c0f5777743..162bc3507f 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -26,8 +26,7 @@ export type DbEntitiesRow = { etag: string; generation: number; full_name: string; - metadata: string; - spec: string | null; + data: string; }; export type DbEntityRequest = { diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index 571602b966..7232b876c7 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -197,7 +197,7 @@ export class HigherOrderOperations implements HigherOrderOperation { } const delta = process.hrtime(startTimestamp); - const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e6).toFixed(1); + const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e9).toFixed(1); this.logger.info( `Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${durationMs} seconds`, ); diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts index 3f32a34da4..507da37b39 100644 --- a/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts @@ -21,7 +21,7 @@ * The RDN is the name of the leftmost attribute that identifies the item; for * example, for an item with the fully qualified DN * uid=john,ou=people,ou=spotify,dc=spotify,dc=net the generated entity would - * have this attribute, with the value "john". + * have this annotation, with the value "john". */ export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn'; @@ -31,7 +31,7 @@ export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn'; * * The DN is the fully qualified name that identifies the item; for example, * for an item with the DN uid=john,ou=people,ou=spotify,dc=spotify,dc=net the - * generated entity would have this attribute, with that full string as its + * generated entity would have this annotation, with that full string as its * value. */ export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn'; @@ -42,7 +42,7 @@ export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn'; * * The UUID is the globally unique ID that identifies the item; for example, * for an item with the UUID 76ef928a-b251-1037-9840-d78227f36a7e, the - * generated entity would have this attribute, with that full string as its + * generated entity would have this annotation, with that full string as its * value. */ export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid';