From 8650112701671551b34cc553abfaf6db24cf3cd9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 3 Jun 2021 09:49:18 +0200 Subject: [PATCH 1/6] Catalog: Enable new processing engine Signed-off-by: Johan Haals --- packages/backend/src/plugins/catalog.ts | 44 ++-- .../20210302150147_refresh_state.js | 0 .../migrationsv2/20200511113813_init.js | 133 ---------- ...0200520140700_location_update_log_table.js | 43 ---- ...7114117_location_update_log_latest_view.js | 45 ---- .../migrationsv2/20200702153613_entities.js | 236 ------------------ ..._location_update_log_latest_deduplicate.js | 44 ---- ...904_location_update_log_duplication_fix.js | 87 ------- .../20200807120600_entitySearch.js | 41 --- .../20200809202832_add_bootstrap_location.js | 42 ---- .../20200923104503_case_insensitivity.js | 32 --- .../20201005122705_add_entity_full_name.js | 60 ----- .../20201006130744_entity_data_column.js | 66 ----- ...6203131_entity_remove_redundant_columns.js | 50 ---- .../20201007201501_index_entity_search.js | 37 --- .../20201019130742_add_relations_table.js | 54 ---- .../20201123205611_relations_table_uniq.js | 93 ------- .../migrationsv2/20201210185851_fk_index.js | 45 ---- .../20201230103504_update_log_varchar.js | 73 ------ .../20210209121210_locations_fk_index.js | 45 ---- .../src/next/NextCatalogBuilder.ts | 2 +- 21 files changed, 23 insertions(+), 1249 deletions(-) rename plugins/catalog-backend/{migrationsv2 => migrations}/20210302150147_refresh_state.js (100%) delete mode 100644 plugins/catalog-backend/migrationsv2/20200511113813_init.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200702153613_entities.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js delete mode 100644 plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js delete mode 100644 plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js delete mode 100644 plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 63a3e53c81..8474478f9e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -29,49 +29,49 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { /* - * ** WARNING ** - * DO NOT enable the experimental catalog, it will brick your database migrations. - * This is solely for internal backstage development. + * This environment variable exists as an emergency option during the release + * of the new catalog processing engine. + * If you experience any issues, make sure to report them as this flag + * will be removed in a subsequent release. */ - if (process.env.EXPERIMENTAL_CATALOG === '1') { - const builder = new NextCatalogBuilder(env); + if (process.env.LEGACY_CATALOG === '1') { + const builder = new CatalogBuilder(env); const { entitiesCatalog, + locationsCatalog, + higherOrderOperation, locationAnalyzer, - processingEngine, - locationService, } = await builder.build(); - // TODO(jhaals): run and manage in background. - await processingEngine.start(); + useHotCleanup( + module, + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), + ); - return await createNextRouter({ + return await createRouter({ entitiesCatalog, + locationsCatalog, + higherOrderOperation, locationAnalyzer, - locationService, logger: env.logger, config: env.config, }); } - - const builder = new CatalogBuilder(env); + const builder = new NextCatalogBuilder(env); const { entitiesCatalog, - locationsCatalog, - higherOrderOperation, locationAnalyzer, + processingEngine, + locationService, } = await builder.build(); - useHotCleanup( - module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), - ); + // TODO(jhaals): run and manage in background. + await processingEngine.start(); - return await createRouter({ + return await createNextRouter({ entitiesCatalog, - locationsCatalog, - higherOrderOperation, locationAnalyzer, + locationService, logger: env.logger, config: env.config, }); diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js similarity index 100% rename from plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js rename to plugins/catalog-backend/migrations/20210302150147_refresh_state.js diff --git a/plugins/catalog-backend/migrationsv2/20200511113813_init.js b/plugins/catalog-backend/migrationsv2/20200511113813_init.js deleted file mode 100644 index 7f3d75e35c..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200511113813_init.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - return ( - knex.schema - // - // locations - // - .createTable('locations', table => { - table.comment( - 'Registered locations that shall be contiuously scanned for catalog item updates', - ); - table - .uuid('id') - .primary() - .notNullable() - .comment('Auto-generated ID of the location'); - table.string('type').notNullable().comment('The type of location'); - table - .string('target') - .notNullable() - .comment('The actual target of the location'); - }) - // - // entities - // - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .string('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .string('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }) - // - // entities_search - // - .createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }) - ); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema - .dropTable('entities_search') - .alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }) - .dropTable('entities') - .dropTable('locations'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js deleted file mode 100644 index d8093fc9b4..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - return knex.schema.createTable('location_update_log', table => { - table.uuid('id').primary(); - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema.dropTableIfExists('location_update_log'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js deleted file mode 100644 index a0f0f33a65..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Get list sorted by created_at timestamp in descending order - // Grouped by location_id - return knex.schema.raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - ORDER BY created_at DESC; - `); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema.raw(`DROP VIEW location_update_log_latest;`); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js deleted file mode 100644 index 0f1c204f9b..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.dropForeign(['entity_id']); - }); - await knex.schema.alterTable('entities', table => { - table.dropPrimary('entities_pkey'); - }); - } - await knex.schema.alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }); - // Setup temporary tables - await knex.schema.renameTable('entities_search', 'tmp_entities_search'); - await knex.schema.renameTable('entities', 'tmp_entities'); - - // - // entities - // - await knex.schema - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .text('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .text('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }); - - await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); - - // - // entities_search - // - await knex.schema.createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }); - await knex.schema.raw( - `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, - ); - - // Clean up - await knex.schema.dropTable('tmp_entities'); - return knex.schema.dropTable('tmp_entities_search'); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.dropForeign(['entity_id']); - }); - await knex.schema.alterTable('entities', table => { - table.dropPrimary('entities_pkey'); - }); - } - await knex.schema.alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }); - - // Setup temporary tables - await knex.schema.renameTable('entities_search', 'tmp_entities_search'); - await knex.schema.renameTable('entities', 'tmp_entities'); - - // - // entities - // - await knex.schema - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .string('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .string('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }); - - await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); - - // - // entities_search - // - await knex.schema.createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }); - await knex.schema.raw( - `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, - ); - - // Clean up - await knex.schema.dropTable('tmp_entities'); - return knex.schema.dropTable('tmp_entities_search'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js deleted file mode 100644 index 87b41a80fc..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = function up(knex) { - return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; -`); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = function down(knex) { - knex.schema.raw(`DROP VIEW location_update_log_latest;`); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js deleted file mode 100644 index de2b194cff..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = function up(knex) { - return knex.schema - .raw('DROP VIEW location_update_log_latest;') - .dropTable('location_update_log') - .createTable('location_update_log', table => { - table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = function down(knex) { - return knex.schema - .raw('DROP VIEW location_update_log_latest;') - .dropTable('location_update_log') - .createTable('location_update_log', table => { - table.uuid('id').primary(); - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js deleted file mode 100644 index 45226e53b4..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.text('value').nullable().alter(); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.string('value').nullable().alter(); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js deleted file mode 100644 index a90813fe85..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Adds a single 'bootstrap' location that can be used to trigger work in processors. - // This is primarily here to fulfill foreign key constraints. - await knex('locations').insert({ - id: require('uuid').v4(), - type: 'bootstrap', - target: 'bootstrap', - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex('locations') - .where({ - type: 'bootstrap', - target: 'bootstrap', - }) - .del(); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js deleted file mode 100644 index ea5ba9e58d..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex('entities') - .where({ namespace: null }) - .update({ namespace: 'default' }); - await knex('entities_search').update({ - key: knex.raw('LOWER(key)'), - value: knex.raw('LOWER(value)'), - }); -}; - -exports.down = async function down() {}; diff --git a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js deleted file mode 100644 index aae1861658..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table.text('full_name').nullable(); - }); - - await knex('entities').update({ - full_name: knex.raw( - "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", - ), - }); - - // SQLite does not support alter column - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.text('full_name').notNullable().alter(); - }); - } - - await knex.schema.alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['full_name'], 'entities_unique_full_name'); - table.dropUnique([], 'entities_unique_name'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.dropUnique([], 'entities_unique_full_name'); - table.unique(['kind', 'namespace', 'name'], 'entities_unique_name'); - }); - - await knex.schema.alterTable('entities_search', table => { - table.dropColumn('full_name'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js deleted file mode 100644 index a8964efbf6..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table - .text('data') - .nullable() - .comment('The entire JSON data blob of the entity'); - }); - - await knex('entities').update({ - // apiVersion and kind should not contain any JSON unsafe chars, and both - // metadata and spec are already valid serialized JSON - data: knex.raw( - `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, - ), - }); - - await knex.schema.alterTable('entities', table => { - table.dropColumn('metadata'); - table.dropColumn('spec'); - }); - - // SQLite does not support ALTER COLUMN. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.text('data').notNullable().alter(); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - table - .text('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .text('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - table.dropColumn('data'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js deleted file mode 100644 index f40df5f73e..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table.dropColumn('api_version'); - table.dropColumn('kind'); - table.dropColumn('name'); - table.dropColumn('namespace'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table.string('kind').notNullable().comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js deleted file mode 100644 index 77bf0529eb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities_search', table => { - table.index(['key'], 'entities_search_key'); - table.index(['value'], 'entities_search_value'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities_search', table => { - table.dropIndex('', 'entities_search_key'); - table.dropIndex('', 'entities_search_value'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js deleted file mode 100644 index 85e729f814..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - - table.primary(['source_full_name', 'type', 'target_full_name']); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.dropTable('entities_relations'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js deleted file mode 100644 index 9e8198b5eb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client === 'sqlite3') { - // sqlite doesn't support dropPrimary so we recreate it properly instead - await knex.schema.dropTable('entities_relations'); - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - table.index('source_full_name', 'source_full_name_idx'); - }); - } else { - await knex.schema.alterTable('entities_relations', table => { - table.dropPrimary(); - table.index('source_full_name', 'source_full_name_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client === 'sqlite3') { - await knex.schema.dropTable('entities_relations'); - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - - table.primary(['source_full_name', 'type', 'target_full_name']); - }); - } else { - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'source_full_name_idx'); - table.primary(['source_full_name', 'type', 'target_full_name']); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js deleted file mode 100644 index abb26cd5fc..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_relations', table => { - table.index('originating_entity_id', 'originating_entity_id_idx'); - }); - await knex.schema.alterTable('entities_search', table => { - table.index('entity_id', 'entity_id_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'originating_entity_id_idx'); - }); - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'entity_id_idx'); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js deleted file mode 100644 index d924b0414a..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - // We actually just want to widen columns, but can't do that while a - // view is dependent on them - so we just reconstruct it exactly as it was - await knex.schema - .raw('DROP VIEW location_update_log_latest;') - .alterTable('location_update_log', table => { - table.text('message').alter(); - table.text('entity_name').nullable().alter(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema - .raw('DROP VIEW location_update_log_latest;') - .alterTable('location_update_log', table => { - table.string('message').alter(); - table.string('entity_name').nullable().alter(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js deleted file mode 100644 index ccfb1faffb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.index('location_id', 'entity_location_id_idx'); - }); - await knex.schema.alterTable('location_update_log', table => { - table.index('location_id', 'update_log_location_id_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.dropIndex([], 'entity_location_id_idx'); - }); - await knex.schema.alterTable('location_update_log', table => { - table.dropIndex([], 'update_log_location_id_idx'); - }); - } -}; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index e10d13c129..b6b0a04d04 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -246,7 +246,7 @@ export class NextCatalogBuilder { await dbClient.migrate.latest({ directory: resolvePackagePath( '@backstage/plugin-catalog-backend', - 'migrationsv2', + 'migrations', ), }); From b45e29410a8ec199840b3f36362c30e0a126e584 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 3 Jun 2021 13:19:57 +0200 Subject: [PATCH 2/6] Add changeset Signed-off-by: Johan Haals --- .changeset/dull-poets-learn.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/dull-poets-learn.md diff --git a/.changeset/dull-poets-learn.md b/.changeset/dull-poets-learn.md new file mode 100644 index 0000000000..0574d0ebc6 --- /dev/null +++ b/.changeset/dull-poets-learn.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +This release enables the new catalog processing engine which is a major milestone for the catalog! + +This update makes processing more scalable across multiple instances, adds support for deletions and ui flagging of entities that are no longer referenced by a location. + +As this is a major internal change we have taken some precaution by offering a `LEGACY_CATALOG=1` environment variable that you can set when starting the backend in order to run the previous version. If you do so for any reason make sure to raise an issue immediately as this is a temporary "break the glass" option which will be removed in a subsequent release. From f8e773e3e979564dd81ae3a476c8bf517cc2f8a6 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 4 Jun 2021 08:48:54 +0200 Subject: [PATCH 3/6] chore: Update migrations dir in tests Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/database/DatabaseManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts index 22ae4e782f..d501bc6c61 100644 --- a/plugins/catalog-backend/src/next/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/next/database/DatabaseManager.ts @@ -36,7 +36,7 @@ export class DatabaseManager { ): Promise { const migrationsDir = resolvePackagePath( '@backstage/plugin-catalog-backend', - 'migrationsv2', + 'migrations', ); await knex.migrate.latest({ From f68632411b66be9d7d80ce8ef6ce8acb6794382f Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 14 Jun 2021 16:10:18 +0200 Subject: [PATCH 4/6] catalog: Reuse builder for next/legacy catalog Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- packages/backend/src/plugins/catalog.ts | 38 +------------------ .../src/service/CatalogBuilder.ts | 9 +++++ plugins/catalog-backend/src/service/router.ts | 37 ++++++++++++++++++ 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 8474478f9e..57afe4fefd 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -14,13 +14,9 @@ * limitations under the License. */ -import { useHotCleanup } from '@backstage/backend-common'; import { CatalogBuilder, createRouter, - NextCatalogBuilder, - runPeriodically, - createNextRouter, } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; @@ -28,36 +24,7 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin( env: PluginEnvironment, ): Promise { - /* - * This environment variable exists as an emergency option during the release - * of the new catalog processing engine. - * If you experience any issues, make sure to report them as this flag - * will be removed in a subsequent release. - */ - if (process.env.LEGACY_CATALOG === '1') { - const builder = new CatalogBuilder(env); - const { - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - locationAnalyzer, - } = await builder.build(); - - useHotCleanup( - module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), - ); - - return await createRouter({ - entitiesCatalog, - locationsCatalog, - higherOrderOperation, - locationAnalyzer, - logger: env.logger, - config: env.config, - }); - } - const builder = new NextCatalogBuilder(env); + const builder = await CatalogBuilder.create(env); const { entitiesCatalog, locationAnalyzer, @@ -65,10 +32,9 @@ export default async function createPlugin( locationService, } = await builder.build(); - // TODO(jhaals): run and manage in background. await processingEngine.start(); - return await createNextRouter({ + return await createRouter({ entitiesCatalog, locationAnalyzer, locationService, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f742e3c0db..9bec4442d7 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -66,6 +66,7 @@ import { } from '../ingestion/processors/PlaceholderProcessor'; import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; +import { NextCatalogBuilder } from '../next'; export type CatalogEnvironment = { logger: Logger; @@ -103,6 +104,10 @@ export class CatalogBuilder { private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; + static async create(env: CatalogEnvironment): Promise { + return new NextCatalogBuilder(env); + } + constructor(env: CatalogEnvironment) { this.env = env; this.entityPolicies = []; @@ -112,6 +117,10 @@ export class CatalogBuilder { this.processors = []; this.processorsReplace = false; this.parser = undefined; + + env.logger.warn( + "Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead", + ); } /** diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index ae58255a4e..525ba76c0c 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -28,6 +28,7 @@ import { Logger } from 'winston'; import yn from 'yn'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types'; +import { LocationService } from '../next/types'; import { basicEntityFilter, parseEntityFilterParams, @@ -45,6 +46,7 @@ export interface RouterOptions { locationsCatalog?: LocationsCatalog; higherOrderOperation?: HigherOrderOperation; locationAnalyzer?: LocationAnalyzer; + locationService?: LocationService; logger: Logger; config: Config; } @@ -57,6 +59,7 @@ export async function createRouter( locationsCatalog, higherOrderOperation, locationAnalyzer, + locationService, config, logger, } = options; @@ -145,6 +148,40 @@ export async function createRouter( }); } + if (locationService) { + router + .post('/locations', async (req, res) => { + const input = await validateRequestBody(req, locationSpecSchema); + const dryRun = yn(req.query.dryRun, { default: false }); + + // when in dryRun addLocation is effectively a read operation so we don't + // need to disallow readonly + if (!dryRun) { + disallowReadonlyMode(readonlyEnabled); + } + + const output = await locationService.createLocation(input, dryRun); + res.status(201).json(output); + }) + .get('/locations', async (_req, res) => { + const locations = await locationService.listLocations(); + res.status(200).json(locations.map(l => ({ data: l }))); + }) + + .get('/locations/:id', async (req, res) => { + const { id } = req.params; + const output = await locationService.getLocation(id); + res.status(200).json(output); + }) + .delete('/locations/:id', async (req, res) => { + disallowReadonlyMode(readonlyEnabled); + + const { id } = req.params; + await locationService.deleteLocation(id); + res.status(204).end(); + }); + } + if (higherOrderOperation) { router.post('/locations', async (req, res) => { const input = await validateRequestBody(req, locationSpecSchema); From 9c3840b1ba2566177052ca87c7f4e94cbe49c1f6 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 15 Jun 2021 09:17:10 +0200 Subject: [PATCH 5/6] create-app: Enable new catalog processing engine Signed-off-by: Johan Haals --- .../packages/backend/src/plugins/catalog.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts index 838228cdb4..a132f0a8ae 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -1,30 +1,26 @@ -import { useHotCleanup } from '@backstage/backend-common'; import { CatalogBuilder, - createRouter, - runPeriodically + createRouter } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; export default async function createPlugin(env: PluginEnvironment): Promise { - const builder = new CatalogBuilder(env); + const builder = await CatalogBuilder.create(env); const { entitiesCatalog, locationsCatalog, - higherOrderOperation, + locationService, + processingEngine, locationAnalyzer, } = await builder.build(); - useHotCleanup( - module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), - ); + await processingEngine.start(); return await createRouter({ entitiesCatalog, locationsCatalog, - higherOrderOperation, + locationService, locationAnalyzer, logger: env.logger, config: env.config, From 0307a28b4ac3472196aa79525d7ff7040a8abecf Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 15 Jun 2021 09:17:55 +0200 Subject: [PATCH 6/6] Add changeset for create-app describing upgrade path Signed-off-by: Johan Haals --- .changeset/dull-poets-learn.md | 45 +++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.changeset/dull-poets-learn.md b/.changeset/dull-poets-learn.md index 0574d0ebc6..b504c0c6a3 100644 --- a/.changeset/dull-poets-learn.md +++ b/.changeset/dull-poets-learn.md @@ -1,9 +1,52 @@ --- '@backstage/plugin-catalog-backend': patch +'@backstage/create-app': patch --- This release enables the new catalog processing engine which is a major milestone for the catalog! This update makes processing more scalable across multiple instances, adds support for deletions and ui flagging of entities that are no longer referenced by a location. -As this is a major internal change we have taken some precaution by offering a `LEGACY_CATALOG=1` environment variable that you can set when starting the backend in order to run the previous version. If you do so for any reason make sure to raise an issue immediately as this is a temporary "break the glass" option which will be removed in a subsequent release. +**Changes Required** to `catalog.ts` + +```diff +-import { useHotCleanup } from '@backstage/backend-common'; + import { + CatalogBuilder, +- createRouter, +- runPeriodically ++ createRouter + } from '@backstage/plugin-catalog-backend'; + import { Router } from 'express'; + import { PluginEnvironment } from '../types'; + + export default async function createPlugin(env: PluginEnvironment): Promise { +- const builder = new CatalogBuilder(env); ++ const builder = await CatalogBuilder.create(env); + const { + entitiesCatalog, + locationsCatalog, +- higherOrderOperation, ++ locationService, ++ processingEngine, + locationAnalyzer, + } = await builder.build(); + +- useHotCleanup( +- module, +- runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), +- ); ++ await processingEngine.start(); + + return await createRouter({ + entitiesCatalog, + locationsCatalog, +- higherOrderOperation, ++ locationService, + locationAnalyzer, + logger: env.logger, + config: env.config, +``` + +As this is a major internal change we have taken some precaution by still allowing the old catalog to be enabled by keeping your `catalog.ts` in it's current state. +If you encounter any issues and have to revert to the previous catalog engine make sure to raise an issue immediately as the old catalog engine is deprecated and will be removed in a future release.