diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 63a3e53c81..8474478f9e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -29,49 +29,49 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { /* - * ** WARNING ** - * DO NOT enable the experimental catalog, it will brick your database migrations. - * This is solely for internal backstage development. + * This environment variable exists as an emergency option during the release + * of the new catalog processing engine. + * If you experience any issues, make sure to report them as this flag + * will be removed in a subsequent release. */ - if (process.env.EXPERIMENTAL_CATALOG === '1') { - const builder = new NextCatalogBuilder(env); + if (process.env.LEGACY_CATALOG === '1') { + const builder = new CatalogBuilder(env); const { entitiesCatalog, + locationsCatalog, + higherOrderOperation, locationAnalyzer, - processingEngine, - locationService, } = await builder.build(); - // TODO(jhaals): run and manage in background. - await processingEngine.start(); + useHotCleanup( + module, + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), + ); - return await createNextRouter({ + return await createRouter({ entitiesCatalog, + locationsCatalog, + higherOrderOperation, locationAnalyzer, - locationService, logger: env.logger, config: env.config, }); } - - const builder = new CatalogBuilder(env); + const builder = new NextCatalogBuilder(env); const { entitiesCatalog, - locationsCatalog, - higherOrderOperation, locationAnalyzer, + processingEngine, + locationService, } = await builder.build(); - useHotCleanup( - module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), - ); + // TODO(jhaals): run and manage in background. + await processingEngine.start(); - return await createRouter({ + return await createNextRouter({ entitiesCatalog, - locationsCatalog, - higherOrderOperation, locationAnalyzer, + locationService, logger: env.logger, config: env.config, }); diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js similarity index 100% rename from plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js rename to plugins/catalog-backend/migrations/20210302150147_refresh_state.js diff --git a/plugins/catalog-backend/migrationsv2/20200511113813_init.js b/plugins/catalog-backend/migrationsv2/20200511113813_init.js deleted file mode 100644 index 7f3d75e35c..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200511113813_init.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - return ( - knex.schema - // - // locations - // - .createTable('locations', table => { - table.comment( - 'Registered locations that shall be contiuously scanned for catalog item updates', - ); - table - .uuid('id') - .primary() - .notNullable() - .comment('Auto-generated ID of the location'); - table.string('type').notNullable().comment('The type of location'); - table - .string('target') - .notNullable() - .comment('The actual target of the location'); - }) - // - // entities - // - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .string('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .string('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }) - // - // entities_search - // - .createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }) - ); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema - .dropTable('entities_search') - .alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }) - .dropTable('entities') - .dropTable('locations'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js deleted file mode 100644 index d8093fc9b4..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - return knex.schema.createTable('location_update_log', table => { - table.uuid('id').primary(); - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema.dropTableIfExists('location_update_log'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js deleted file mode 100644 index a0f0f33a65..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Get list sorted by created_at timestamp in descending order - // Grouped by location_id - return knex.schema.raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - ORDER BY created_at DESC; - `); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - return knex.schema.raw(`DROP VIEW location_update_log_latest;`); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js deleted file mode 100644 index 0f1c204f9b..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.dropForeign(['entity_id']); - }); - await knex.schema.alterTable('entities', table => { - table.dropPrimary('entities_pkey'); - }); - } - await knex.schema.alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }); - // Setup temporary tables - await knex.schema.renameTable('entities_search', 'tmp_entities_search'); - await knex.schema.renameTable('entities', 'tmp_entities'); - - // - // entities - // - await knex.schema - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .text('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .text('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }); - - await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); - - // - // entities_search - // - await knex.schema.createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }); - await knex.schema.raw( - `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, - ); - - // Clean up - await knex.schema.dropTable('tmp_entities'); - return knex.schema.dropTable('tmp_entities_search'); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.dropForeign(['entity_id']); - }); - await knex.schema.alterTable('entities', table => { - table.dropPrimary('entities_pkey'); - }); - } - await knex.schema.alterTable('entities', table => { - table.dropUnique([], 'entities_unique_name'); - }); - - // Setup temporary tables - await knex.schema.renameTable('entities_search', 'tmp_entities_search'); - await knex.schema.renameTable('entities', 'tmp_entities'); - - // - // entities - // - await knex.schema - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('etag') - .notNullable() - .comment( - 'An opaque string that changes for each update operation to any part of the entity, including metadata.', - ); - table - .string('generation') - .notNullable() - .unsigned() - .comment( - 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', - ); - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table - .string('kind') - .notNullable() - .comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - table - .string('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .string('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - }) - .alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); - }); - - await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); - - // - // entities_search - // - await knex.schema.createTable('entities_search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); - table - .uuid('entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .comment('The entity that matches this key/value'); - table - .string('key') - .notNullable() - .comment('A key that occurs in the entity'); - table - .string('value') - .nullable() - .comment('The corresponding value to match on'); - }); - await knex.schema.raw( - `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, - ); - - // Clean up - await knex.schema.dropTable('tmp_entities'); - return knex.schema.dropTable('tmp_entities_search'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js deleted file mode 100644 index 87b41a80fc..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = function up(knex) { - return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; -`); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = function down(knex) { - knex.schema.raw(`DROP VIEW location_update_log_latest;`); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js deleted file mode 100644 index de2b194cff..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = function up(knex) { - return knex.schema - .raw('DROP VIEW location_update_log_latest;') - .dropTable('location_update_log') - .createTable('location_update_log', table => { - table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = function down(knex) { - return knex.schema - .raw('DROP VIEW location_update_log_latest;') - .dropTable('location_update_log') - .createTable('location_update_log', table => { - table.uuid('id').primary(); - table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); - table.string('message'); - table - .uuid('location_id') - .references('id') - .inTable('locations') - .onUpdate('CASCADE') - .onDelete('CASCADE'); - table.string('entity_name').nullable(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(created_at) AS MAXDATE - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.created_at = t2.MAXDATE - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js deleted file mode 100644 index 45226e53b4..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.text('value').nullable().alter(); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_search', table => { - table.string('value').nullable().alter(); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js deleted file mode 100644 index a90813fe85..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - // Adds a single 'bootstrap' location that can be used to trigger work in processors. - // This is primarily here to fulfill foreign key constraints. - await knex('locations').insert({ - id: require('uuid').v4(), - type: 'bootstrap', - target: 'bootstrap', - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex('locations') - .where({ - type: 'bootstrap', - target: 'bootstrap', - }) - .del(); -}; diff --git a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js deleted file mode 100644 index ea5ba9e58d..0000000000 --- a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex('entities') - .where({ namespace: null }) - .update({ namespace: 'default' }); - await knex('entities_search').update({ - key: knex.raw('LOWER(key)'), - value: knex.raw('LOWER(value)'), - }); -}; - -exports.down = async function down() {}; diff --git a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js deleted file mode 100644 index aae1861658..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table.text('full_name').nullable(); - }); - - await knex('entities').update({ - full_name: knex.raw( - "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", - ), - }); - - // SQLite does not support alter column - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.text('full_name').notNullable().alter(); - }); - } - - await knex.schema.alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.unique(['full_name'], 'entities_unique_full_name'); - table.dropUnique([], 'entities_unique_name'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - table.dropUnique([], 'entities_unique_full_name'); - table.unique(['kind', 'namespace', 'name'], 'entities_unique_name'); - }); - - await knex.schema.alterTable('entities_search', table => { - table.dropColumn('full_name'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js deleted file mode 100644 index a8964efbf6..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table - .text('data') - .nullable() - .comment('The entire JSON data blob of the entity'); - }); - - await knex('entities').update({ - // apiVersion and kind should not contain any JSON unsafe chars, and both - // metadata and spec are already valid serialized JSON - data: knex.raw( - `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, - ), - }); - - await knex.schema.alterTable('entities', table => { - table.dropColumn('metadata'); - table.dropColumn('spec'); - }); - - // SQLite does not support ALTER COLUMN. - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.text('data').notNullable().alter(); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - table - .text('metadata') - .notNullable() - .comment('The entire metadata JSON blob of the entity'); - table - .text('spec') - .nullable() - .comment('The entire spec JSON blob of the entity'); - table.dropColumn('data'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js deleted file mode 100644 index f40df5f73e..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities', table => { - table.dropColumn('api_version'); - table.dropColumn('kind'); - table.dropColumn('name'); - table.dropColumn('namespace'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities', table => { - table - .string('api_version') - .notNullable() - .comment('The apiVersion field of the entity'); - table.string('kind').notNullable().comment('The kind field of the entity'); - table - .string('name') - .nullable() - .comment('The metadata.name field of the entity'); - table - .string('namespace') - .nullable() - .comment('The metadata.namespace field of the entity'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js deleted file mode 100644 index 77bf0529eb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.alterTable('entities_search', table => { - table.index(['key'], 'entities_search_key'); - table.index(['value'], 'entities_search_value'); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.alterTable('entities_search', table => { - table.dropIndex('', 'entities_search_key'); - table.dropIndex('', 'entities_search_value'); - }); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js deleted file mode 100644 index 85e729f814..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - - table.primary(['source_full_name', 'type', 'target_full_name']); - }); -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - await knex.schema.dropTable('entities_relations'); -}; diff --git a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js deleted file mode 100644 index 9e8198b5eb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client === 'sqlite3') { - // sqlite doesn't support dropPrimary so we recreate it properly instead - await knex.schema.dropTable('entities_relations'); - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - table.index('source_full_name', 'source_full_name_idx'); - }); - } else { - await knex.schema.alterTable('entities_relations', table => { - table.dropPrimary(); - table.index('source_full_name', 'source_full_name_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client === 'sqlite3') { - await knex.schema.dropTable('entities_relations'); - await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); - table - .uuid('originating_entity_id') - .references('id') - .inTable('entities') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .string('source_full_name') - .notNullable() - .comment('The full name of the source entity of the relation'); - table - .string('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .string('target_full_name') - .notNullable() - .comment('The full name of the target entity of the relation'); - - table.primary(['source_full_name', 'type', 'target_full_name']); - }); - } else { - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'source_full_name_idx'); - table.primary(['source_full_name', 'type', 'target_full_name']); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js deleted file mode 100644 index abb26cd5fc..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_relations', table => { - table.index('originating_entity_id', 'originating_entity_id_idx'); - }); - await knex.schema.alterTable('entities_search', table => { - table.index('entity_id', 'entity_id_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'originating_entity_id_idx'); - }); - await knex.schema.alterTable('entities_relations', table => { - table.dropIndex([], 'entity_id_idx'); - }); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js deleted file mode 100644 index d924b0414a..0000000000 --- a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - // We actually just want to widen columns, but can't do that while a - // view is dependent on them - so we just reconstruct it exactly as it was - await knex.schema - .raw('DROP VIEW location_update_log_latest;') - .alterTable('location_update_log', table => { - table.text('message').alter(); - table.text('entity_name').nullable().alter(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema - .raw('DROP VIEW location_update_log_latest;') - .alterTable('location_update_log', table => { - table.string('message').alter(); - table.string('entity_name').nullable().alter(); - }).raw(` - CREATE VIEW location_update_log_latest AS - SELECT t1.* FROM location_update_log t1 - JOIN - ( - SELECT location_id, MAX(id) AS MAXID - FROM location_update_log - GROUP BY location_id - ) t2 - ON t1.location_id = t2.location_id - AND t1.id = t2.MAXID - GROUP BY t1.location_id, t1.id - ORDER BY created_at DESC; - `); - } -}; diff --git a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js deleted file mode 100644 index ccfb1faffb..0000000000 --- a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @ts-check - -/** - * @param {import('knex').Knex} knex - */ -exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.index('location_id', 'entity_location_id_idx'); - }); - await knex.schema.alterTable('location_update_log', table => { - table.index('location_id', 'update_log_location_id_idx'); - }); - } -}; - -/** - * @param {import('knex').Knex} knex - */ -exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { - await knex.schema.alterTable('entities', table => { - table.dropIndex([], 'entity_location_id_idx'); - }); - await knex.schema.alterTable('location_update_log', table => { - table.dropIndex([], 'update_log_location_id_idx'); - }); - } -}; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index e10d13c129..b6b0a04d04 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -246,7 +246,7 @@ export class NextCatalogBuilder { await dbClient.migrate.latest({ directory: resolvePackagePath( '@backstage/plugin-catalog-backend', - 'migrationsv2', + 'migrations', ), });