From 9b6b51931419766a3fb53d66a0f2145d1b3b96e4 Mon Sep 17 00:00:00 2001 From: Daniel Johansson Date: Fri, 3 Jul 2020 09:44:31 +0200 Subject: [PATCH] Update migrations to support postgres --- .../src/identity/DatabaseKeyStore.ts | 1 + ...7114117_location_update_log_latest_view.js | 27 +- .../migrations/20200702153613_entities.js | 240 ++++++++++++++++++ .../src/database/DatabaseManager.ts | 2 + 4 files changed, 258 insertions(+), 12 deletions(-) create mode 100644 plugins/catalog-backend/migrations/20200702153613_entities.js diff --git a/plugins/auth-backend/src/identity/DatabaseKeyStore.ts b/plugins/auth-backend/src/identity/DatabaseKeyStore.ts index c175cd6e12..a9fc29b397 100644 --- a/plugins/auth-backend/src/identity/DatabaseKeyStore.ts +++ b/plugins/auth-backend/src/identity/DatabaseKeyStore.ts @@ -42,6 +42,7 @@ export class DatabaseKeyStore implements KeyStore { await database.migrate.latest({ directory: migrationsDir, + tableName: 'knex_migrations_auth', }); return new DatabaseKeyStore(options); diff --git a/plugins/catalog-backend/migrations/20200527114117_location_update_log_latest_view.js b/plugins/catalog-backend/migrations/20200527114117_location_update_log_latest_view.js index e51be30b81..083d0a38aa 100644 --- a/plugins/catalog-backend/migrations/20200527114117_location_update_log_latest_view.js +++ b/plugins/catalog-backend/migrations/20200527114117_location_update_log_latest_view.js @@ -20,18 +20,21 @@ * @param {import('knex')} knex */ exports.up = async function up(knex) { - // Need to first order by date of creation - const query = knex - .select() - .from('location_update_log') - .orderBy('location_update_log.created_at', 'desc'); - - // And only then to do the grouping to get the latest per location - const groupedQuery = knex(query).groupBy('location_id').select(); - - await knex.schema.raw( - `CREATE VIEW location_update_log_latest AS ${groupedQuery.toString()};`, - ); + // Get list sorted by created_at timestamp in descending order + // Grouped by location_id + return knex.schema.raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + ORDER BY created_at DESC; + `); }; /** diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js new file mode 100644 index 0000000000..c97331796d --- /dev/null +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -0,0 +1,240 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + // Drop constraints (Postgres) + try { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } catch (e) { + // SQLite does not support FK and PK, carry on + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + // Drop constraints (Postgres) + try { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } catch (e) { + // SQLite does not support FK and PK, carry on + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 28751f4830..02a72f8f6e 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -44,6 +44,7 @@ export class DatabaseManager { ): Promise { await knex.migrate.latest({ directory: migrationsDir, + tableName: 'knex_migrations_catalog', }); const { logger, fieldNormalizer } = { ...defaultOptions, ...options }; return new CommonDatabase(knex, fieldNormalizer, logger); @@ -74,6 +75,7 @@ export class DatabaseManager { }); await knex.migrate.latest({ directory: migrationsDir, + tableName: 'knex_migrations_catalog', }); const { logger, fieldNormalizer } = defaultOptions; return new CommonDatabase(knex, fieldNormalizer, logger);