paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

Merge pull request #5901 from backstage/jhaals/legacy-catalog

Browse Source 
Catalog: Make the new processing engine the default
This commit is contained in:
Johan Haals
2021-06-15 11:40:53 +02:00
committed by GitHub
parent e2261dbec1 0307a28b4a
commit a0f2f63374
26 changed files with 111 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js → plugins/catalog-backend/migrations/20210302150147_refresh_state.js
View File
plugins/catalog-backend/migrationsv2/20200511113813_init.js
-133
View File
@@ -1,133 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
return (
knex.schema
//
// locations
//
.createTable('locations', table => {
table.comment(
'Registered locations that shall be contiuously scanned for catalog item updates',
);
table
.uuid('id')
.primary()
.notNullable()
.comment('Auto-generated ID of the location');
table.string('type').notNullable().comment('The type of location');
table
.string('target')
.notNullable()
.comment('The actual target of the location');
})
//
// entities
//
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.string('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.string('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
})
//
// entities_search
//
.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
})
);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema
.dropTable('entities_search')
.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
})
.dropTable('entities')
.dropTable('locations');
};
plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js
-43
View File
@@ -1,43 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
return knex.schema.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema.dropTableIfExists('location_update_log');
};
plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Get list sorted by created_at timestamp in descending order
// Grouped by location_id
return knex.schema.raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema.raw(`DROP VIEW location_update_log_latest;`);
};
plugins/catalog-backend/migrationsv2/20200702153613_entities.js
-236
View File
@@ -1,236 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// SQLite does not support FK and PK
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.dropForeign(['entity_id']);
});
await knex.schema.alterTable('entities', table => {
table.dropPrimary('entities_pkey');
});
}
await knex.schema.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
});
// Setup temporary tables
await knex.schema.renameTable('entities_search', 'tmp_entities_search');
await knex.schema.renameTable('entities', 'tmp_entities');
//
// entities
//
await knex.schema
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.text('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.text('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
});
await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`);
//
// entities_search
//
await knex.schema.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
});
await knex.schema.raw(
`INSERT INTO entities_search SELECT * FROM tmp_entities_search`,
);
// Clean up
await knex.schema.dropTable('tmp_entities');
return knex.schema.dropTable('tmp_entities_search');
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
// SQLite does not support FK and PK
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.dropForeign(['entity_id']);
});
await knex.schema.alterTable('entities', table => {
table.dropPrimary('entities_pkey');
});
}
await knex.schema.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
});
// Setup temporary tables
await knex.schema.renameTable('entities_search', 'tmp_entities_search');
await knex.schema.renameTable('entities', 'tmp_entities');
//
// entities
//
await knex.schema
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.string('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.string('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
});
await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`);
//
// entities_search
//
await knex.schema.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
});
await knex.schema.raw(
`INSERT INTO entities_search SELECT * FROM tmp_entities_search`,
);
// Clean up
await knex.schema.dropTable('tmp_entities');
return knex.schema.dropTable('tmp_entities_search');
};
plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js
-44
View File
@@ -1,44 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = function up(knex) {
return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = function down(knex) {
knex.schema.raw(`DROP VIEW location_update_log_latest;`);
};
plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js
-87
View File
@@ -1,87 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = function up(knex) {
return knex.schema
.raw('DROP VIEW location_update_log_latest;')
.dropTable('location_update_log')
.createTable('location_update_log', table => {
table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = function down(knex) {
return knex.schema
.raw('DROP VIEW location_update_log_latest;')
.dropTable('location_update_log')
.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js
-41
View File
@@ -1,41 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Sqlite does not support alter column.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.text('value').nullable().alter();
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
// Sqlite does not support alter column.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.string('value').nullable().alter();
});
}
};
plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Adds a single 'bootstrap' location that can be used to trigger work in processors.
// This is primarily here to fulfill foreign key constraints.
await knex('locations').insert({
id: require('uuid').v4(),
type: 'bootstrap',
target: 'bootstrap',
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex('locations')
.where({
type: 'bootstrap',
target: 'bootstrap',
})
.del();
};
plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js
-32
View File
@@ -1,32 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex('entities')
.where({ namespace: null })
.update({ namespace: 'default' });
await knex('entities_search').update({
key: knex.raw('LOWER(key)'),
value: knex.raw('LOWER(value)'),
});
};
exports.down = async function down() {};
plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js
-60
View File
@@ -1,60 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table.text('full_name').nullable();
});
await knex('entities').update({
full_name: knex.raw(
"LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)",
),
});
// SQLite does not support alter column
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.text('full_name').notNullable().alter();
});
}
await knex.schema.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['full_name'], 'entities_unique_full_name');
table.dropUnique([], 'entities_unique_name');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.dropUnique([], 'entities_unique_full_name');
table.unique(['kind', 'namespace', 'name'], 'entities_unique_name');
});
await knex.schema.alterTable('entities_search', table => {
table.dropColumn('full_name');
});
};
plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js
-66
View File
@@ -1,66 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table
.text('data')
.nullable()
.comment('The entire JSON data blob of the entity');
});
await knex('entities').update({
// apiVersion and kind should not contain any JSON unsafe chars, and both
// metadata and spec are already valid serialized JSON
data: knex.raw(
`'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`,
),
});
await knex.schema.alterTable('entities', table => {
table.dropColumn('metadata');
table.dropColumn('spec');
});
// SQLite does not support ALTER COLUMN.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.text('data').notNullable().alter();
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
table
.text('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.text('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
table.dropColumn('data');
});
};
plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js
-50
View File
@@ -1,50 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table.dropColumn('api_version');
table.dropColumn('kind');
table.dropColumn('name');
table.dropColumn('namespace');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table.string('kind').notNullable().comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
});
};
plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js
-37
View File
@@ -1,37 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities_search', table => {
table.index(['key'], 'entities_search_key');
table.index(['value'], 'entities_search_value');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities_search', table => {
table.dropIndex('', 'entities_search_key');
table.dropIndex('', 'entities_search_value');
});
};
plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js
-54
View File
@@ -1,54 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.dropTable('entities_relations');
};
plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js
-93
View File
@@ -1,93 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client === 'sqlite3') {
// sqlite doesn't support dropPrimary so we recreate it properly instead
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.index('source_full_name', 'source_full_name_idx');
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropPrimary();
table.index('source_full_name', 'source_full_name_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client === 'sqlite3') {
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'source_full_name_idx');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
}
};
plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_relations', table => {
table.index('originating_entity_id', 'originating_entity_id_idx');
});
await knex.schema.alterTable('entities_search', table => {
table.index('entity_id', 'entity_id_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'originating_entity_id_idx');
});
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'entity_id_idx');
});
}
};
plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js
-73
View File
@@ -1,73 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
// We actually just want to widen columns, but can't do that while a
// view is dependent on them - so we just reconstruct it exactly as it was
await knex.schema
.raw('DROP VIEW location_update_log_latest;')
.alterTable('location_update_log', table => {
table.text('message').alter();
table.text('entity_name').nullable().alter();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema
.raw('DROP VIEW location_update_log_latest;')
.alterTable('location_update_log', table => {
table.string('message').alter();
table.string('entity_name').nullable().alter();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
}
};
plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.index('location_id', 'entity_location_id_idx');
});
await knex.schema.alterTable('location_update_log', table => {
table.index('location_id', 'update_log_location_id_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.dropIndex([], 'entity_location_id_idx');
});
await knex.schema.alterTable('location_update_log', table => {
table.dropIndex([], 'update_log_location_id_idx');
});
}
};
plugins/catalog-backend/src/next/NextCatalogBuilder.ts
+1 -1
View File
@@ -257,7 +257,7 @@ export class NextCatalogBuilder {
await dbClient.migrate.latest({
directory: resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrationsv2',
'migrations',
),
});
plugins/catalog-backend/src/next/database/DatabaseManager.ts
+1 -1
View File
@@ -36,7 +36,7 @@ export class DatabaseManager {
): Promise<Database> {
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrationsv2',
'migrations',
);
await knex.migrate.latest({
plugins/catalog-backend/src/service/CatalogBuilder.ts
+9
View File
@@ -66,6 +66,7 @@ import {
} from '../ingestion/processors/PlaceholderProcessor';
import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
import { LocationAnalyzer } from '../ingestion/types';
import { NextCatalogBuilder } from '../next';
export type CatalogEnvironment = {
logger: Logger;
@@ -103,6 +104,10 @@ export class CatalogBuilder {
private processorsReplace: boolean;
private parser: CatalogProcessorParser | undefined;
static async create(env: CatalogEnvironment): Promise<NextCatalogBuilder> {
return new NextCatalogBuilder(env);
}
constructor(env: CatalogEnvironment) {
this.env = env;
this.entityPolicies = [];
@@ -112,6 +117,10 @@ export class CatalogBuilder {
this.processors = [];
this.processorsReplace = false;
this.parser = undefined;
env.logger.warn(
"Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead",
);
}
/**
plugins/catalog-backend/src/service/router.ts
+37
View File
@@ -28,6 +28,7 @@ import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types';
import { LocationService } from '../next/types';
import {
basicEntityFilter,
parseEntityFilterParams,
@@ -45,6 +46,7 @@ export interface RouterOptions {
locationsCatalog?: LocationsCatalog;
higherOrderOperation?: HigherOrderOperation;
locationAnalyzer?: LocationAnalyzer;
locationService?: LocationService;
logger: Logger;
config: Config;
}
@@ -57,6 +59,7 @@ export async function createRouter(
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
locationService,
config,
logger,
} = options;
@@ -145,6 +148,40 @@ export async function createRouter(
});
}
if (locationService) {
router
.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);
const dryRun = yn(req.query.dryRun, { default: false });
// when in dryRun addLocation is effectively a read operation so we don't
// need to disallow readonly
if (!dryRun) {
disallowReadonlyMode(readonlyEnabled);
}
const output = await locationService.createLocation(input, dryRun);
res.status(201).json(output);
})
.get('/locations', async (_req, res) => {
const locations = await locationService.listLocations();
res.status(200).json(locations.map(l => ({ data: l })));
})
.get('/locations/:id', async (req, res) => {
const { id } = req.params;
const output = await locationService.getLocation(id);
res.status(200).json(output);
})
.delete('/locations/:id', async (req, res) => {
disallowReadonlyMode(readonlyEnabled);
const { id } = req.params;
await locationService.deleteLocation(id);
res.status(204).end();
});
}
if (higherOrderOperation) {
router.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);