paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 34 Packages Projects Releases Wiki Activity

Merge pull request #5901 from backstage/jhaals/legacy-catalog

Browse Source 
Catalog: Make the new processing engine the default
This commit is contained in:
Johan Haals
2021-06-15 11:40:53 +02:00
committed by GitHub
parent e2261dbec1 0307a28b4a
commit a0f2f63374
26 changed files with 111 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/dull-poets-learn.md
+52
View File
@@ -0,0 +1,52 @@
---
'@backstage/plugin-catalog-backend': patch
'@backstage/create-app': patch
---
This release enables the new catalog processing engine which is a major milestone for the catalog!
This update makes processing more scalable across multiple instances, adds support for deletions and ui flagging of entities that are no longer referenced by a location.
**Changes Required** to `catalog.ts`
```diff
-import { useHotCleanup } from '@backstage/backend-common';
import {
CatalogBuilder,
- createRouter,
- runPeriodically
+ createRouter
} from '@backstage/plugin-catalog-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createPlugin(env: PluginEnvironment): Promise<Router> {
- const builder = new CatalogBuilder(env);
+ const builder = await CatalogBuilder.create(env);
const {
entitiesCatalog,
locationsCatalog,
- higherOrderOperation,
+ locationService,
+ processingEngine,
locationAnalyzer,
} = await builder.build();
- useHotCleanup(
- module,
- runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
- );
+ await processingEngine.start();
return await createRouter({
entitiesCatalog,
locationsCatalog,
- higherOrderOperation,
+ locationService,
locationAnalyzer,
logger: env.logger,
config: env.config,
```
As this is a major internal change we have taken some precaution by still allowing the old catalog to be enabled by keeping your `catalog.ts` in it's current state.
If you encounter any issues and have to revert to the previous catalog engine make sure to raise an issue immediately as the old catalog engine is deprecated and will be removed in a future release.
packages/backend/src/plugins/catalog.ts
+5 -39
View File
@@ -14,13 +14,9 @@
* limitations under the License.
*/
import { useHotCleanup } from '@backstage/backend-common';
import {
CatalogBuilder,
createRouter,
NextCatalogBuilder,
runPeriodically,
createNextRouter,
} from '@backstage/plugin-catalog-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
@@ -28,50 +24,20 @@ import { PluginEnvironment } from '../types';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
/*
* ** WARNING **
* DO NOT enable the experimental catalog, it will brick your database migrations.
* This is solely for internal backstage development.
*/
if (process.env.EXPERIMENTAL_CATALOG === '1') {
const builder = new NextCatalogBuilder(env);
const {
entitiesCatalog,
locationAnalyzer,
processingEngine,
locationService,
} = await builder.build();
// TODO(jhaals): run and manage in background.
await processingEngine.start();
return await createNextRouter({
entitiesCatalog,
locationAnalyzer,
locationService,
logger: env.logger,
config: env.config,
});
}
const builder = new CatalogBuilder(env);
const builder = await CatalogBuilder.create(env);
const {
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
processingEngine,
locationService,
} = await builder.build();
useHotCleanup(
module,
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
);
await processingEngine.start();
return await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
locationService,
logger: env.logger,
config: env.config,
});
packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts
+6 -10
View File
@@ -1,30 +1,26 @@
import { useHotCleanup } from '@backstage/backend-common';
import {
CatalogBuilder,
createRouter,
runPeriodically
createRouter
} from '@backstage/plugin-catalog-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createPlugin(env: PluginEnvironment): Promise<Router> {
const builder = new CatalogBuilder(env);
const builder = await CatalogBuilder.create(env);
const {
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationService,
processingEngine,
locationAnalyzer,
} = await builder.build();
useHotCleanup(
module,
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
);
await processingEngine.start();
return await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
locationService,
locationAnalyzer,
logger: env.logger,
config: env.config,
plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js → plugins/catalog-backend/migrations/20210302150147_refresh_state.js
View File
plugins/catalog-backend/migrationsv2/20200511113813_init.js
-133
View File
@@ -1,133 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
return (
knex.schema
//
// locations
//
.createTable('locations', table => {
table.comment(
'Registered locations that shall be contiuously scanned for catalog item updates',
);
table
.uuid('id')
.primary()
.notNullable()
.comment('Auto-generated ID of the location');
table.string('type').notNullable().comment('The type of location');
table
.string('target')
.notNullable()
.comment('The actual target of the location');
})
//
// entities
//
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.string('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.string('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
})
//
// entities_search
//
.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
})
);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema
.dropTable('entities_search')
.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
})
.dropTable('entities')
.dropTable('locations');
};
plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js
-43
View File
@@ -1,43 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
return knex.schema.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema.dropTableIfExists('location_update_log');
};
plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Get list sorted by created_at timestamp in descending order
// Grouped by location_id
return knex.schema.raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
return knex.schema.raw(`DROP VIEW location_update_log_latest;`);
};
plugins/catalog-backend/migrationsv2/20200702153613_entities.js
-236
View File
@@ -1,236 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// SQLite does not support FK and PK
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.dropForeign(['entity_id']);
});
await knex.schema.alterTable('entities', table => {
table.dropPrimary('entities_pkey');
});
}
await knex.schema.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
});
// Setup temporary tables
await knex.schema.renameTable('entities_search', 'tmp_entities_search');
await knex.schema.renameTable('entities', 'tmp_entities');
//
// entities
//
await knex.schema
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.text('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.text('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
});
await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`);
//
// entities_search
//
await knex.schema.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
});
await knex.schema.raw(
`INSERT INTO entities_search SELECT * FROM tmp_entities_search`,
);
// Clean up
await knex.schema.dropTable('tmp_entities');
return knex.schema.dropTable('tmp_entities_search');
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
// SQLite does not support FK and PK
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.dropForeign(['entity_id']);
});
await knex.schema.alterTable('entities', table => {
table.dropPrimary('entities_pkey');
});
}
await knex.schema.alterTable('entities', table => {
table.dropUnique([], 'entities_unique_name');
});
// Setup temporary tables
await knex.schema.renameTable('entities_search', 'tmp_entities_search');
await knex.schema.renameTable('entities', 'tmp_entities');
//
// entities
//
await knex.schema
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('etag')
.notNullable()
.comment(
'An opaque string that changes for each update operation to any part of the entity, including metadata.',
);
table
.string('generation')
.notNullable()
.unsigned()
.comment(
'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.',
);
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.string('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.string('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
});
await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`);
//
// entities_search
//
await knex.schema.createTable('entities_search', table => {
table.comment(
'Flattened key-values from the entities, used for quick filtering',
);
table
.uuid('entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.comment('The entity that matches this key/value');
table
.string('key')
.notNullable()
.comment('A key that occurs in the entity');
table
.string('value')
.nullable()
.comment('The corresponding value to match on');
});
await knex.schema.raw(
`INSERT INTO entities_search SELECT * FROM tmp_entities_search`,
);
// Clean up
await knex.schema.dropTable('tmp_entities');
return knex.schema.dropTable('tmp_entities_search');
};
plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js
-44
View File
@@ -1,44 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = function up(knex) {
return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = function down(knex) {
knex.schema.raw(`DROP VIEW location_update_log_latest;`);
};
plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js
-87
View File
@@ -1,87 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = function up(knex) {
return knex.schema
.raw('DROP VIEW location_update_log_latest;')
.dropTable('location_update_log')
.createTable('location_update_log', table => {
table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = function down(knex) {
return knex.schema
.raw('DROP VIEW location_update_log_latest;')
.dropTable('location_update_log')
.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('entity_name').nullable();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(created_at) AS MAXDATE
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.created_at = t2.MAXDATE
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
};
plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js
-41
View File
@@ -1,41 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Sqlite does not support alter column.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.text('value').nullable().alter();
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
// Sqlite does not support alter column.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_search', table => {
table.string('value').nullable().alter();
});
}
};
plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Adds a single 'bootstrap' location that can be used to trigger work in processors.
// This is primarily here to fulfill foreign key constraints.
await knex('locations').insert({
id: require('uuid').v4(),
type: 'bootstrap',
target: 'bootstrap',
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex('locations')
.where({
type: 'bootstrap',
target: 'bootstrap',
})
.del();
};
plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js
-32
View File
@@ -1,32 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex('entities')
.where({ namespace: null })
.update({ namespace: 'default' });
await knex('entities_search').update({
key: knex.raw('LOWER(key)'),
value: knex.raw('LOWER(value)'),
});
};
exports.down = async function down() {};
plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js
-60
View File
@@ -1,60 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table.text('full_name').nullable();
});
await knex('entities').update({
full_name: knex.raw(
"LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)",
),
});
// SQLite does not support alter column
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.text('full_name').notNullable().alter();
});
}
await knex.schema.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.unique(['full_name'], 'entities_unique_full_name');
table.dropUnique([], 'entities_unique_name');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
table.dropUnique([], 'entities_unique_full_name');
table.unique(['kind', 'namespace', 'name'], 'entities_unique_name');
});
await knex.schema.alterTable('entities_search', table => {
table.dropColumn('full_name');
});
};
plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js
-66
View File
@@ -1,66 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table
.text('data')
.nullable()
.comment('The entire JSON data blob of the entity');
});
await knex('entities').update({
// apiVersion and kind should not contain any JSON unsafe chars, and both
// metadata and spec are already valid serialized JSON
data: knex.raw(
`'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`,
),
});
await knex.schema.alterTable('entities', table => {
table.dropColumn('metadata');
table.dropColumn('spec');
});
// SQLite does not support ALTER COLUMN.
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.text('data').notNullable().alter();
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
table
.text('metadata')
.notNullable()
.comment('The entire metadata JSON blob of the entity');
table
.text('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
table.dropColumn('data');
});
};
plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js
-50
View File
@@ -1,50 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities', table => {
table.dropColumn('api_version');
table.dropColumn('kind');
table.dropColumn('name');
table.dropColumn('namespace');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities', table => {
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table.string('kind').notNullable().comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
});
};
plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js
-37
View File
@@ -1,37 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.alterTable('entities_search', table => {
table.index(['key'], 'entities_search_key');
table.index(['value'], 'entities_search_value');
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('entities_search', table => {
table.dropIndex('', 'entities_search_key');
table.dropIndex('', 'entities_search_value');
});
};
plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js
-54
View File
@@ -1,54 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
await knex.schema.dropTable('entities_relations');
};
plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js
-93
View File
@@ -1,93 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client === 'sqlite3') {
// sqlite doesn't support dropPrimary so we recreate it properly instead
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.index('source_full_name', 'source_full_name_idx');
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropPrimary();
table.index('source_full_name', 'source_full_name_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client === 'sqlite3') {
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'source_full_name_idx');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
}
};
plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_relations', table => {
table.index('originating_entity_id', 'originating_entity_id_idx');
});
await knex.schema.alterTable('entities_search', table => {
table.index('entity_id', 'entity_id_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'originating_entity_id_idx');
});
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'entity_id_idx');
});
}
};
plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js
-73
View File
@@ -1,73 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
// We actually just want to widen columns, but can't do that while a
// view is dependent on them - so we just reconstruct it exactly as it was
await knex.schema
.raw('DROP VIEW location_update_log_latest;')
.alterTable('location_update_log', table => {
table.text('message').alter();
table.text('entity_name').nullable().alter();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema
.raw('DROP VIEW location_update_log_latest;')
.alterTable('location_update_log', table => {
table.string('message').alter();
table.string('entity_name').nullable().alter();
}).raw(`
CREATE VIEW location_update_log_latest AS
SELECT t1.* FROM location_update_log t1
JOIN
(
SELECT location_id, MAX(id) AS MAXID
FROM location_update_log
GROUP BY location_id
) t2
ON t1.location_id = t2.location_id
AND t1.id = t2.MAXID
GROUP BY t1.location_id, t1.id
ORDER BY created_at DESC;
`);
}
};
plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.index('location_id', 'entity_location_id_idx');
});
await knex.schema.alterTable('location_update_log', table => {
table.index('location_id', 'update_log_location_id_idx');
});
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client !== 'sqlite3') {
await knex.schema.alterTable('entities', table => {
table.dropIndex([], 'entity_location_id_idx');
});
await knex.schema.alterTable('location_update_log', table => {
table.dropIndex([], 'update_log_location_id_idx');
});
}
};
plugins/catalog-backend/src/next/NextCatalogBuilder.ts
+1 -1
View File
@@ -257,7 +257,7 @@ export class NextCatalogBuilder {
await dbClient.migrate.latest({
directory: resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrationsv2',
'migrations',
),
});
plugins/catalog-backend/src/next/database/DatabaseManager.ts
+1 -1
View File
@@ -36,7 +36,7 @@ export class DatabaseManager {
): Promise<Database> {
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
'migrationsv2',
'migrations',
);
await knex.migrate.latest({
plugins/catalog-backend/src/service/CatalogBuilder.ts
+9
View File
@@ -66,6 +66,7 @@ import {
} from '../ingestion/processors/PlaceholderProcessor';
import { defaultEntityDataParser } from '../ingestion/processors/util/parse';
import { LocationAnalyzer } from '../ingestion/types';
import { NextCatalogBuilder } from '../next';
export type CatalogEnvironment = {
logger: Logger;
@@ -103,6 +104,10 @@ export class CatalogBuilder {
private processorsReplace: boolean;
private parser: CatalogProcessorParser | undefined;
static async create(env: CatalogEnvironment): Promise<NextCatalogBuilder> {
return new NextCatalogBuilder(env);
}
constructor(env: CatalogEnvironment) {
this.env = env;
this.entityPolicies = [];
@@ -112,6 +117,10 @@ export class CatalogBuilder {
this.processors = [];
this.processorsReplace = false;
this.parser = undefined;
env.logger.warn(
"Creating the catalog with 'new CatalogBuilder(env)' is deprecated! Use CatalogBuilder.create(env) instead",
);
}
/**
plugins/catalog-backend/src/service/router.ts
+37
View File
@@ -28,6 +28,7 @@ import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types';
import { LocationService } from '../next/types';
import {
basicEntityFilter,
parseEntityFilterParams,
@@ -45,6 +46,7 @@ export interface RouterOptions {
locationsCatalog?: LocationsCatalog;
higherOrderOperation?: HigherOrderOperation;
locationAnalyzer?: LocationAnalyzer;
locationService?: LocationService;
logger: Logger;
config: Config;
}
@@ -57,6 +59,7 @@ export async function createRouter(
locationsCatalog,
higherOrderOperation,
locationAnalyzer,
locationService,
config,
logger,
} = options;
@@ -145,6 +148,40 @@ export async function createRouter(
});
}
if (locationService) {
router
.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);
const dryRun = yn(req.query.dryRun, { default: false });
// when in dryRun addLocation is effectively a read operation so we don't
// need to disallow readonly
if (!dryRun) {
disallowReadonlyMode(readonlyEnabled);
}
const output = await locationService.createLocation(input, dryRun);
res.status(201).json(output);
})
.get('/locations', async (_req, res) => {
const locations = await locationService.listLocations();
res.status(200).json(locations.map(l => ({ data: l })));
})
.get('/locations/:id', async (req, res) => {
const { id } = req.params;
const output = await locationService.getLocation(id);
res.status(200).json(output);
})
.delete('/locations/:id', async (req, res) => {
disallowReadonlyMode(readonlyEnabled);
const { id } = req.params;
await locationService.deleteLocation(id);
res.status(204).end();
});
}
if (higherOrderOperation) {
router.post('/locations', async (req, res) => {
const input = await validateRequestBody(req, locationSpecSchema);