From a3f090b2f391d47c9a36e8aeb53041798a9e02ac Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 21 Apr 2021 19:58:02 +0200 Subject: [PATCH 01/19] catalog-backend: WIP EntityProvider mutation interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Signed-off-by: Patrik Oldsberg --- .../next/DefaultCatalogProcessingEngine.ts | 15 ++-- .../src/next/DefaultLocationStore.ts | 68 ++++++++++--------- .../src/next/NextCatalogBuilder.ts | 3 +- plugins/catalog-backend/src/next/types.ts | 21 +++--- 4 files changed, 55 insertions(+), 52 deletions(-) diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 934ab3a1cc..b423b61041 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -19,6 +19,8 @@ import { CatalogProcessingEngine, EntityProvider, EntityMessage, + EntityProviderConnection, + EntityProviderMutation, ProcessingStateManager, CatalogProcessingOrchestrator, } from './types'; @@ -27,8 +29,13 @@ import { Logger } from 'winston'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { Stitcher } from './Stitcher'; +class Connection implements EntityProviderConnection { + constructor(private readonly stateManager: ProcessingStateManager) {} + + async applyMutation(mutation: EntityProviderMutation): Promise {} +} + export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { - private subscriptions: Subscription[] = []; private running: boolean = false; constructor( @@ -41,11 +48,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async start() { for (const provider of this.entityProviders) { - const id = 'databaseProvider'; - const subscription = provider - .entityChange$() - .subscribe({ next: m => this.onNext(id, m) }); - this.subscriptions.push(subscription); + provider.connect(new Connection(this.stateManager)); } this.running = true; diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index b21fe9da02..a6b4adcca4 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -16,24 +16,29 @@ import { LocationSpec, Location } from '@backstage/catalog-model'; import { Database } from '../database'; -import { LocationStore } from './types'; +import { + LocationStore, + EntityProvider, + EntityProviderConnection, +} from './types'; import { v4 as uuidv4 } from 'uuid'; +import { locationSpecToLocationEntity } from './util'; import { ConflictError } from '@backstage/errors'; -import { Observable } from '@backstage/core'; -import ObservableImpl from 'zen-observable'; export type LocationMessage = | { all: Location[] } | { added: Location[]; removed: Location[] }; -export class DefaultLocationStore implements LocationStore { - private subscribers = new Set< - ZenObservable.SubscriptionObserver - >(); +export class DefaultLocationStore implements LocationStore, EntityProvider { + private _connection: EntityProviderConnection | undefined; constructor(private readonly db: Database) {} createLocation(spec: LocationSpec): Promise { + if (!this.connection) { + throw new Error('location store is not initialized'); + } + return this.db.transaction(async tx => { // TODO: id should really be type and target combined and not a uuid. @@ -55,7 +60,11 @@ export class DefaultLocationStore implements LocationStore { target: spec.target, }); - this.notifyAddition(location); + await this.connection.applyMutation({ + type: 'delta', + added: [locationSpecToLocationEntity(location)], + removed: [], + }); return location; }); @@ -75,40 +84,33 @@ export class DefaultLocationStore implements LocationStore { } deleteLocation(id: string): Promise { + if (!this.connection) { + throw new Error('location store is not initialized'); + } + return this.db.transaction(async tx => { const location = await this.db.location(id); if (!location) { throw new ConflictError(`No location found with id: ${id}`); } await this.db.removeLocation(tx, id); - this.notifyDeletion(location); - }); - } - - private notifyAddition(location: Location) { - for (const subscriber of this.subscribers) { - subscriber.next({ - added: [location], - removed: [], - }); - } - } - - private notifyDeletion(location: Location) { - for (const subscriber of this.subscribers) { - subscriber.next({ + await this.connection.applyMutation({ + type: 'delta', added: [], - removed: [location], + removed: [locationSpecToLocationEntity(location)], }); - } + }); } - location$(): Observable { - return new ObservableImpl(subscriber => { - this.subscribers.add(subscriber); - return () => { - this.subscribers.delete(subscriber); - }; - }); + private get connection(): EntityProviderConnection { + if (!this._connection) { + throw new Error('location store is not initialized'); + } + + return this._connection; + } + + async connect(connection: EntityProviderConnection): Promise { + this._connection = connection; } } diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index c3e0a6bd9c..1796847164 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -272,11 +272,10 @@ export class NextCatalogBuilder { const entitiesCatalog = new NextEntitiesCatalog(dbClient); const locationStore = new DefaultLocationStore(db); - const dbLocationProvider = new DatabaseLocationProvider(locationStore); const stitcher = new Stitcher(dbClient, logger); const processingEngine = new DefaultCatalogProcessingEngine( logger, - [dbLocationProvider], // entityproviders + [locationStore], // entityproviders stateManager, orchestrator, stitcher, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 0aeb290224..c3854d631f 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, EntityName, @@ -21,7 +22,6 @@ import { EntityRelationSpec, } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; -import { Observable } from '@backstage/core'; // << nooo export interface LocationEntity { apiVersion: 'backstage.io/v1alpha1'; @@ -45,20 +45,11 @@ export interface LocationService { deleteLocation(id: string): Promise; } -export type EntityMessage = - | { all: Entity[] } - | { added: Entity[]; removed: EntityName[] }; - export interface LocationStore { - // extends EntityProvider createLocation(spec: LocationSpec): Promise; listLocations(): Promise; getLocation(id: string): Promise; deleteLocation(id: string): Promise; - - location$(): Observable< - { all: Location[] } | { added: Location[]; removed: Location[] } - >; } export interface CatalogProcessingEngine { @@ -66,8 +57,16 @@ export interface CatalogProcessingEngine { stop(): Promise; } +export type EntityProviderMutation = + | { type: 'full'; entities: Iterable } + | { type: 'delta'; added: Iterable; removed: Iterable }; + +export interface EntityProviderConnection { + applyMutation(mutation: EntityProviderMutation): Promise; +} + export interface EntityProvider { - entityChange$(): Observable; + connect(connection: EntityProviderConnection): Promise; } export type EntityProcessingRequest = { From 92a4a4832beccc622f83473328555f03d094bba7 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 22 Apr 2021 19:41:40 +0200 Subject: [PATCH 02/19] chore: worked on some more of the new catalog migration with fixing deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Oldsberg Co-authored-by: Johan Haals Co-authored-by: Fredrik Adelöw Signed-off-by: blam --- .../20210302150147_refresh_state.js | 33 ++-- .../next/DefaultCatalogProcessingEngine.ts | 38 +++- .../src/next/DefaultLocationStore.ts | 4 - .../src/next/DefaultProcessingStateManager.ts | 13 +- .../src/next/database/DatabaseManager.ts | 114 +++++++++++ .../DefaultProcessingDatabase.test.ts | 31 +++ .../database/DefaultProcessingDatabase.ts | 180 +++++++++++++++++- .../src/next/database/types.ts | 17 ++ plugins/catalog-backend/src/next/types.ts | 21 +- 9 files changed, 405 insertions(+), 46 deletions(-) create mode 100644 plugins/catalog-backend/src/next/database/DatabaseManager.ts create mode 100644 plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 6abab6270b..14c75b2530 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -59,13 +59,14 @@ exports.up = async function up(knex) { .notNullable() .comment('JSON array containing all errors related to entity'); table - .dateTime('next_update_at') + .dateTime('next_update_at') // TOOD: timezone or change to epoch-millis or similar .notNullable() .comment('Timestamp of when entity should be updated'); table - .dateTime('last_discovery_at') + .dateTime('last_discovery_at') // TOOD: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp of which this entity was discovered'); + table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); }); @@ -94,38 +95,35 @@ exports.up = async function up(knex) { 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', ); table - .text('source_special_key') + .text('source_key') .nullable() .comment( 'When the reference source is not an entity, this is an opaque identifier for that source.', ); table - .text('source_entity_id') + .text('source_entity_ref') .nullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment( - 'When the reference source is an entity, this is the ID of the source entity.', + 'When the reference source is an entity, this is the EntityRef of the source entity.', ); table - .text('target_entity_id') + .text('target_entity_ref') .notNullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') - .comment('The ID of the target entity.'); + .comment('The EntityRef of the target entity.'); + table.index('source_key', 'refresh_state_references_source_key_idx'); table.index( - 'source_special_key', - 'refresh_state_references_source_special_key_idx', + 'source_entity_ref', + 'refresh_state_references_source_entity_ref_idx', ); table.index( - 'source_entity_id', - 'refresh_state_references_source_entity_id_idx', - ); - table.index( - 'target_entity_id', - 'refresh_state_references_target_entity_id_idx', + 'target_entity_ref', + 'refresh_state_references_target_entity_ref_idx', ); }); @@ -165,6 +163,7 @@ exports.down = async function down(knex) { table.dropIndex([], 'refresh_state_references_target_entity_id_idx'); }); await knex.schema.alterTable('refresh_state', table => { + table.dropIndex([], 'refresh_state_entity_id_idx'); table.dropIndex([], 'refresh_state_entity_ref_idx'); table.dropIndex([], 'refresh_state_next_update_at_idx'); }); diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index b423b61041..7f49bf832d 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -30,9 +30,31 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; import { Stitcher } from './Stitcher'; class Connection implements EntityProviderConnection { - constructor(private readonly stateManager: ProcessingStateManager) {} + constructor( + private readonly config: { + stateManager: ProcessingStateManager; + id: string; + }, + ) {} - async applyMutation(mutation: EntityProviderMutation): Promise {} + async applyMutation(mutation: EntityProviderMutation): Promise { + if (mutation.type === 'full') { + await this.config.stateManager.replaceProcessingItems({ + id: this.config.id, + type: 'full', + items: mutation.entities, + }); + + return; + } + + await this.config.stateManager.replaceProcessingItems({ + id: this.config.id, + type: 'delta', + added: mutation.added, + removed: mutation.removed, + }); + } } export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { @@ -48,7 +70,9 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async start() { for (const provider of this.entityProviders) { - provider.connect(new Connection(this.stateManager)); + // TODO: this ID should be some form of identifier for the EntityProvider + const id = 'databaseProvider'; + provider.connect(new Connection({ stateManager: this.stateManager, id })); } this.running = true; @@ -57,12 +81,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { const { id, entity, - state: intialState, + state: initialState, } = await this.stateManager.getNextProcessingItem(); const result = await this.orchestrator.process({ entity, - state: intialState, + state: initialState, }); for (const error of result.errors) { @@ -95,10 +119,6 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async stop() { this.running = false; - - for (const subscription of this.subscriptions) { - subscription.unsubscribe(); - } } private async onNext(id: string, message: EntityMessage) { diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index a6b4adcca4..dff424b1a9 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -35,10 +35,6 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { constructor(private readonly db: Database) {} createLocation(spec: LocationSpec): Promise { - if (!this.connection) { - throw new Error('location store is not initialized'); - } - return this.db.transaction(async tx => { // TODO: id should really be type and target combined and not a uuid. diff --git a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts index 91090bc139..d2765c3101 100644 --- a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts +++ b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts @@ -17,14 +17,23 @@ import { ProcessingDatabase, RefreshStateItem } from './database/types'; import { AddProcessingItemRequest, - ProccessingItem, + ProcessingItem, ProcessingItemResult, ProcessingStateManager, + ReplaceProcessingItemsRequest, } from './types'; export class DefaultProcessingStateManager implements ProcessingStateManager { constructor(private readonly db: ProcessingDatabase) {} + replaceProcessingItems( + request: ReplaceProcessingItemsRequest, + ): Promise { + return this.db.transaction(async tx => { + await this.db.replaceUnprocessedEntities(tx, request); + }); + } + async setProcessingItemResult(result: ProcessingItemResult) { return this.db.transaction(async tx => { await this.db.updateProcessedEntity(tx, { @@ -44,7 +53,7 @@ export class DefaultProcessingStateManager implements ProcessingStateManager { }); } - async getNextProcessingItem(): Promise { + async getNextProcessingItem(): Promise { const entities = await new Promise(resolve => this.popFromQueue(resolve), ); diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts new file mode 100644 index 0000000000..4c9e45950f --- /dev/null +++ b/plugins/catalog-backend/src/next/database/DatabaseManager.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import knexFactory, { Knex } from 'knex'; +import { v4 as uuidv4 } from 'uuid'; +import { Logger } from 'winston'; +import { CommonDatabase } from '../../database/CommonDatabase'; +import { Database } from '../../database/types'; +import fs from 'fs-extra'; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +export class DatabaseManager { + public static async createDatabase( + knex: Knex, + options: Partial = {}, + ): Promise { + const allMigrations = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrations', + ); + + const migrationsDir = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrationsv2', + ); + await fs.copy(allMigrations, migrationsDir); + + await knex.migrate.latest({ + directory: migrationsDir, + }); + const { logger } = { ...defaultOptions, ...options }; + return new CommonDatabase(knex, logger); + } + + public static async createInMemoryDatabase(): Promise { + const knex = await this.createInMemoryDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createInMemoryDatabaseConnection(): Promise { + const knex = knexFactory({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + return knex; + } + + public static async createTestDatabase(): Promise { + const knex = await this.createTestDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createTestDatabaseConnection(): Promise { + const config: Knex.Config = { + /* + client: 'pg', + connection: { + host: 'localhost', + user: 'postgres', + password: 'postgres', + }, + */ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }; + + let knex = knexFactory(config); + if (typeof config.connection !== 'string') { + const tempDbName = `d${uuidv4().replace(/-/g, '')}`; + await knex.raw(`CREATE DATABASE ${tempDbName};`); + knex = knexFactory({ + ...config, + connection: { + ...config.connection, + database: tempDbName, + }, + }); + } + + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + return knex; + } +} diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts new file mode 100644 index 0000000000..fd2e9e6750 --- /dev/null +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; +import { DatabaseManager } from './DatabaseManager'; +import { Knex } from 'knex'; + +describe('Default Processing Database', () => { + let db: Knex | undefined; + + beforeEach(async () => { + db = await DatabaseManager.createTestDatabaseConnection(); + await DatabaseManager.createDatabase(db); + }); + + it('should write some stuff', async () => { + await db('refresh_state_referencess').select(); + }); +}); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index cf2d4e121c..b799bd6284 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -15,6 +15,7 @@ */ import { ConflictError, NotFoundError } from '@backstage/errors'; +import { stringifyEntityRef, Entity } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { Transaction } from '../../database'; import lodash from 'lodash'; @@ -24,20 +25,21 @@ import { AddUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, GetProcessableEntitiesResult, + ReplaceUnprocessedEntitiesOptions, } from './types'; import type { Logger } from 'winston'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; + import { v4 as uuid } from 'uuid'; export type DbRefreshStateRow = { entity_id: string; entity_ref: string; unprocessed_entity: string; - processed_entity: string; - cache: string; + processed_entity?: string; + cache?: string; next_update_at: string; last_discovery_at: string; // remove? - errors: string; + errors?: string; }; export type DbRelationsRow = { @@ -47,10 +49,10 @@ export type DbRelationsRow = { type: string; }; -export type DbRefreshStateReferences = { - source_special_key?: string; - source_entity_id?: string; - target_entity_id: string; +export type DbRefreshStateReferencesRow = { + source_key?: string; + source_entity_ref?: string; + target_entity_ref: string; }; // The number of items that are sent per batch to the database layer, when @@ -128,6 +130,164 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } + async replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const entityIds = new Array(); + + if (options.type === 'full') { + const oldRefs = await tx( + 'refresh_state_references', + ) + .where({ source_key: options.sourceKey }) + .select('target_entity_ref') + .then(rows => rows.map(r => r.target_entity_ref)); + + const items = options.items.map(entity => ({ + entity, + ref: stringifyEntityRef(entity), + id: uuid(), + })); + + const oldRefsSet = new Set(oldRefs); + const newRefsSet = new Set(items.map(item => item.ref)); + const toAdd = items.filter(item => !oldRefsSet.has(item.ref)); + const toRemove = oldRefs.filter(ref => !newRefsSet.has(ref)); + + if (toRemove.length) { + // TODO(freben): Batch split, to not hit variable limits? + + // get all refs where source = any of the toRemove + // delete all state rows matching any of the toRemove + // revisit the targets of all of those refs + // verify if they have references, otherwise delete from refresh_state + + const current = [...toRemove]; + while (true) { + + tx.withRecursive('r', function refs() { + return tx.select({ }) + .from({ r1: 'refresh_state_references' }) + .where({ source_key: options.sourceKey }) + .whereIn('target_entity_ref', toRemove) + .unionAll(function recurse() { + return tx.select({ }) + .from({ r2: 'refresh_state_references' }) + .whereNotExists() + }) + }) + .select().from('refs'); + + const nextLayerToRemove = await tx( + 'refresh_state_references', + ) + .whereIn('source_entity_ref', toRemove) + .leftOuterJoin('refresh_state_references AS b', function f() { + + }) + + .whereNotNull('b.source_target_ref') + .select('target_entity_ref'); + + await tx('refresh_state') + .whereIn('entity_ref', toRemove) + .delete(); + + const refsThatStillHaveASourcePointingAtThe = await tx( + 'refresh_state_references', + ) + .whereIn('target_entity_ref', nextLayerTargetRefs) + .groupBy('target_entity_ref') + .select('target_entity_ref'); + + } + + + + + if (toAdd.length) { + const state: Knex.DbRecord[] = toAdd.map(item => ({ + entity_id: item.id, + entity_ref: item.ref, + unprocessed_entity: JSON.stringify(item.entity), + errors: '', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + })); + const stateReferences: DbRefreshStateReferencesRow[] = toAdd.map( + item => ({ + source_key: options.sourceKey, + target_entity_ref: item.ref, + }), + ); + // TODO(freben): Concurrency? If we did these one by one, a .onConflict().merge would have made sense + await tx.batchInsert('refresh_state', state, BATCH_SIZE); + await tx.batchInsert('refresh_state_references', stateReferences, BATCH_SIZE); + } + + + for (const entity of options.items) { + const entityRef = stringifyEntityRef(entity); + await tx('refresh_state') + .insert({ + entity_id: uuid(), + entity_ref: entityRef, + unprocessed_entity: JSON.stringify(entity), + errors: '', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }) + .onConflict('entity_ref') + .merge(['unprocessed_entity', 'last_discovery_at']); + + const [{ entity_id: entityId }] = await tx( + 'refresh_state', + ).where({ entity_ref: entityRef }); + entityIds.push(entityId); + } + const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( + entityId => ({ + source_key: options.sourceKey, + target_entityRef: entityRef, + }), + ); + await tx.batchInsert( + 'refresh_state_references', + referenceRows, + BATCH_SIZE, + ); + return; + } + + for (const entity of options.removed) { + const entityRef = stringifyEntityRef(entity); + const [result] = await tx('refresh_state') + .where({ entity_ref: entityRef }) + .select('entity_id'); + + if (!result) { + this.logger.info( + `Unable to delete entity '${entityRef}', entity does not exist in refresh state`, + ); + continue; + } + + const referenceResults = await tx( + 'refresh_state_references', + ) + // todo correct key? + .where({ source_entity_id: result.entity_id }) + .select(); + + await tx('refresh_state_references') + // todo correct key? + .where({ source_entity_id: result.entity_id }) + .delete(); + } + } + async addUnprocessedEntities( txOpaque: Transaction, options: AddUnprocessedEntitiesOptions, @@ -160,11 +320,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ? { source_special_key: options.id } : { source_entity_id: options.id }; // copied from update refs - await tx('refresh_state_references') + await tx('refresh_state_references') .where(key) .delete(); - const referenceRows: DbRefreshStateReferences[] = entityIds.map( + const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( entityId => ({ ...key, target_entity_id: entityId, diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 22e9e90c8e..b926676887 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -50,6 +50,19 @@ export type GetProcessableEntitiesResult = { items: RefreshStateItem[]; }; +export type ReplaceUnprocessedEntitiesOptions = + | { + sourceKey: string; + items: Entity[]; + type: 'full'; + } + | { + sourceKey: string; + added: Entity[]; + removed: Entity[]; + type: 'delta'; + }; + export interface ProcessingDatabase { transaction(fn: (tx: Transaction) => Promise): Promise; @@ -58,6 +71,10 @@ export interface ProcessingDatabase { options: AddUnprocessedEntitiesOptions, ): Promise; + replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise; getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index c3854d631f..f8b4de1c41 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -58,8 +58,8 @@ export interface CatalogProcessingEngine { } export type EntityProviderMutation = - | { type: 'full'; entities: Iterable } - | { type: 'delta'; added: Iterable; removed: Iterable }; + | { type: 'full'; entities: Entity[] } + | { type: 'delta'; added: Entity[]; removed: Entity[] }; export interface EntityProviderConnection { applyMutation(mutation: EntityProviderMutation): Promise; @@ -108,14 +108,27 @@ export type AddProcessingItemRequest = { entities: Entity[]; }; -export type ProccessingItem = { +export type ProcessingItem = { id: string; entity: Entity; state: Map; }; +export type ReplaceProcessingItemsRequest = + | { + id: string; + items: Entity[]; + type: 'full'; + } + | { + id: string; + added: Entity[]; + removed: Entity[]; + type: 'delta'; + }; export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; - getNextProcessingItem(): Promise; + getNextProcessingItem(): Promise; addProcessingItems(request: AddProcessingItemRequest): Promise; + replaceProcessingItems(request: ReplaceProcessingItemsRequest): Promise; } From f62db5992f7b9ba66dbcb291a9d88ab1264aca07 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 23 Apr 2021 15:05:04 +0200 Subject: [PATCH 03/19] feat: added support for full sync replace with an awesome sql statement courtesy of @freben MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: blam --- .../20210302150147_refresh_state.js | 3 + .../DefaultProcessingDatabase.test.ts | 271 +++++++++++++++++- .../database/DefaultProcessingDatabase.ts | 220 +++++++------- 3 files changed, 390 insertions(+), 104 deletions(-) diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 14c75b2530..2afeda02c0 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -94,6 +94,9 @@ exports.up = async function up(knex) { table.comment( 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', ); + table + .increments('id') + .comment('Primary key to distinguish unique lines from each other'); table .text('source_key') .nullable() diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index fd2e9e6750..ee04a2baf8 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -16,16 +16,283 @@ // import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { DatabaseManager } from './DatabaseManager'; import { Knex } from 'knex'; +import { + DbRefreshStateReferencesRow, + DbRefreshStateRow, + DefaultProcessingDatabase, +} from './DefaultProcessingDatabase'; + +import { Entity } from '@backstage/catalog-model'; +import * as uuid from 'uuid'; +import { getVoidLogger } from '@backstage/backend-common'; describe('Default Processing Database', () => { let db: Knex | undefined; + let processingDatabase: DefaultProcessingDatabase | undefined; + + const logger = getVoidLogger(); beforeEach(async () => { db = await DatabaseManager.createTestDatabaseConnection(); await DatabaseManager.createDatabase(db); + + processingDatabase = new DefaultProcessingDatabase(db!, logger); }); - it('should write some stuff', async () => { - await db('refresh_state_referencess').select(); + describe('replaceUnprocessedEntities', () => { + const insertRefRow = async (ref: DbRefreshStateReferencesRow) => { + return db!( + 'refresh_state_references', + ).insert(ref); + }; + + const insertRefreshStateRow = async (ref: DbRefreshStateRow) => { + await db!('refresh_state').insert(ref); + }; + + const createLocations = async (entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow({ + entity_id: uuid.v4(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '', + next_update_at: 'now()', + last_discovery_at: 'now()', + }); + } + }; + + it('replaces all existing state correctly for simple dependency chains', async () => { + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + database -> location:default/second -> location:default/root-2 + */ + await createLocations([ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/second', + ]); + + await insertRefRow({ + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow({ + source_key: 'database', + target_entity_ref: 'location:default/second', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/second', + target_entity_ref: 'location:default/root-2', + }); + + await processingDatabase!.transaction(async tx => { + await processingDatabase!.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db!( + 'refresh_state', + ).select(); + + const currentRefRowState = await db!( + 'refresh_state_references', + ).select(); + + for (const ref of ['location:default/root', 'location:default/root-1']) { + expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/root-1' && + t.source_key === 'config', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/new-root' && + t.source_key === 'config', + ), + ).toBeTruthy(); + }); + + it('should work for more complex chains', async () => { + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + config -> location:default/root -> location:default/root-1a -> location:default/root-2 + */ + await createLocations([ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/root-1a', + ]); + + await insertRefRow({ + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1a', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow({ + source_entity_ref: 'location:default/root-1a', + target_entity_ref: 'location:default/root-2', + }); + + await processingDatabase!.transaction(async tx => { + await processingDatabase!.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db!( + 'refresh_state', + ).select(); + + const currentRefRowState = await db!( + 'refresh_state_references', + ).select(); + + const deletedRefs = [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-1a', + 'location:default/root-2', + ]; + + for (const ref of deletedRefs) { + expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1a', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1a' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + }); }); }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index b799bd6284..f0c5443651 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -135,7 +135,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: ReplaceUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const entityIds = new Array(); if (options.type === 'full') { const oldRefs = await tx( @@ -158,55 +157,95 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { if (toRemove.length) { // TODO(freben): Batch split, to not hit variable limits? - - // get all refs where source = any of the toRemove - // delete all state rows matching any of the toRemove - // revisit the targets of all of those refs - // verify if they have references, otherwise delete from refresh_state - - const current = [...toRemove]; - while (true) { - - tx.withRecursive('r', function refs() { - return tx.select({ }) - .from({ r1: 'refresh_state_references' }) - .where({ source_key: options.sourceKey }) - .whereIn('target_entity_ref', toRemove) - .unionAll(function recurse() { - return tx.select({ }) - .from({ r2: 'refresh_state_references' }) - .whereNotExists() - }) + /* + WITH RECURSIVE + -- All the refs that can be reached from each individual root + root_reach(id, entity_ref) AS ( + -- Start with all roots + SELECT id, target_entity_ref + FROM refresh_state_references + WHERE source_key IS NOT NULL + UNION + -- For each match, select all children + SELECT root_reach.id, target_entity_ref + FROM refresh_state_references, root_reach + WHERE source_entity_ref = root_reach.entity_ref + ) + -- Start out with our own matching row (see the WHERE that + -- matches on source_key and target_entity_ref below) + SELECT us.entity_ref + FROM refresh_state_references + -- Expand the entire tree that emanates from that row + JOIN root_reach AS us + ON us.id = refresh_state_references.id + -- Expand with all roots that target the same node but + -- aren't ourselves + LEFT OUTER JOIN root_reach AS them + ON them.entity_ref = us.entity_ref + AND them.id != us.id + -- Keep only the matches that had no other rooots + WHERE refresh_state_references.source_key = "R1" + AND refresh_state_references.target_entity_ref = "A" + AND them.id IS NULL; + */ + const removedCount = await tx('refresh_state') + .whereIn('entity_ref', function orphanedEntityRefs(orphans) { + return ( + orphans + // All the refs that can be reached from each individual root + .withRecursive('root_reach', function rootReach(outer) { + // Start with all roots + return outer + .select({ id: 'id', entity_ref: 'target_entity_ref' }) + .from('refresh_state_references') + .whereNotNull('source_key') + .union(function recurse(inner) { + return ( + inner + // For each match, select all children + .select({ + id: 'root_reach.id', + entity_ref: + 'refresh_state_references.target_entity_ref', + }) + .from('refresh_state_references') + .crossJoin('root_reach', { + 'root_reach.entity_ref': + 'refresh_state_references.source_entity_ref', + }) + ); + }); + }) + .select('us.entity_ref') + // Start out with our own matching row + .from('refresh_state_references') + .where('source_key', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + // Expand the entire tree that emanates from that row + .leftJoin({ us: 'root_reach' }, function us() { + this.on('us.id', '=', 'refresh_state_references.id'); + }) + // Expand with all roots that target the same node but aren't ourselves + .leftOuterJoin({ them: 'root_reach' }, function them() { + this.on('them.entity_ref', '=', 'us.entity_ref'); + this.andOn('them.id', '!=', 'us.id'); + }) + // Keep only the matches that had no other rooots + .whereNull('them.id') + ); }) - .select().from('refs'); + .delete(); - const nextLayerToRemove = await tx( - 'refresh_state_references', - ) - .whereIn('source_entity_ref', toRemove) - .leftOuterJoin('refresh_state_references AS b', function f() { - - }) - - .whereNotNull('b.source_target_ref') - .select('target_entity_ref'); - - await tx('refresh_state') - .whereIn('entity_ref', toRemove) - .delete(); - - const refsThatStillHaveASourcePointingAtThe = await tx( - 'refresh_state_references', - ) - .whereIn('target_entity_ref', nextLayerTargetRefs) - .groupBy('target_entity_ref') - .select('target_entity_ref'); + await tx('refresh_state_references') + .where('source_key', '=', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + .delete(); + console.log( + `REMOVED, ${removedCount} entities: ${JSON.stringify(toRemove)}`, + ); } - - - if (toAdd.length) { const state: Knex.DbRecord[] = toAdd.map(item => ({ entity_id: item.id, @@ -224,67 +263,44 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); // TODO(freben): Concurrency? If we did these one by one, a .onConflict().merge would have made sense await tx.batchInsert('refresh_state', state, BATCH_SIZE); - await tx.batchInsert('refresh_state_references', stateReferences, BATCH_SIZE); - } - - - for (const entity of options.items) { - const entityRef = stringifyEntityRef(entity); - await tx('refresh_state') - .insert({ - entity_id: uuid(), - entity_ref: entityRef, - unprocessed_entity: JSON.stringify(entity), - errors: '', - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - }) - .onConflict('entity_ref') - .merge(['unprocessed_entity', 'last_discovery_at']); - - const [{ entity_id: entityId }] = await tx( - 'refresh_state', - ).where({ entity_ref: entityRef }); - entityIds.push(entityId); - } - const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( - entityId => ({ - source_key: options.sourceKey, - target_entityRef: entityRef, - }), - ); - await tx.batchInsert( - 'refresh_state_references', - referenceRows, - BATCH_SIZE, - ); - return; - } - - for (const entity of options.removed) { - const entityRef = stringifyEntityRef(entity); - const [result] = await tx('refresh_state') - .where({ entity_ref: entityRef }) - .select('entity_id'); - - if (!result) { - this.logger.info( - `Unable to delete entity '${entityRef}', entity does not exist in refresh state`, + await tx.batchInsert( + 'refresh_state_references', + stateReferences, + BATCH_SIZE, ); - continue; } - const referenceResults = await tx( - 'refresh_state_references', - ) - // todo correct key? - .where({ source_entity_id: result.entity_id }) - .select(); + // for (const entity of options.items) { + // const entityRef = stringifyEntityRef(entity); + // await tx('refresh_state') + // .insert({ + // entity_id: uuid(), + // entity_ref: entityRef, + // unprocessed_entity: JSON.stringify(entity), + // errors: '', + // next_update_at: tx.fn.now(), + // last_discovery_at: tx.fn.now(), + // }) + // .onConflict('entity_ref') + // .merge(['unprocessed_entity', 'last_discovery_at']); - await tx('refresh_state_references') - // todo correct key? - .where({ source_entity_id: result.entity_id }) - .delete(); + // const [{ entity_id: entityId }] = await tx( + // 'refresh_state', + // ).where({ entity_ref: entityRef }); + // entityIds.push(entityId); + // } + // const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( + // entityId => ({ + // source_key: options.sourceKey, + // target_entityRef: entityRef, + // }), + // ); + // await tx.batchInsert( + // 'refresh_state_references', + // referenceRows, + // BATCH_SIZE, + // ); + // return; } } From 6e545108e7418ee0bd9f57678f59f1a14e27f9be Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 23 Apr 2021 16:01:36 +0200 Subject: [PATCH 04/19] feat: support delta and full sync with the same code and simplify things MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: blam --- .../DefaultProcessingDatabase.test.ts | 89 +++++++ .../database/DefaultProcessingDatabase.ts | 242 ++++++++---------- 2 files changed, 201 insertions(+), 130 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index ee04a2baf8..c4784c081d 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -294,5 +294,94 @@ describe('Default Processing Database', () => { ), ).toBeFalsy(); }); + + it('should add new locations using the delta options', async () => { + await processingDatabase!.transaction(async tx => { + await processingDatabase!.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + removed: [], + added: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db!( + 'refresh_state', + ).select(); + + const currentRefRowState = await db!( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + }); + + it('should remove old locations using the delta options', async () => { + await createLocations(['location:default/new-root']); + + await insertRefRow({ + source_key: 'lols', + target_entity_ref: 'location:default/new-root', + }); + + await processingDatabase!.transaction(async tx => { + await processingDatabase!.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + added: [], + removed: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await db!( + 'refresh_state', + ).select(); + + const currentRefRowState = await db!( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + }); }); }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index f0c5443651..a80325d469 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -130,34 +130,48 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } + private async createDelta( + tx: Knex.Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise<{ toAdd: Entity[]; toRemove: string[] }> { + if (options.type === 'delta') { + return { + toAdd: options.added, + toRemove: options.removed.map(e => stringifyEntityRef(e)), + }; + } + + const oldRefs = await tx( + 'refresh_state_references', + ) + .where({ source_key: options.sourceKey }) + .select('target_entity_ref') + .then(rows => rows.map(r => r.target_entity_ref)); + + const items = options.items.map(entity => ({ + entity, + ref: stringifyEntityRef(entity), + })); + + const oldRefsSet = new Set(oldRefs); + const newRefsSet = new Set(items.map(item => item.ref)); + const toAdd = items.filter(item => !oldRefsSet.has(item.ref)); + const toRemove = oldRefs.filter(ref => !newRefsSet.has(ref)); + + return { toAdd: toAdd.map(({ entity }) => entity), toRemove }; + } + async replaceUnprocessedEntities( txOpaque: Transaction, options: ReplaceUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - if (options.type === 'full') { - const oldRefs = await tx( - 'refresh_state_references', - ) - .where({ source_key: options.sourceKey }) - .select('target_entity_ref') - .then(rows => rows.map(r => r.target_entity_ref)); + const { toAdd, toRemove } = await this.createDelta(tx, options); - const items = options.items.map(entity => ({ - entity, - ref: stringifyEntityRef(entity), - id: uuid(), - })); - - const oldRefsSet = new Set(oldRefs); - const newRefsSet = new Set(items.map(item => item.ref)); - const toAdd = items.filter(item => !oldRefsSet.has(item.ref)); - const toRemove = oldRefs.filter(ref => !newRefsSet.has(ref)); - - if (toRemove.length) { - // TODO(freben): Batch split, to not hit variable limits? - /* + if (toRemove.length) { + // TODO(freben): Batch split, to not hit variable limits? + /* WITH RECURSIVE -- All the refs that can be reached from each individual root root_reach(id, entity_ref) AS ( @@ -188,119 +202,87 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { AND refresh_state_references.target_entity_ref = "A" AND them.id IS NULL; */ - const removedCount = await tx('refresh_state') - .whereIn('entity_ref', function orphanedEntityRefs(orphans) { - return ( - orphans - // All the refs that can be reached from each individual root - .withRecursive('root_reach', function rootReach(outer) { - // Start with all roots - return outer - .select({ id: 'id', entity_ref: 'target_entity_ref' }) - .from('refresh_state_references') - .whereNotNull('source_key') - .union(function recurse(inner) { - return ( - inner - // For each match, select all children - .select({ - id: 'root_reach.id', - entity_ref: - 'refresh_state_references.target_entity_ref', - }) - .from('refresh_state_references') - .crossJoin('root_reach', { - 'root_reach.entity_ref': - 'refresh_state_references.source_entity_ref', - }) - ); - }); - }) - .select('us.entity_ref') - // Start out with our own matching row - .from('refresh_state_references') - .where('source_key', options.sourceKey) - .whereIn('target_entity_ref', toRemove) - // Expand the entire tree that emanates from that row - .leftJoin({ us: 'root_reach' }, function us() { - this.on('us.id', '=', 'refresh_state_references.id'); - }) - // Expand with all roots that target the same node but aren't ourselves - .leftOuterJoin({ them: 'root_reach' }, function them() { - this.on('them.entity_ref', '=', 'us.entity_ref'); - this.andOn('them.id', '!=', 'us.id'); - }) - // Keep only the matches that had no other rooots - .whereNull('them.id') - ); - }) - .delete(); + const removedCount = await tx('refresh_state') + .whereIn('entity_ref', function orphanedEntityRefs(orphans) { + return ( + orphans + // All the refs that can be reached from each individual root + .withRecursive('root_reach', function rootReach(outer) { + // Start with all roots + return outer + .select({ id: 'id', entity_ref: 'target_entity_ref' }) + .from('refresh_state_references') + .whereNotNull('source_key') + .union(function recurse(inner) { + return ( + inner + // For each match, select all children + .select({ + id: 'root_reach.id', + entity_ref: + 'refresh_state_references.target_entity_ref', + }) + .from('refresh_state_references') + .crossJoin('root_reach', { + 'root_reach.entity_ref': + 'refresh_state_references.source_entity_ref', + }) + ); + }); + }) + .select('us.entity_ref') + // Start out with our own matching row + .from('refresh_state_references') + .where('source_key', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + // Expand the entire tree that emanates from that row + .leftJoin({ us: 'root_reach' }, function us() { + this.on('us.id', '=', 'refresh_state_references.id'); + }) + // Expand with all roots that target the same node but aren't ourselves + .leftOuterJoin({ them: 'root_reach' }, function them() { + this.on('them.entity_ref', '=', 'us.entity_ref'); + this.andOn('them.id', '!=', 'us.id'); + }) + // Keep only the matches that had no other rooots + .whereNull('them.id') + ); + }) + .delete(); - await tx('refresh_state_references') - .where('source_key', '=', options.sourceKey) - .whereIn('target_entity_ref', toRemove) - .delete(); + await tx('refresh_state_references') + .where('source_key', '=', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + .delete(); - console.log( - `REMOVED, ${removedCount} entities: ${JSON.stringify(toRemove)}`, - ); - } + this.logger.debug( + `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, + ); + } - if (toAdd.length) { - const state: Knex.DbRecord[] = toAdd.map(item => ({ - entity_id: item.id, - entity_ref: item.ref, - unprocessed_entity: JSON.stringify(item.entity), - errors: '', - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - })); - const stateReferences: DbRefreshStateReferencesRow[] = toAdd.map( - item => ({ - source_key: options.sourceKey, - target_entity_ref: item.ref, - }), - ); - // TODO(freben): Concurrency? If we did these one by one, a .onConflict().merge would have made sense - await tx.batchInsert('refresh_state', state, BATCH_SIZE); - await tx.batchInsert( - 'refresh_state_references', - stateReferences, - BATCH_SIZE, - ); - } + if (toAdd.length) { + const state: Knex.DbRecord[] = toAdd.map(entity => ({ + entity_id: uuid(), + entity_ref: stringifyEntityRef(entity), + unprocessed_entity: JSON.stringify(entity), + errors: '', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + })); - // for (const entity of options.items) { - // const entityRef = stringifyEntityRef(entity); - // await tx('refresh_state') - // .insert({ - // entity_id: uuid(), - // entity_ref: entityRef, - // unprocessed_entity: JSON.stringify(entity), - // errors: '', - // next_update_at: tx.fn.now(), - // last_discovery_at: tx.fn.now(), - // }) - // .onConflict('entity_ref') - // .merge(['unprocessed_entity', 'last_discovery_at']); - - // const [{ entity_id: entityId }] = await tx( - // 'refresh_state', - // ).where({ entity_ref: entityRef }); - // entityIds.push(entityId); - // } - // const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( - // entityId => ({ - // source_key: options.sourceKey, - // target_entityRef: entityRef, - // }), - // ); - // await tx.batchInsert( - // 'refresh_state_references', - // referenceRows, - // BATCH_SIZE, - // ); - // return; + const stateReferences: DbRefreshStateReferencesRow[] = toAdd.map( + entity => ({ + source_key: options.sourceKey, + target_entity_ref: stringifyEntityRef(entity), + }), + ); + // TODO(freben): Concurrency? If we did these one by one, a .onConflict().merge would have made sense + await tx.batchInsert('refresh_state', state, BATCH_SIZE); + await tx.batchInsert( + 'refresh_state_references', + stateReferences, + BATCH_SIZE, + ); } } From b8e80eab23b8e4aadd79b75de16a42839c6e4c45 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 10:14:45 +0200 Subject: [PATCH 05/19] catalog-backend: Remove obervables Signed-off-by: Johan Haals --- plugins/catalog-backend/package.json | 4 +- .../src/next/DatabaseLocationProvider.ts | 71 ------------------- .../next/DefaultCatalogProcessingEngine.ts | 27 +------ .../src/next/DefaultLocationStore.ts | 4 -- plugins/catalog-backend/src/next/types.ts | 5 +- 5 files changed, 5 insertions(+), 106 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/DatabaseLocationProvider.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9afb537637..5f70ec60e1 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -33,7 +33,6 @@ "@backstage/backend-common": "^0.6.3", "@backstage/catalog-model": "^0.7.7", "@backstage/config": "^0.1.4", - "@backstage/core": "^0.7.6", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", "@backstage/plugin-search-backend-node": "^0.1.3", @@ -61,8 +60,7 @@ "winston": "^3.2.1", "yaml": "^1.9.2", "yn": "^4.0.0", - "yup": "^0.29.3", - "zen-observable": "^0.8.15" + "yup": "^0.29.3" }, "devDependencies": { "@backstage/cli": "^0.6.9", diff --git a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts deleted file mode 100644 index eaf5a78556..0000000000 --- a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Observable } from '@backstage/core'; -import { ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { EntityProvider, LocationStore, EntityMessage } from './types'; -import ObservableImpl from 'zen-observable'; -import { - locationSpecToLocationEntity, - locationSpecToMetadataName, -} from './util'; - -export class DatabaseLocationProvider implements EntityProvider { - private subscribers = new Set< - ZenObservable.SubscriptionObserver - >(); - - constructor(private readonly store: LocationStore) { - store.location$().subscribe({ - next: locations => { - if ('all' in locations) { - this.notify({ - all: locations.all.map(l => locationSpecToLocationEntity(l)), - }); - } else { - this.notify({ - added: locations.added.map(l => locationSpecToLocationEntity(l)), - removed: locations.removed.map(l => ({ - kind: 'Location', - namespace: ENTITY_DEFAULT_NAMESPACE, - name: locationSpecToMetadataName(l), - })), - }); - } - }, - }); - } - - private notify(message: EntityMessage) { - for (const subscriber of this.subscribers) { - subscriber.next(message); - } - } - - entityChange$(): Observable { - return new ObservableImpl(subscriber => { - this.store.listLocations().then(locations => { - subscriber.next({ - all: locations.map(l => locationSpecToLocationEntity(l)), - }); - this.subscribers.add(subscriber); - }); - return () => { - this.subscribers.delete(subscriber); - }; - }); - } -} diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 7f49bf832d..b71b2c1045 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -14,11 +14,9 @@ * limitations under the License. */ -import { Subscription } from '@backstage/core'; import { CatalogProcessingEngine, EntityProvider, - EntityMessage, EntityProviderConnection, EntityProviderMutation, ProcessingStateManager, @@ -40,7 +38,7 @@ class Connection implements EntityProviderConnection { async applyMutation(mutation: EntityProviderMutation): Promise { if (mutation.type === 'full') { await this.config.stateManager.replaceProcessingItems({ - id: this.config.id, + sourceKey: this.config.id, type: 'full', items: mutation.entities, }); @@ -49,7 +47,7 @@ class Connection implements EntityProviderConnection { } await this.config.stateManager.replaceProcessingItems({ - id: this.config.id, + sourceKey: this.config.id, type: 'delta', added: mutation.added, removed: mutation.removed, @@ -120,25 +118,4 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async stop() { this.running = false; } - - private async onNext(id: string, message: EntityMessage) { - if ('all' in message) { - // TODO unhandled rejection - await this.stateManager.addProcessingItems({ - id, - type: 'provider', - entities: message.all, - }); - } - - if ('added' in message) { - await this.stateManager.addProcessingItems({ - id, - type: 'provider', - entities: message.added, - }); - - // TODO deletions of message.removed - } - } } diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index dff424b1a9..21206f0a7b 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -25,10 +25,6 @@ import { v4 as uuidv4 } from 'uuid'; import { locationSpecToLocationEntity } from './util'; import { ConflictError } from '@backstage/errors'; -export type LocationMessage = - | { all: Location[] } - | { added: Location[]; removed: Location[] }; - export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index f8b4de1c41..a39ab9a8ff 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -16,7 +16,6 @@ import { Entity, - EntityName, LocationSpec, Location, EntityRelationSpec, @@ -116,12 +115,12 @@ export type ProcessingItem = { export type ReplaceProcessingItemsRequest = | { - id: string; + sourceKey: string; items: Entity[]; type: 'full'; } | { - id: string; + sourceKey: string; added: Entity[]; removed: Entity[]; type: 'delta'; From 464863419888171a95d805f3d51cc44bd8950de8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 13:56:32 +0200 Subject: [PATCH 06/19] Refactor deferredEntities processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../src/next/DefaultLocationStore.ts | 8 + .../src/next/DefaultProcessingStateManager.ts | 39 ++- .../src/next/NextCatalogBuilder.ts | 1 - plugins/catalog-backend/src/next/Stitcher.ts | 4 +- .../DefaultProcessingDatabase.test.ts | 83 ++++-- .../database/DefaultProcessingDatabase.ts | 236 ++++++++++-------- .../src/next/database/types.ts | 7 +- 7 files changed, 222 insertions(+), 156 deletions(-) diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index 21206f0a7b..40870cff7b 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -104,5 +104,13 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { async connect(connection: EntityProviderConnection): Promise { this._connection = connection; + const locations = await this.db.locations(); + const entities = locations.map(location => { + return locationSpecToLocationEntity(location); + }); + await this.connection.applyMutation({ + type: 'full', + entities, + }); } } diff --git a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts index d2765c3101..0003b7a7cf 100644 --- a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts +++ b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts @@ -49,35 +49,28 @@ export class DefaultProcessingStateManager implements ProcessingStateManager { async addProcessingItems(request: AddProcessingItemRequest) { return this.db.transaction(async tx => { - await this.db.addUnprocessedEntities(tx, request); + // await this.db.addUnprocessedEntities(tx, request); }); } async getNextProcessingItem(): Promise { - const entities = await new Promise(resolve => - this.popFromQueue(resolve), - ); - const { id, state, unprocessedEntity } = entities[0]; - return { - id, - entity: unprocessedEntity, - state, - }; - } - - async popFromQueue(resolve: (rows: RefreshStateItem[]) => void) { - const entities = await this.db.transaction(async tx => { - return this.db.getProcessableEntities(tx, { - processBatchSize: 1, + for (;;) { + const { items } = await this.db.transaction(async tx => { + return this.db.getProcessableEntities(tx, { + processBatchSize: 1, + }); }); - }); - // No entities require refresh, wait and try again. - if (!entities.items.length) { - setTimeout(() => this.popFromQueue(resolve), 1000); - return; + if (items.length) { + const { id, state, unprocessedEntity } = items[0]; + return { + id, + entity: unprocessedEntity, + state, + }; + } + + await new Promise(resolve => setTimeout(resolve, 1000)); } - - resolve(entities.items); } } diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 1796847164..d720bd331a 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -67,7 +67,6 @@ import { LocationAnalyzer } from '../ingestion/types'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; -import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; import { DefaultLocationStore } from './DefaultLocationStore'; import { DefaultProcessingStateManager } from './DefaultProcessingStateManager'; import { CatalogProcessingEngine } from '../next/types'; diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 3e5e3ac24b..f53b7c6fad 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -76,8 +76,8 @@ export class Stitcher { const [reference_count_result] = await tx( 'refresh_state_references', ) - .where({ target_entity_id: entity.metadata.uid }) - .count({ reference_count: 'target_entity_id' }); + .where({ target_entity_ref: entityRef }) + .count({ reference_count: 'target_entity_ref' }); if (Number(reference_count_result.reference_count) === 0) { this.logger.debug(`${entityRef} is orphan`); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index c4784c081d..14990a2a79 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -27,27 +27,26 @@ import * as uuid from 'uuid'; import { getVoidLogger } from '@backstage/backend-common'; describe('Default Processing Database', () => { - let db: Knex | undefined; - let processingDatabase: DefaultProcessingDatabase | undefined; - + let db: Knex; + let processingDatabase: DefaultProcessingDatabase; const logger = getVoidLogger(); beforeEach(async () => { db = await DatabaseManager.createTestDatabaseConnection(); await DatabaseManager.createDatabase(db); - processingDatabase = new DefaultProcessingDatabase(db!, logger); + processingDatabase = new DefaultProcessingDatabase(db, logger); }); describe('replaceUnprocessedEntities', () => { const insertRefRow = async (ref: DbRefreshStateReferencesRow) => { - return db!( - 'refresh_state_references', - ).insert(ref); + return db('refresh_state_references').insert( + ref, + ); }; const insertRefreshStateRow = async (ref: DbRefreshStateRow) => { - await db!('refresh_state').insert(ref); + await db('refresh_state').insert(ref); }; const createLocations = async (entityRefs: string[]) => { @@ -101,8 +100,8 @@ describe('Default Processing Database', () => { target_entity_ref: 'location:default/root-2', }); - await processingDatabase!.transaction(async tx => { - await processingDatabase!.replaceUnprocessedEntities(tx, { + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { type: 'full', sourceKey: 'config', items: [ @@ -117,11 +116,11 @@ describe('Default Processing Database', () => { }); }); - const currentRefreshState = await db!( + const currentRefreshState = await db( 'refresh_state', ).select(); - const currentRefRowState = await db!( + const currentRefRowState = await db( 'refresh_state_references', ).select(); @@ -205,8 +204,8 @@ describe('Default Processing Database', () => { target_entity_ref: 'location:default/root-2', }); - await processingDatabase!.transaction(async tx => { - await processingDatabase!.replaceUnprocessedEntities(tx, { + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { type: 'full', sourceKey: 'config', items: [ @@ -221,11 +220,11 @@ describe('Default Processing Database', () => { }); }); - const currentRefreshState = await db!( + const currentRefreshState = await db( 'refresh_state', ).select(); - const currentRefRowState = await db!( + const currentRefRowState = await db( 'refresh_state_references', ).select(); @@ -296,8 +295,8 @@ describe('Default Processing Database', () => { }); it('should add new locations using the delta options', async () => { - await processingDatabase!.transaction(async tx => { - await processingDatabase!.replaceUnprocessedEntities(tx, { + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { type: 'delta', sourceKey: 'lols', removed: [], @@ -313,11 +312,11 @@ describe('Default Processing Database', () => { }); }); - const currentRefreshState = await db!( + const currentRefreshState = await db( 'refresh_state', ).select(); - const currentRefRowState = await db!( + const currentRefRowState = await db( 'refresh_state_references', ).select(); @@ -336,6 +335,42 @@ describe('Default Processing Database', () => { ).toBeTruthy(); }); + it('should not remove locations that are referenced elsewhere', async () => { + /* + config-1 -> location:default/root + config-2 -> location:default/root + */ + await createLocations(['location:default/root']); + + await insertRefRow({ + source_key: 'config-1', + target_entity_ref: 'location:default/root', + }); + await insertRefRow({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }); + + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config-1', + items: [], + }); + }); + + const currentRefRowState = await db( + 'refresh_state_references', + ).select(); + + expect(currentRefRowState).toEqual({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }); + + // TODO: assert that root wasn't removed + }); + it('should remove old locations using the delta options', async () => { await createLocations(['location:default/new-root']); @@ -344,8 +379,8 @@ describe('Default Processing Database', () => { target_entity_ref: 'location:default/new-root', }); - await processingDatabase!.transaction(async tx => { - await processingDatabase!.replaceUnprocessedEntities(tx, { + await processingDatabase.transaction(async tx => { + await processingDatabase.replaceUnprocessedEntities(tx, { type: 'delta', sourceKey: 'lols', added: [], @@ -361,11 +396,11 @@ describe('Default Processing Database', () => { }); }); - const currentRefreshState = await db!( + const currentRefreshState = await db( 'refresh_state', ).select(); - const currentRefRowState = await db!( + const currentRefRowState = await db( 'refresh_state_references', ).select(); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index a80325d469..dc5e6036f5 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -26,10 +26,12 @@ import { UpdateProcessedEntityOptions, GetProcessableEntitiesResult, ReplaceUnprocessedEntitiesOptions, + RefreshStateItem, } from './types'; import type { Logger } from 'winston'; import { v4 as uuid } from 'uuid'; +import { JsonObject } from '@backstage/config'; export type DbRefreshStateRow = { entity_id: string; @@ -88,7 +90,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, }) .where('entity_id', id); - if (refreshResult === 0) { throw new NotFoundError(`Processing state not found for ${id}`); } @@ -96,8 +97,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { entities: deferredEntities, - id, - type: 'entity', + entityRef: stringifyEntityRef(processedEntity), }); // Update fragments @@ -172,80 +172,104 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { if (toRemove.length) { // TODO(freben): Batch split, to not hit variable limits? /* - WITH RECURSIVE - -- All the refs that can be reached from each individual root - root_reach(id, entity_ref) AS ( - -- Start with all roots - SELECT id, target_entity_ref - FROM refresh_state_references - WHERE source_key IS NOT NULL - UNION - -- For each match, select all children - SELECT root_reach.id, target_entity_ref - FROM refresh_state_references, root_reach - WHERE source_entity_ref = root_reach.entity_ref - ) - -- Start out with our own matching row (see the WHERE that - -- matches on source_key and target_entity_ref below) - SELECT us.entity_ref - FROM refresh_state_references - -- Expand the entire tree that emanates from that row - JOIN root_reach AS us - ON us.id = refresh_state_references.id - -- Expand with all roots that target the same node but - -- aren't ourselves - LEFT OUTER JOIN root_reach AS them - ON them.entity_ref = us.entity_ref - AND them.id != us.id - -- Keep only the matches that had no other rooots - WHERE refresh_state_references.source_key = "R1" - AND refresh_state_references.target_entity_ref = "A" - AND them.id IS NULL; - */ + WITH RECURSIVE + -- All the nodes that can be reached downwards from our root + descendants(root_id, entity_ref) AS ( + SELECT id, target_entity_ref + FROM refresh_state_references + WHERE source_key = "R1" AND target_entity_ref = "A" + UNION + SELECT descendants.root_id, target_entity_ref + FROM descendants + JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref + ), + -- All the nodes that can be reached upwards from the descendants + ancestors(root_id, via_entity_ref, to_entity_ref) AS ( + SELECT NULL, entity_ref, entity_ref + FROM descendants + UNION + SELECT + CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, + source_entity_ref, + ancestors.to_entity_ref + FROM ancestors + JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref + ) + -- Start out with all of the descendants + SELECT descendants.entity_ref + FROM descendants + -- Expand with all ancestors that point to those, but aren't the current root + LEFT OUTER JOIN ancestors + ON ancestors.to_entity_ref = descendants.entity_ref + AND ancestors.root_id IS NOT NULL + AND ancestors.root_id != descendants.root_id + -- Exclude all lines that had such a foreign ancestor + WHERE ancestors.root_id IS NULL; + */ const removedCount = await tx('refresh_state') .whereIn('entity_ref', function orphanedEntityRefs(orphans) { return ( orphans - // All the refs that can be reached from each individual root - .withRecursive('root_reach', function rootReach(outer) { - // Start with all roots + // All the nodes that can be reached downwards from our root + .withRecursive('descendants', function descendants(outer) { return outer - .select({ id: 'id', entity_ref: 'target_entity_ref' }) + .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) .from('refresh_state_references') - .whereNotNull('source_key') - .union(function recurse(inner) { - return ( - inner - // For each match, select all children - .select({ - id: 'root_reach.id', - entity_ref: - 'refresh_state_references.target_entity_ref', - }) - .from('refresh_state_references') - .crossJoin('root_reach', { - 'root_reach.entity_ref': - 'refresh_state_references.source_entity_ref', - }) - ); + .where('source_key', options.sourceKey) + .whereIn('target_entity_ref', toRemove) + .union(function recursive(inner) { + return inner + .select({ + root_id: 'descendants.root_id', + entity_ref: + 'refresh_state_references.target_entity_ref', + }) + .from('descendants') + .join('refresh_state_references', { + 'descendants.entity_ref': + 'refresh_state_references.source_entity_ref', + }); }); }) - .select('us.entity_ref') - // Start out with our own matching row - .from('refresh_state_references') - .where('source_key', options.sourceKey) - .whereIn('target_entity_ref', toRemove) - // Expand the entire tree that emanates from that row - .leftJoin({ us: 'root_reach' }, function us() { - this.on('us.id', '=', 'refresh_state_references.id'); + // All the nodes that can be reached upwards from the descendants + .withRecursive('ancestors', function ancestors(outer) { + return outer + .select({ + root_id: tx.raw('NULL', []), + via_entity_ref: 'entity_ref', + to_entity_ref: 'entity_ref', + }) + .from('descendants') + .union(function recursive(inner) { + return inner + .select({ + root_id: tx.raw( + 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', + [], + ), + via_entity_ref: 'source_entity_ref', + to_entity_ref: 'ancestors.to_entity_ref', + }) + .from('ancestors') + .join('refresh_state_references', { + target_entity_ref: 'ancestors.via_entity_ref', + }); + }); }) - // Expand with all roots that target the same node but aren't ourselves - .leftOuterJoin({ them: 'root_reach' }, function them() { - this.on('them.entity_ref', '=', 'us.entity_ref'); - this.andOn('them.id', '!=', 'us.id'); + // Start out with all of the descendants + .select('descendants.entity_ref') + .from('descendants') + // Expand with all ancestors that point to those, but aren't the current root + .leftOuterJoin('ancestors', function keepaliveRoots() { + this.on( + 'ancestors.to_entity_ref', + '=', + 'descendants.entity_ref', + ); + this.andOnNotNull('ancestors.root_id'); + this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); }) - // Keep only the matches that had no other rooots - .whereNull('them.id') + .whereNull('ancestors.root_id') ); }) .delete(); @@ -291,44 +315,45 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: AddUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const entityIds = new Array(); - for (const entity of options.entities) { - const entityRef = stringifyEntityRef(entity); - await tx('refresh_state') - .insert({ + const stateRows = options.entities.map( + entity => + ({ entity_id: uuid(), - entity_ref: entityRef, + entity_ref: stringifyEntityRef(entity), unprocessed_entity: JSON.stringify(entity), errors: '', next_update_at: tx.fn.now(), last_discovery_at: tx.fn.now(), - }) + } as Knex.DbRecord), + ); + const stateReferenceRows = stateRows.map( + stateRow => + ({ + source_entity_ref: options.entityRef, + target_entity_ref: stateRow.entity_ref, + } as Knex.DbRecord), + ); + + // Upsert all of the unprocessed entities into the refresh_state table, by + // their entity ref. + // TODO(freben): Can this be batched somehow? + for (const row of stateRows) { + await tx('refresh_state') + .insert(row) .onConflict('entity_ref') .merge(['unprocessed_entity', 'last_discovery_at']); - - const [{ entity_id: entityId }] = await tx( - 'refresh_state', - ).where({ entity_ref: entityRef }); - entityIds.push(entityId); } - const key = - options.type === 'provider' - ? { source_special_key: options.id } - : { source_entity_id: options.id }; - // copied from update refs + // Replace all references for the originating entity before creating new ones await tx('refresh_state_references') - .where(key) + .where({ source_entity_ref: options.entityRef }) .delete(); - - const referenceRows: DbRefreshStateReferencesRow[] = entityIds.map( - entityId => ({ - ...key, - target_entity_id: entityId, - }), + await tx.batchInsert( + 'refresh_state_references', + stateReferenceRows, + BATCH_SIZE, ); - await tx.batchInsert('refresh_state_references', referenceRows, BATCH_SIZE); } async getProcessableEntities( @@ -356,16 +381,23 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { }); return { - items: items.map(i => ({ - id: i.entity_id, - entityRef: i.entity_ref, - unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, - processedEntity: JSON.parse(i.processed_entity) as Entity, - nextUpdateAt: i.next_update_at, - lastDiscoveryAt: i.last_discovery_at, - state: JSON.parse(i.cache), - errors: i.errors, - })), + items: items.map( + i => + ({ + id: i.entity_id, + entityRef: i.entity_ref, + unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, + processedEntity: i.processed_entity + ? (JSON.parse(i.processed_entity) as Entity) + : undefined, + nextUpdateAt: i.next_update_at, + lastDiscoveryAt: i.last_discovery_at, + state: i.cache + ? JSON.parse(i.cache) + : new Map(), + errors: i.errors, + } as RefreshStateItem), + ), }; } diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index b926676887..d7a1dcd20b 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -19,8 +19,7 @@ import { JsonObject } from '@backstage/config'; import { Transaction } from '../../database/types'; export type AddUnprocessedEntitiesOptions = { - type: 'entity' | 'provider'; - id: string; + entityRef: string; entities: Entity[]; }; @@ -39,11 +38,11 @@ export type RefreshStateItem = { id: string; entityRef: string; unprocessedEntity: Entity; - processedEntity: Entity; + processedEntity?: Entity; nextUpdateAt: string; lastDiscoveryAt: string; // remove? state: Map; - errors: string; + errors?: string; }; export type GetProcessableEntitiesResult = { From f22ac8403507761e188e28ae25bb49433c694d2d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 14:02:41 +0200 Subject: [PATCH 07/19] Remove addProcessingItems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/Stitcher.ts | 4 ++-- plugins/catalog-backend/src/next/types.ts | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index f53b7c6fad..57f5ae9167 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -19,7 +19,7 @@ import { Logger } from 'winston'; import { Transaction } from '../database'; import { ConflictError } from '@backstage/errors'; import { - DbRefreshStateReferences, + DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, } from './database/DefaultProcessingDatabase'; @@ -73,7 +73,7 @@ export class Stitcher { return; } - const [reference_count_result] = await tx( + const [reference_count_result] = await tx( 'refresh_state_references', ) .where({ target_entity_ref: entityRef }) diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index a39ab9a8ff..ef3bab797b 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -101,12 +101,6 @@ export type ProcessingItemResult = { deferredEntities: Entity[]; }; -export type AddProcessingItemRequest = { - type: 'entity' | 'provider'; - id: string; - entities: Entity[]; -}; - export type ProcessingItem = { id: string; entity: Entity; @@ -128,6 +122,5 @@ export type ReplaceProcessingItemsRequest = export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; getNextProcessingItem(): Promise; - addProcessingItems(request: AddProcessingItemRequest): Promise; replaceProcessingItems(request: ReplaceProcessingItemsRequest): Promise; } From d77f5fea4406de005f25d052a90423f494d8bfcb Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 14:12:57 +0200 Subject: [PATCH 08/19] Add moar tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../DefaultProcessingDatabase.test.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 14990a2a79..f67b8603db 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -359,16 +359,26 @@ describe('Default Processing Database', () => { }); }); + const currentRefreshState = await db( + 'refresh_state', + ).select(); + const currentRefRowState = await db( 'refresh_state_references', ).select(); - expect(currentRefRowState).toEqual({ - source_key: 'config-2', - target_entity_ref: 'location:default/root', - }); + expect(currentRefRowState).toEqual([ + expect.objectContaining({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }), + ]); - // TODO: assert that root wasn't removed + expect(currentRefreshState).toEqual([ + expect.objectContaining({ + entity_ref: 'location:default/root', + }), + ]); }); it('should remove old locations using the delta options', async () => { From c1156a4b4748dfe9661ca9727e11184065d35d87 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 14:15:29 +0200 Subject: [PATCH 09/19] Remove dead code Signed-off-by: Johan Haals --- .../src/next/DefaultProcessingStateManager.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts index 0003b7a7cf..9c1d20ae9c 100644 --- a/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts +++ b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts @@ -14,9 +14,8 @@ * limitations under the License. */ -import { ProcessingDatabase, RefreshStateItem } from './database/types'; +import { ProcessingDatabase } from './database/types'; import { - AddProcessingItemRequest, ProcessingItem, ProcessingItemResult, ProcessingStateManager, @@ -47,12 +46,6 @@ export class DefaultProcessingStateManager implements ProcessingStateManager { }); } - async addProcessingItems(request: AddProcessingItemRequest) { - return this.db.transaction(async tx => { - // await this.db.addUnprocessedEntities(tx, request); - }); - } - async getNextProcessingItem(): Promise { for (;;) { const { items } = await this.db.transaction(async tx => { From 656e1d82b281f4f3a9518e27d1bfe014393c2c91 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 15:38:31 +0200 Subject: [PATCH 10/19] Add all migration files to migrationsv2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../migrationsv2/20200511113813_init.js | 133 ++++++++++ ...0200520140700_location_update_log_table.js | 43 ++++ ...7114117_location_update_log_latest_view.js | 45 ++++ .../migrationsv2/20200702153613_entities.js | 236 ++++++++++++++++++ ..._location_update_log_latest_deduplicate.js | 44 ++++ ...904_location_update_log_duplication_fix.js | 87 +++++++ .../20200807120600_entitySearch.js | 41 +++ .../20200809202832_add_bootstrap_location.js | 42 ++++ .../20200923104503_case_insensitivity.js | 32 +++ .../20201005122705_add_entity_full_name.js | 60 +++++ .../20201006130744_entity_data_column.js | 66 +++++ ...6203131_entity_remove_redundant_columns.js | 50 ++++ .../20201007201501_index_entity_search.js | 37 +++ .../20201019130742_add_relations_table.js | 54 ++++ .../20201123205611_relations_table_uniq.js | 93 +++++++ .../migrationsv2/20201210185851_fk_index.js | 45 ++++ .../20201230103504_update_log_varchar.js | 73 ++++++ .../20210209121210_locations_fk_index.js | 45 ++++ .../src/next/database/DatabaseManager.ts | 15 -- 19 files changed, 1226 insertions(+), 15 deletions(-) create mode 100644 plugins/catalog-backend/migrationsv2/20200511113813_init.js create mode 100644 plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js create mode 100644 plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js create mode 100644 plugins/catalog-backend/migrationsv2/20200702153613_entities.js create mode 100644 plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js create mode 100644 plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js create mode 100644 plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js create mode 100644 plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js create mode 100644 plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js create mode 100644 plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js create mode 100644 plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js create mode 100644 plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js create mode 100644 plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js create mode 100644 plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js create mode 100644 plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js create mode 100644 plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js create mode 100644 plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js create mode 100644 plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js diff --git a/plugins/catalog-backend/migrationsv2/20200511113813_init.js b/plugins/catalog-backend/migrationsv2/20200511113813_init.js new file mode 100644 index 0000000000..7f3d75e35c --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200511113813_init.js @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + return ( + knex.schema + // + // locations + // + .createTable('locations', table => { + table.comment( + 'Registered locations that shall be contiuously scanned for catalog item updates', + ); + table + .uuid('id') + .primary() + .notNullable() + .comment('Auto-generated ID of the location'); + table.string('type').notNullable().comment('The type of location'); + table + .string('target') + .notNullable() + .comment('The actual target of the location'); + }) + // + // entities + // + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }) + // + // entities_search + // + .createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }) + ); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema + .dropTable('entities_search') + .alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }) + .dropTable('entities') + .dropTable('locations'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js new file mode 100644 index 0000000000..d8093fc9b4 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200520140700_location_update_log_table.js @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + return knex.schema.createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema.dropTableIfExists('location_update_log'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js new file mode 100644 index 0000000000..a0f0f33a65 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200527114117_location_update_log_latest_view.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Get list sorted by created_at timestamp in descending order + // Grouped by location_id + return knex.schema.raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + ORDER BY created_at DESC; + `); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200702153613_entities.js b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js new file mode 100644 index 0000000000..0f1c204f9b --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200702153613_entities.js @@ -0,0 +1,236 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.dropForeign(['entity_id']); + }); + await knex.schema.alterTable('entities', table => { + table.dropPrimary('entities_pkey'); + }); + } + await knex.schema.alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }); + + // Setup temporary tables + await knex.schema.renameTable('entities_search', 'tmp_entities_search'); + await knex.schema.renameTable('entities', 'tmp_entities'); + + // + // entities + // + await knex.schema + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('etag') + .notNullable() + .comment( + 'An opaque string that changes for each update operation to any part of the entity, including metadata.', + ); + table + .string('generation') + .notNullable() + .unsigned() + .comment( + 'A positive nonzero number that indicates the current generation of data for this entity; the value is incremented each time the spec changes.', + ); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }); + + await knex.schema.raw(`INSERT INTO entities SELECT * FROM tmp_entities`); + + // + // entities_search + // + await knex.schema.createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }); + await knex.schema.raw( + `INSERT INTO entities_search SELECT * FROM tmp_entities_search`, + ); + + // Clean up + await knex.schema.dropTable('tmp_entities'); + return knex.schema.dropTable('tmp_entities_search'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js new file mode 100644 index 0000000000..87b41a80fc --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200721115244_location_update_log_latest_deduplicate.js @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = function up(knex) { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; +`); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = function down(knex) { + knex.schema.raw(`DROP VIEW location_update_log_latest;`); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js new file mode 100644 index 0000000000..de2b194cff --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200805163904_location_update_log_duplication_fix.js @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = function up(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.bigIncrements('id').primary(); // instead of uuid, so we can MAX it + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = function down(knex) { + return knex.schema + .raw('DROP VIEW location_update_log_latest;') + .dropTable('location_update_log') + .createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('entity_name').nullable(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(created_at) AS MAXDATE + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.created_at = t2.MAXDATE + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js new file mode 100644 index 0000000000..45226e53b4 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200807120600_entitySearch.js @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.text('value').nullable().alter(); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_search', table => { + table.string('value').nullable().alter(); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js new file mode 100644 index 0000000000..a90813fe85 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200809202832_add_bootstrap_location.js @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Adds a single 'bootstrap' location that can be used to trigger work in processors. + // This is primarily here to fulfill foreign key constraints. + await knex('locations').insert({ + id: require('uuid').v4(), + type: 'bootstrap', + target: 'bootstrap', + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex('locations') + .where({ + type: 'bootstrap', + target: 'bootstrap', + }) + .del(); +}; diff --git a/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js new file mode 100644 index 0000000000..ea5ba9e58d --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20200923104503_case_insensitivity.js @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex('entities') + .where({ namespace: null }) + .update({ namespace: 'default' }); + await knex('entities_search').update({ + key: knex.raw('LOWER(key)'), + value: knex.raw('LOWER(value)'), + }); +}; + +exports.down = async function down() {}; diff --git a/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js new file mode 100644 index 0000000000..aae1861658 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201005122705_add_entity_full_name.js @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table.text('full_name').nullable(); + }); + + await knex('entities').update({ + full_name: knex.raw( + "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", + ), + }); + + // SQLite does not support alter column + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.text('full_name').notNullable().alter(); + }); + } + + await knex.schema.alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['full_name'], 'entities_unique_full_name'); + table.dropUnique([], 'entities_unique_name'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.dropUnique([], 'entities_unique_full_name'); + table.unique(['kind', 'namespace', 'name'], 'entities_unique_name'); + }); + + await knex.schema.alterTable('entities_search', table => { + table.dropColumn('full_name'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js new file mode 100644 index 0000000000..a8964efbf6 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201006130744_entity_data_column.js @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('data') + .nullable() + .comment('The entire JSON data blob of the entity'); + }); + + await knex('entities').update({ + // apiVersion and kind should not contain any JSON unsafe chars, and both + // metadata and spec are already valid serialized JSON + data: knex.raw( + `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, + ), + }); + + await knex.schema.alterTable('entities', table => { + table.dropColumn('metadata'); + table.dropColumn('spec'); + }); + + // SQLite does not support ALTER COLUMN. + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.text('data').notNullable().alter(); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + table + .text('metadata') + .notNullable() + .comment('The entire metadata JSON blob of the entity'); + table + .text('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + table.dropColumn('data'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js new file mode 100644 index 0000000000..f40df5f73e --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201006203131_entity_remove_redundant_columns.js @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities', table => { + table.dropColumn('api_version'); + table.dropColumn('kind'); + table.dropColumn('name'); + table.dropColumn('namespace'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities', table => { + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table.string('kind').notNullable().comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js new file mode 100644 index 0000000000..77bf0529eb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201007201501_index_entity_search.js @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('entities_search', table => { + table.index(['key'], 'entities_search_key'); + table.index(['value'], 'entities_search_value'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('entities_search', table => { + table.dropIndex('', 'entities_search_key'); + table.dropIndex('', 'entities_search_value'); + }); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js new file mode 100644 index 0000000000..85e729f814 --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201019130742_add_relations_table.js @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('entities_relations'); +}; diff --git a/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js new file mode 100644 index 0000000000..9e8198b5eb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201123205611_relations_table_uniq.js @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client === 'sqlite3') { + // sqlite doesn't support dropPrimary so we recreate it properly instead + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + table.index('source_full_name', 'source_full_name_idx'); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropPrimary(); + table.index('source_full_name', 'source_full_name_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client === 'sqlite3') { + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'source_full_name_idx'); + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js new file mode 100644 index 0000000000..abb26cd5fc --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201210185851_fk_index.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_relations', table => { + table.index('originating_entity_id', 'originating_entity_id_idx'); + }); + await knex.schema.alterTable('entities_search', table => { + table.index('entity_id', 'entity_id_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'originating_entity_id_idx'); + }); + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'entity_id_idx'); + }); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js new file mode 100644 index 0000000000..d924b0414a --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20201230103504_update_log_varchar.js @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + // We actually just want to widen columns, but can't do that while a + // view is dependent on them - so we just reconstruct it exactly as it was + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.text('message').alter(); + table.text('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.string('message').alter(); + table.string('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; diff --git a/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js new file mode 100644 index 0000000000..ccfb1faffb --- /dev/null +++ b/plugins/catalog-backend/migrationsv2/20210209121210_locations_fk_index.js @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.index('location_id', 'entity_location_id_idx'); + }); + await knex.schema.alterTable('location_update_log', table => { + table.index('location_id', 'update_log_location_id_idx'); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema.alterTable('entities', table => { + table.dropIndex([], 'entity_location_id_idx'); + }); + await knex.schema.alterTable('location_update_log', table => { + table.dropIndex([], 'update_log_location_id_idx'); + }); + } +}; diff --git a/plugins/catalog-backend/src/next/database/DatabaseManager.ts b/plugins/catalog-backend/src/next/database/DatabaseManager.ts index 4c9e45950f..f660f73ecb 100644 --- a/plugins/catalog-backend/src/next/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/next/database/DatabaseManager.ts @@ -20,7 +20,6 @@ import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { CommonDatabase } from '../../database/CommonDatabase'; import { Database } from '../../database/types'; -import fs from 'fs-extra'; export type CreateDatabaseOptions = { logger: Logger; @@ -35,16 +34,10 @@ export class DatabaseManager { knex: Knex, options: Partial = {}, ): Promise { - const allMigrations = resolvePackagePath( - '@backstage/plugin-catalog-backend', - 'migrations', - ); - const migrationsDir = resolvePackagePath( '@backstage/plugin-catalog-backend', 'migrationsv2', ); - await fs.copy(allMigrations, migrationsDir); await knex.migrate.latest({ directory: migrationsDir, @@ -79,14 +72,6 @@ export class DatabaseManager { public static async createTestDatabaseConnection(): Promise { const config: Knex.Config = { - /* - client: 'pg', - connection: { - host: 'localhost', - user: 'postgres', - password: 'postgres', - }, - */ client: 'sqlite3', connection: ':memory:', useNullAsDefault: true, From fae7eb02965c17534168b0c85580ea5367e05477 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 15:43:22 +0200 Subject: [PATCH 11/19] Add configLocationProvider and provider identifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../src/next/ConfigLocationProvider.ts | 49 +++++++++++++++++++ .../next/DefaultCatalogProcessingEngine.ts | 9 ++-- .../src/next/DefaultLocationStore.ts | 4 ++ .../src/next/NextCatalogBuilder.ts | 7 +-- plugins/catalog-backend/src/next/types.ts | 1 + 5 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 plugins/catalog-backend/src/next/ConfigLocationProvider.ts diff --git a/plugins/catalog-backend/src/next/ConfigLocationProvider.ts b/plugins/catalog-backend/src/next/ConfigLocationProvider.ts new file mode 100644 index 0000000000..5b93fcfd98 --- /dev/null +++ b/plugins/catalog-backend/src/next/ConfigLocationProvider.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityProviderConnection, EntityProvider } from './types'; +import path from 'path'; +import { Config } from '@backstage/config'; +import { locationSpecToLocationEntity } from './util'; + +export class ConfigLocationProvider implements EntityProvider { + private connection: EntityProviderConnection | undefined; + + constructor(private readonly config: Config) {} + + getProviderName(): string { + return 'ConfigLocationProvider'; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + + const locationConfigs = + this.config.getOptionalConfigArray('catalog.locations') ?? []; + + const entities = locationConfigs.map(location => + locationSpecToLocationEntity({ + type: location.getString('type'), + target: path.resolve(location.getString('target')), + }), + ); + + await this.connection.applyMutation({ + type: 'full', + entities, + }); + } +} diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index b71b2c1045..1934b0aea7 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -68,9 +68,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { async start() { for (const provider of this.entityProviders) { - // TODO: this ID should be some form of identifier for the EntityProvider - const id = 'databaseProvider'; - provider.connect(new Connection({ stateManager: this.stateManager, id })); + provider.connect( + new Connection({ + stateManager: this.stateManager, + id: provider.getProviderName(), + }), + ); } this.running = true; diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index 40870cff7b..a56e5d1585 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -30,6 +30,10 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { constructor(private readonly db: Database) {} + getProviderName(): string { + return 'DefaultLocationStore'; + } + createLocation(spec: LocationSpec): Promise { return this.db.transaction(async tx => { // TODO: id should really be type and target combined and not a uuid. diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index d720bd331a..0699a13f58 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,6 +50,7 @@ import { GithubDiscoveryProcessor, GithubOrgReaderProcessor, LdapOrgReaderProcessor, + LocationEntityProcessor, MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, @@ -73,6 +74,7 @@ import { CatalogProcessingEngine } from '../next/types'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { Stitcher } from './Stitcher'; import { CommonDatabase } from '../database/CommonDatabase'; +import { ConfigLocationProvider } from './ConfigLocationProvider'; export type CatalogEnvironment = { logger: Logger; @@ -272,9 +274,10 @@ export class NextCatalogBuilder { const locationStore = new DefaultLocationStore(db); const stitcher = new Stitcher(dbClient, logger); + const configLocationProvider = new ConfigLocationProvider(config); const processingEngine = new DefaultCatalogProcessingEngine( logger, - [locationStore], // entityproviders + [locationStore, configLocationProvider], stateManager, orchestrator, stitcher, @@ -322,7 +325,6 @@ export class NextCatalogBuilder { // These are always there no matter what const processors: CatalogProcessor[] = [ - StaticLocationProcessor.fromConfig(config), new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), new BuiltinKindsEntityProcessor(), ]; @@ -338,7 +340,6 @@ export class NextCatalogBuilder { MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), - // new LocationEntityProcessor({ integrations }), new AnnotateLocationEntityProcessor({ integrations }), ); } diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index ef3bab797b..e0616fb336 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -65,6 +65,7 @@ export interface EntityProviderConnection { } export interface EntityProvider { + getProviderName(): string; connect(connection: EntityProviderConnection): Promise; } From e00ba124546a742639973fd26edb39635dff5351 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Apr 2021 16:53:44 +0200 Subject: [PATCH 12/19] chore: remove unused imports Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/NextCatalogBuilder.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 0699a13f58..24b763a098 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,11 +50,9 @@ import { GithubDiscoveryProcessor, GithubOrgReaderProcessor, LdapOrgReaderProcessor, - LocationEntityProcessor, MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, - StaticLocationProcessor, UrlReaderProcessor, } from '../ingestion'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; From 8ce7d6e8f81d621c66e069c64cfe67f4e42fed3c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Apr 2021 14:16:49 +0200 Subject: [PATCH 13/19] catalog-backend: Add ConfigLocationProvider tests Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../src/next/ConfigLocationProvider.test.ts | 67 +++++++++++++++++++ .../src/next/ConfigLocationProvider.ts | 14 ++-- 2 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 plugins/catalog-backend/src/next/ConfigLocationProvider.test.ts diff --git a/plugins/catalog-backend/src/next/ConfigLocationProvider.test.ts b/plugins/catalog-backend/src/next/ConfigLocationProvider.test.ts new file mode 100644 index 0000000000..9aaba13c48 --- /dev/null +++ b/plugins/catalog-backend/src/next/ConfigLocationProvider.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigLocationProvider } from './ConfigLocationProvider'; +import { EntityProviderConnection } from './types'; +import { ConfigReader } from '@backstage/config'; +import { resolvePackagePath } from '@backstage/backend-common'; +import path from 'path'; + +describe('Config Location Provider', () => { + it('should apply mutation with the correct paths in the config', async () => { + const mockConfig = new ConfigReader({ + catalog: { + locations: [ + { type: 'file', target: './lols.yaml' }, + { type: 'url', target: 'https://github.com/backstage/backstage' }, + ], + }, + }); + + const mockConnection = ({ + applyMutation: jest.fn(), + } as unknown) as EntityProviderConnection; + const locationProvider = new ConfigLocationProvider(mockConfig); + + await locationProvider.connect(mockConnection); + + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: path.join( + resolvePackagePath('@backstage/plugin-catalog-backend'), + './lols.yaml', + ), + type: 'file', + }, + }), + ]), + }); + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: 'https://github.com/backstage/backstage', + type: 'url', + }, + }), + ]), + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/ConfigLocationProvider.ts b/plugins/catalog-backend/src/next/ConfigLocationProvider.ts index 5b93fcfd98..8a65487df1 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationProvider.ts +++ b/plugins/catalog-backend/src/next/ConfigLocationProvider.ts @@ -34,12 +34,14 @@ export class ConfigLocationProvider implements EntityProvider { const locationConfigs = this.config.getOptionalConfigArray('catalog.locations') ?? []; - const entities = locationConfigs.map(location => - locationSpecToLocationEntity({ - type: location.getString('type'), - target: path.resolve(location.getString('target')), - }), - ); + const entities = locationConfigs.map(location => { + const type = location.getString('type'); + const target = location.getString('target'); + return locationSpecToLocationEntity({ + type, + target: type === 'file' ? path.resolve(target) : target, + }); + }); await this.connection.applyMutation({ type: 'full', From 512889f3ca11f266786b0344900ef71ee432e4b9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Apr 2021 18:28:55 +0200 Subject: [PATCH 14/19] WIP tests for DefaultProcessingDatabase Signed-off-by: Johan Haals --- .../DefaultProcessingDatabase.test.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index f67b8603db..a5c1e56cff 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -429,4 +429,66 @@ describe('Default Processing Database', () => { ).toBeFalsy(); }); }); + + describe('updateProcessedEntity', () => { + it('should throw if the entity does not exist', async () => { + await processingDatabase.transaction(async tx => { + await expect( + processingDatabase.updateProcessedEntity(tx, { + id: '9', + processedEntity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + deferredEntities: [], + relations: [], + }), + ).rejects.toThrow('Processing state not found for 9'); + }); + }); + + it('should update a processed entity', async () => { + await db('refresh_state').insert({ + entity_id: '123', + entity_ref: 'Component:default/wacka', + unprocessed_entity: '', + errors: '', + next_update_at: 'now()', + last_discovery_at: 'now()', + }); + + const deferredEntity = { + apiVersion: '1.0.0', + metadata: { + name: 'deferred', + }, + kind: 'Location', + } as Entity; + + await processingDatabase.transaction(async tx => { + await processingDatabase.updateProcessedEntity(tx, { + id: '123', + processedEntity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + deferredEntities: [deferredEntity], + relations: [], + }); + }); + + console.log( + await db('refresh_state') + .where({ entity_ref: 'deferred' }) + .select(), + ); + expect(1).toBeDefined(); + }); + }); }); From 412f96edc10bccc1ccf0716f63dca1bc0dcd244b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Apr 2021 18:30:00 +0200 Subject: [PATCH 15/19] Combine stitiching queries Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/Stitcher.ts | 118 ++++++++++++++----- 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 57f5ae9167..04a8ab8cfc 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -26,6 +26,14 @@ import { import { Entity, parseEntityRef } from '@backstage/catalog-model'; import { createHash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; +import { buildEntitySearch } from '../database/search'; +import { DbEntitiesSearchRow } from '../database/types'; + +// The number of items that are sent per batch to the database layer, when +// doing .batchInsert calls to knex. This needs to be low enough to not cause +// errors in the underlying engine due to exceeding query limits, but large +// enough to get the speed benefits. +const BATCH_SIZE = 50; export type DbFinalEntitiesRow = { entity_id: string; @@ -49,64 +57,114 @@ export class Stitcher { for (const entityRef of entityRefs) { await this.transaction(async txOpaque => { const tx = txOpaque as Knex.Transaction; - const [result] = await tx('refresh_state') - .select('entity_id', 'processed_entity') - .where({ entity_ref: entityRef }); - if (!result) { + const result: Array<{ + entityId: string; + processedEntity?: string; + errors: string; + incomingReferenceCount: string | number; + previousEtag?: string; + relationType?: string; + relationTarget?: string; + }> = await tx + .with('incoming_references', function incomingReferences(builder) { + return builder + .from('refresh_state_references') + .where({ target_entity_ref: entityRef }) + .count({ count: '*' }); + }) + .select({ + entityId: 'refresh_state.entity_id', + processedEntity: 'refresh_state.processed_entity', + errors: 'refresh_state.errors', + incomingReferenceCount: 'incoming_references.count', + previousEtag: 'final_entities.etag', + relationType: 'relations.type', + relationTarget: 'relations.target_entity_ref', + }) + .from('refresh_state') + .leftJoin('incoming_references', {}) + .leftOuterJoin('final_entities', { + 'final_entities.entity_id': 'refresh_state.entity_id', + }) + .leftOuterJoin('relations', { + 'relations.source_entity_ref': 'refresh_state.entity_ref', + }) + .where({ 'refresh_state.entity_ref': entityRef }); + + if (!result.length) { this.logger.debug( `Unable to stitch ${entityRef}, item does not exist in refresh state table`, ); return; - } else if (!result.processed_entity) { + } + + const { + entityId, + processedEntity, + errors, + incomingReferenceCount, + previousEtag, + } = result[0]; + + if (!processedEntity) { this.logger.debug( `Unable to stitch ${entityRef}, the entity has not yet been processed`, ); return; } - const entity: Entity = JSON.parse(result.processed_entity); + const entity = JSON.parse(processedEntity) as Entity; + const isOrphan = Number(incomingReferenceCount) === 0; - const entityId = entity?.metadata?.uid; - if (!entityId) { - this.logger.error(`missing ID in entity ${JSON.stringify(entity)}`); - return; - } - - const [reference_count_result] = await tx( - 'refresh_state_references', - ) - .where({ target_entity_ref: entityRef }) - .count({ reference_count: 'target_entity_ref' }); - - if (Number(reference_count_result.reference_count) === 0) { - this.logger.debug(`${entityRef} is orphan`); + if (isOrphan) { + this.logger.debug(`${entityRef} is an orphan`); entity.metadata.annotations = { ...entity.metadata.annotations, ['backstage.io/orphan']: 'true', }; } - const relationResults = await tx('relations') - .where({ source_entity_ref: entityRef }) - .select(); - - // TODO: entityRef is lower case and should be uppercase in the final result. - entity.relations = relationResults.map(relation => ({ - type: relation.type, - target: parseEntityRef(relation.target_entity_ref), - })); + // TODO: entityRef is lower case and should be uppercase in the final result + entity.relations = result + .filter(row => row.relationType) + .map(row => ({ + type: row.relationType!, + target: parseEntityRef(row.relationTarget!), + })); entity.metadata.generation = 1; + + // If the output entity was actually not changed, just abort const etag = generateEntityEtag(entity); + if (etag === previousEtag) { + console.log(`Skipped stitching of ${entityRef}, no changes`); + return; + } + entity.metadata.etag = etag; + await tx('final_entities') .insert({ - finalized_entity: JSON.stringify(entity), entity_id: entityId, + finalized_entity: JSON.stringify(entity), etag, }) .onConflict('entity_id') .merge(['finalized_entity', 'etag']); + + try { + const entries = buildEntitySearch(entityId, entity); + await tx('search') + .where({ entity_id: entityId }) + .delete(); + await tx.batchInsert('search', entries, BATCH_SIZE); + } catch (e) { + this.logger.debug( + `Failed to write search entries for ${entityId}`, + e, + ); + // intentionally ignored + } }); } } From 20346793a20e32299e88858c636985cd4836236d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Apr 2021 18:30:22 +0200 Subject: [PATCH 16/19] Create search table Signed-off-by: Johan Haals --- .../20210302150147_refresh_state.js | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 2afeda02c0..755a645894 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -154,6 +154,28 @@ exports.up = async function up(knex) { table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); + + await knex.schema.createTable('search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + table.index(['key'], 'search_key_idx'); + table.index(['value'], 'search_value_idx'); + }); }; /** @@ -177,6 +199,12 @@ exports.down = async function down(knex) { table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); + await knex.schema.alterTable('search', table => { + table.dropIndex([], 'search_key_idx'); + table.dropIndex([], 'search_value_idx'); + }); + + await knex.schema.dropTable('search'); await knex.schema.dropTable('final_entities'); await knex.schema.dropTable('relations'); await knex.schema.dropTable('references'); From 1a3ce09aee63fa19714f149d73b3985a960fc43c Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 27 Apr 2021 21:23:35 +0200 Subject: [PATCH 17/19] chore(tests): Added some more tests for DefaultLocationStore and fixing migration so it will run properly on PG Signed-off-by: blam --- .../20210302150147_refresh_state.js | 2 +- .../src/next/DefaultLocationStore.test.ts | 146 ++++++++++++++++++ .../src/next/DefaultLocationStore.ts | 26 ++-- 3 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 plugins/catalog-backend/src/next/DefaultLocationStore.test.ts diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 755a645894..bbaf4820cd 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -160,7 +160,7 @@ exports.up = async function up(knex) { 'Flattened key-values from the entities, used for quick filtering', ); table - .uuid('entity_id') + .text('entity_id') .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts new file mode 100644 index 0000000000..be20179e9a --- /dev/null +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts @@ -0,0 +1,146 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DatabaseManager } from './database/DatabaseManager'; +import { DefaultLocationStore } from './DefaultLocationStore'; +import { v4 } from 'uuid'; + +describe('Default Location Store', () => { + const createLocationStore = async () => { + const db = await DatabaseManager.createTestDatabase(); + const connection = { applyMutation: jest.fn() }; + const store = new DefaultLocationStore(db); + await store.connect(connection); + return { store, connection }; + }; + + it('should do a full sync with the locations on connect', async () => { + const { connection } = await createLocationStore(); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }); + + describe('listLocations', () => { + it('lists empty locations when there is no locations', async () => { + const { store } = await createLocationStore(); + + expect(await store.listLocations()).toEqual([]); + }); + + it('lists locations that are added to the db', async () => { + const { store } = await createLocationStore(); + + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + const listLocations = await store.listLocations(); + + expect(listLocations).toHaveLength(1); + expect(listLocations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }), + ]), + ); + }); + }); + + describe('createLocation', () => { + it('throws when the location already exists', async () => { + const { store } = await createLocationStore(); + const spec = { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }; + await store.createLocation(spec); + + await expect(() => store.createLocation(spec)).rejects.toThrow( + new RegExp(`Location ${spec.type}:${spec.target} already exists`), + ); + }); + + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); + + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [], + added: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }); + }); + + describe('deleteLocation', () => { + it('throws if the location does not exist', async () => { + const { store } = await createLocationStore(); + + const id = v4(); + + await expect(() => store.deleteLocation(id)).rejects.toThrow( + new RegExp(`Found no location with ID ${id}`), + ); + }); + + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); + + const location = await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + await store.deleteLocation(location.id); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index a56e5d1585..28db3790b6 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -34,10 +34,8 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { return 'DefaultLocationStore'; } - createLocation(spec: LocationSpec): Promise { + async createLocation(spec: LocationSpec): Promise { return this.db.transaction(async tx => { - // TODO: id should really be type and target combined and not a uuid. - // Attempt to find a previous location matching the spec const previousLocations = await this.listLocations(); const previousLocation = previousLocations.some( @@ -50,6 +48,7 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { ); } + // TODO: id should really be type and target combined and not a uuid. const location = await this.db.addLocation(tx, { id: uuidv4(), type: spec.type, @@ -68,11 +67,17 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { async listLocations(): Promise { const dbLocations = await this.db.locations(); - return dbLocations.map(item => ({ - id: item.id, - target: item.target, - type: item.type, - })); + return ( + dbLocations + // TODO(blam): We should create a mutation to remove this location for everyone + // eventually when it's all done and dusted + .filter(({ type }) => type !== 'bootstrap') + .map(item => ({ + id: item.id, + target: item.target, + type: item.type, + })) + ); } getLocation(id: string): Promise { @@ -86,9 +91,6 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { return this.db.transaction(async tx => { const location = await this.db.location(id); - if (!location) { - throw new ConflictError(`No location found with id: ${id}`); - } await this.db.removeLocation(tx, id); await this.connection.applyMutation({ type: 'delta', @@ -108,7 +110,7 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { async connect(connection: EntityProviderConnection): Promise { this._connection = connection; - const locations = await this.db.locations(); + const locations = await this.listLocations(); const entities = locations.map(location => { return locationSpecToLocationEntity(location); }); From f7167e9aacd2ec721c1998896f12694c1f7b8917 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 28 Apr 2021 09:18:35 +0200 Subject: [PATCH 18/19] Update processed entity tests Signed-off-by: Johan Haals --- .../DefaultProcessingDatabase.test.ts | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index a5c1e56cff..d76bd74260 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -19,10 +19,11 @@ import { Knex } from 'knex'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, + DbRelationsRow, DefaultProcessingDatabase, } from './DefaultProcessingDatabase'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; import * as uuid from 'uuid'; import { getVoidLogger } from '@backstage/backend-common'; @@ -452,8 +453,8 @@ describe('Default Processing Database', () => { it('should update a processed entity', async () => { await db('refresh_state').insert({ - entity_id: '123', - entity_ref: 'Component:default/wacka', + entity_id: '321', + entity_ref: 'location:default/new-root', unprocessed_entity: '', errors: '', next_update_at: 'now()', @@ -468,9 +469,23 @@ describe('Default Processing Database', () => { kind: 'Location', } as Entity; + const relation: EntityRelationSpec = { + source: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + target: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + type: 'url', + }; + await processingDatabase.transaction(async tx => { await processingDatabase.updateProcessedEntity(tx, { - id: '123', + id: '321', processedEntity: { apiVersion: '1.0.0', metadata: { @@ -479,16 +494,24 @@ describe('Default Processing Database', () => { kind: 'Location', } as Entity, deferredEntities: [deferredEntity], - relations: [], + relations: [relation], }); }); - console.log( - await db('refresh_state') - .where({ entity_ref: 'deferred' }) - .select(), - ); - expect(1).toBeDefined(); + const deferredResult = await db('refresh_state') + .where({ entity_ref: 'location:default/deferred' }) + .select(); + expect(deferredResult.length).toBe(1); + + const [relations] = await db('relations') + .where({ originating_entity_id: '321' }) + .select(); + expect(relations).toEqual({ + originating_entity_id: '321', + source_entity_ref: 'component:default/foo', + type: 'url', + target_entity_ref: 'component:default/foo', + }); }); }); }); From 17b8da50a959a0c452a457f4207328d8ae1ca98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 28 Apr 2021 09:18:36 +0200 Subject: [PATCH 19/19] update the stitcher including search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../20210302150147_refresh_state.js | 7 +- .../catalog-backend/src/next/Stitcher.test.ts | 205 ++++++++++++++++++ plugins/catalog-backend/src/next/Stitcher.ts | 59 ++--- .../catalog-backend/src/next/search.test.ts | 160 ++++++++++++++ plugins/catalog-backend/src/next/search.ts | 191 ++++++++++++++++ plugins/catalog-backend/src/next/types.ts | 1 + 6 files changed, 592 insertions(+), 31 deletions(-) create mode 100644 plugins/catalog-backend/src/next/Stitcher.test.ts create mode 100644 plugins/catalog-backend/src/next/search.test.ts create mode 100644 plugins/catalog-backend/src/next/search.ts diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index bbaf4820cd..4d2c30f194 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -33,7 +33,6 @@ exports.up = async function up(knex) { ); table .text('entity_ref') - .unique() .notNullable() .comment('A reference to the entity that the refresh state is tied to'); table @@ -59,13 +58,14 @@ exports.up = async function up(knex) { .notNullable() .comment('JSON array containing all errors related to entity'); table - .dateTime('next_update_at') // TOOD: timezone or change to epoch-millis or similar + .dateTime('next_update_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('Timestamp of when entity should be updated'); table - .dateTime('last_discovery_at') // TOOD: timezone or change to epoch-millis or similar + .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp of which this entity was discovered'); + table.unique(['entity_ref'], 'refresh_state_entity_ref_uniq'); table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); @@ -188,6 +188,7 @@ exports.down = async function down(knex) { table.dropIndex([], 'refresh_state_references_target_entity_id_idx'); }); await knex.schema.alterTable('refresh_state', table => { + table.dropUnique([], 'refresh_state_entity_ref_uniq'); table.dropIndex([], 'refresh_state_entity_id_idx'); table.dropIndex([], 'refresh_state_entity_ref_idx'); table.dropIndex([], 'refresh_state_next_update_at_idx'); diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts new file mode 100644 index 0000000000..327d922c07 --- /dev/null +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -0,0 +1,205 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { DatabaseManager } from './database/DatabaseManager'; +import { + DbRefreshStateReferencesRow, + DbRefreshStateRow, + DbRelationsRow, +} from './database/DefaultProcessingDatabase'; +import { DbSearchRow } from './search'; +import { DbFinalEntitiesRow, Stitcher } from './Stitcher'; + +describe('Stitcher', () => { + let db: Knex; + const logger = getVoidLogger(); + + beforeEach(async () => { + db = await DatabaseManager.createTestDatabaseConnection(); + await DatabaseManager.createDatabase(db); + }); + + it('runs the happy path', async () => { + const stitcher = new Stitcher(db, logger); + + await db.transaction(async tx => { + await tx('refresh_state').insert([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }, + ]); + await tx('refresh_state_references').insert([ + { source_key: 'a', target_entity_ref: 'k:ns/n' }, + ]); + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/other', + }, + ]); + }); + + await stitcher.stitch(new Set(['k:ns/n'])); + + let firstEtag: string; + await db.transaction(async tx => { + const entities = await tx('final_entities'); + + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].finalized_entity); + expect(entity).toEqual({ + relations: [ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + ], + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + firstEtag = entity.metadata.etag; + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }); + + // Re-stitch without any changes + await stitcher.stitch(new Set(['k:ns/n'])); + + await db.transaction(async tx => { + const entities = await tx('final_entities'); + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].finalized_entity); + expect(entities[0].etag).toEqual(firstEtag); + expect(entity.metadata.etag).toEqual(firstEtag); + }); + + // Now add one more relation and re-stitch + await db.transaction(async tx => { + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/third', + }, + ]); + }); + + await stitcher.stitch(new Set(['k:ns/n'])); + + await db.transaction(async tx => { + const entities = await tx('final_entities'); + + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].finalized_entity); + expect(entity).toEqual({ + relations: expect.arrayContaining([ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'third', + }, + }, + ]), + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entities[0].etag).not.toEqual(firstEtag); + expect(entities[0].etag).toEqual(entity.metadata.etag); + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' }, + { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/third' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 04a8ab8cfc..cd9220e934 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -14,20 +14,14 @@ * limitations under the License. */ +import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { ConflictError } from '@backstage/errors'; +import { createHash } from 'crypto'; +import stableStringify from 'fast-json-stable-stringify'; import { Knex } from 'knex'; import { Logger } from 'winston'; import { Transaction } from '../database'; -import { ConflictError } from '@backstage/errors'; -import { - DbRefreshStateReferencesRow, - DbRefreshStateRow, - DbRelationsRow, -} from './database/DefaultProcessingDatabase'; -import { Entity, parseEntityRef } from '@backstage/catalog-model'; -import { createHash } from 'crypto'; -import stableStringify from 'fast-json-stable-stringify'; -import { buildEntitySearch } from '../database/search'; -import { DbEntitiesSearchRow } from '../database/types'; +import { buildEntitySearch, DbSearchRow } from './search'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -58,6 +52,13 @@ export class Stitcher { await this.transaction(async txOpaque => { const tx = txOpaque as Knex.Transaction; + // Selecting from refresh_state and final_entities should yield exactly + // one row (except in abnormal cases where the stitch was invoked for + // something that didn't exist at all, in which case it's zero rows). + // The join with the temporary incoming_references still gives one row. + // The only result set "expanding" join is the one with relations, so + // the output should be at least one row (if zero or one relations were + // found), or at most the same number of rows as relations. const result: Array<{ entityId: string; processedEntity?: string; @@ -90,8 +91,14 @@ export class Stitcher { .leftOuterJoin('relations', { 'relations.source_entity_ref': 'refresh_state.entity_ref', }) - .where({ 'refresh_state.entity_ref': entityRef }); + .where({ 'refresh_state.entity_ref': entityRef }) + .orderBy('relationType', 'asc') + .orderBy('relationTarget', 'asc'); + // If there were no rows returned, it would mean that there was no + // matching row even in the refresh_state. This can happen for example + // if we emit a relation to something that hasn't been ingested yet. + // It's safe to ignore this stitch attempt in that case. if (!result.length) { this.logger.debug( `Unable to stitch ${entityRef}, item does not exist in refresh state table`, @@ -102,11 +109,15 @@ export class Stitcher { const { entityId, processedEntity, - errors, + // errors, incomingReferenceCount, previousEtag, } = result[0]; + // If there was no processed entity in place, the target hasn't been + // through the processing steps yet. It's safe to ignore this stitch + // attempt in that case, since another stitch will be triggered when + // that processing has finished. if (!processedEntity) { this.logger.debug( `Unable to stitch ${entityRef}, the entity has not yet been processed`, @@ -114,6 +125,7 @@ export class Stitcher { return; } + // Grab the processed entity and stitch all of the relevant data into it const entity = JSON.parse(processedEntity) as Entity; const isOrphan = Number(incomingReferenceCount) === 0; @@ -132,15 +144,16 @@ export class Stitcher { type: row.relationType!, target: parseEntityRef(row.relationTarget!), })); - entity.metadata.generation = 1; // If the output entity was actually not changed, just abort const etag = generateEntityEtag(entity); if (etag === previousEtag) { - console.log(`Skipped stitching of ${entityRef}, no changes`); + this.logger.debug(`Skipped stitching of ${entityRef}, no changes`); return; } + entity.metadata.uid = entityId; + entity.metadata.generation = 1; entity.metadata.etag = etag; await tx('final_entities') @@ -152,19 +165,9 @@ export class Stitcher { .onConflict('entity_id') .merge(['finalized_entity', 'etag']); - try { - const entries = buildEntitySearch(entityId, entity); - await tx('search') - .where({ entity_id: entityId }) - .delete(); - await tx.batchInsert('search', entries, BATCH_SIZE); - } catch (e) { - this.logger.debug( - `Failed to write search entries for ${entityId}`, - e, - ); - // intentionally ignored - } + const searchEntries = buildEntitySearch(entityId, entity); + await tx('search').where({ entity_id: entityId }).delete(); + await tx.batchInsert('search', searchEntries, BATCH_SIZE); }); } } diff --git a/plugins/catalog-backend/src/next/search.test.ts b/plugins/catalog-backend/src/next/search.test.ts new file mode 100644 index 0000000000..be9a98fe69 --- /dev/null +++ b/plugins/catalog-backend/src/next/search.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { buildEntitySearch, mapToRows, traverse } from './search'; + +describe('search', () => { + describe('traverse', () => { + it('expands lists of strings to several rows', () => { + const input = { a: ['b', 'c', 'd'] }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a', value: 'b' }, + { key: 'a.b', value: true }, + { key: 'a', value: 'c' }, + { key: 'a.c', value: true }, + { key: 'a', value: 'd' }, + { key: 'a.d', value: true }, + ]); + }); + + it('expands objects', () => { + const input = { a: { b: { c: 'd' }, e: 'f' } }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a.b.c', value: 'd' }, + { key: 'a.e', value: 'f' }, + ]); + }); + + it('expands list of objects', () => { + const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'root.list.a', value: 1 }, + { key: 'root.list.a', value: 2 }, + ]); + }); + + it('skips over special keys', () => { + const input = { + state: { x: 1 }, + relations: [{ y: 2 }], + a: 'a', + metadata: { + b: 'b', + name: 'name', + namespace: 'namespace', + uid: 'uid', + etag: 'etag', + generation: 'generation', + c: 'c', + }, + d: 'd', + }; + const output = traverse(input); + expect(output).toEqual([ + { key: 'a', value: 'a' }, + { key: 'metadata.b', value: 'b' }, + { key: 'metadata.c', value: 'c' }, + { key: 'd', value: 'd' }, + ]); + }); + }); + + describe('mapToRows', () => { + it('converts base types to strings or null', () => { + const input = [ + { key: 'a', value: true }, + { key: 'b', value: false }, + { key: 'c', value: 7 }, + { key: 'd', value: 'string' }, + { key: 'e', value: null }, + { key: 'f', value: undefined }, + ]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a', value: 'true' }, + { entity_id: 'eid', key: 'b', value: 'false' }, + { entity_id: 'eid', key: 'c', value: '7' }, + { entity_id: 'eid', key: 'd', value: 'string' }, + { entity_id: 'eid', key: 'e', value: null }, + { entity_id: 'eid', key: 'f', value: null }, + ]); + }); + + it('emits lowercase version of keys and values', () => { + const input = [{ key: 'fOo', value: 'BaR' }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]); + }); + + it('skips very large values', () => { + const input = [{ key: 'foo', value: 'a'.repeat(10000) }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([]); + }); + }); + + describe('buildEntitySearch', () => { + it('adds special keys even if missing', () => { + const input: Entity = { + apiVersion: 'a', + kind: 'b', + metadata: { name: 'n' }, + }; + expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'apiversion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, + { entity_id: 'eid', key: 'metadata.name', value: 'n' }, + { entity_id: 'eid', key: 'metadata.namespace', value: null }, + { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'metadata.namespace', + value: ENTITY_DEFAULT_NAMESPACE, + }, + ]); + }); + + it('adds relations', () => { + const input: Entity = { + relations: [ + { type: 't1', target: { kind: 'k', namespace: 'ns', name: 'a' } }, + { type: 't2', target: { kind: 'k', namespace: 'ns', name: 'b' } }, + ], + apiVersion: 'a', + kind: 'b', + metadata: { name: 'n' }, + }; + expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'apiversion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, + { entity_id: 'eid', key: 'metadata.name', value: 'n' }, + { entity_id: 'eid', key: 'metadata.namespace', value: null }, + { entity_id: 'eid', key: 'metadata.uid', value: null }, + { + entity_id: 'eid', + key: 'metadata.namespace', + value: ENTITY_DEFAULT_NAMESPACE, + }, + { entity_id: 'eid', key: 'relations', value: 't1:k:ns/a' }, + { entity_id: 'eid', key: 'relations', value: 't2:k:ns/b' }, + ]); + }); + }); +}); diff --git a/plugins/catalog-backend/src/next/search.ts b/plugins/catalog-backend/src/next/search.ts new file mode 100644 index 0000000000..4aa8bae1a8 --- /dev/null +++ b/plugins/catalog-backend/src/next/search.ts @@ -0,0 +1,191 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + ENTITY_DEFAULT_NAMESPACE, + stringifyEntityRef, +} from '@backstage/catalog-model'; + +export type DbSearchRow = { + entity_id: string; + key: string; + value: string | null; +}; + +// These are excluded in the generic loop, either because they do not make sense +// to index, or because they are special-case always inserted whether they are +// null or not +const SPECIAL_KEYS = [ + 'state', + 'relations', + 'metadata.name', + 'metadata.namespace', + 'metadata.uid', + 'metadata.etag', + 'metadata.generation', +]; + +// The maximum length allowed for search values. These columns are indexed, and +// database engines do not like to index on massive values. For example, +// postgres will balk after 8191 byte line sizes. +const MAX_VALUE_LENGTH = 200; + +type Kv = { + key: string; + value: unknown; +}; + +// Helper for traversing through a nested structure and outputting a list of +// path->value entries of the leaves. +// +// For example, this yaml structure +// +// a: 1 +// b: +// c: null +// e: [f, g] +// h: +// - i: 1 +// j: k +// - i: 2 +// j: l +// +// will result in +// +// "a", 1 +// "b.c", null +// "b.e": "f" +// "b.e.f": true +// "b.e": "g" +// "b.e.g": true +// "h.i": 1 +// "h.j": "k" +// "h.i": 2 +// "h.j": "l" +export function traverse(root: unknown): Kv[] { + const output: Kv[] = []; + + function visit(path: string, current: unknown) { + if (SPECIAL_KEYS.includes(path)) { + return; + } + + // empty or scalar + if ( + current === undefined || + current === null || + ['string', 'number', 'boolean'].includes(typeof current) + ) { + output.push({ key: path, value: current }); + return; + } + + // unknown + if (typeof current !== 'object') { + return; + } + + // array + if (Array.isArray(current)) { + for (const item of current) { + // NOTE(freben): The reason that these are output in two different ways, + // is to support use cases where you want to express that MORE than one + // tag is present in a list. Since the EntityFilters structure is a + // record, you can't have several entries of the same key. Therefore + // you will have to match on + // + // { "a.b": ["true"], "a.c": ["true"] } + // + // rather than + // + // { "a": ["b", "c"] } + // + // because the latter means EITHER b or c has to be present. + visit(path, item); + if (typeof item === 'string') { + output.push({ key: `${path}.${item}`, value: true }); + } + } + return; + } + + // object + for (const [key, value] of Object.entries(current!)) { + visit(path ? `${path}.${key}` : key, value); + } + } + + visit('', root); + + return output; +} + +// Translates a number of raw data rows to search table rows +export function mapToRows(input: Kv[], entityId: string): DbSearchRow[] { + const result: DbSearchRow[] = []; + + for (const { key: rawKey, value: rawValue } of input) { + const key = rawKey.toLocaleLowerCase('en-US'); + if (rawValue === undefined || rawValue === null) { + result.push({ entity_id: entityId, key, value: null }); + } else { + const value = String(rawValue).toLocaleLowerCase('en-US'); + if (value.length <= MAX_VALUE_LENGTH) { + result.push({ entity_id: entityId, key, value }); + } + } + } + + return result; +} + +/** + * Generates all of the search rows that are relevant for this entity. + * + * @param entityId The uid of the entity + * @param entity The entity + * @returns A list of entity search rows + */ +export function buildEntitySearch( + entityId: string, + entity: Entity, +): DbSearchRow[] { + // Visit the base structure recursively + const raw = traverse(entity); + + // Start with some special keys that are always present because you want to + // be able to easily search for null specifically + raw.push({ key: 'metadata.name', value: entity.metadata.name }); + raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); + raw.push({ key: 'metadata.uid', value: entity.metadata.uid }); + + // Namespace not specified has the default value "default", so we want to + // match on that as well + if (!entity.metadata.namespace) { + raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE }); + } + + // Visit relations + for (const relation of entity.relations ?? []) { + raw.push({ + key: 'relations', + value: `${relation.type}:${stringifyEntityRef(relation.target)}`, + }); + } + + return mapToRows(raw, entityId); +} diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index e0616fb336..cc9cc48ad6 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -120,6 +120,7 @@ export type ReplaceProcessingItemsRequest = removed: Entity[]; type: 'delta'; }; + export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; getNextProcessingItem(): Promise;