From 4e26ae5820ac89a94b2380d4e120bd15fca5e18c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 31 Mar 2021 16:29:18 +0200 Subject: [PATCH 01/23] Initial catalog proccessing rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelöw Co-authored-by: Tim Signed-off-by: Johan Haals --- .../20210302150147_refresh_state.js | 75 +++++++++ plugins/catalog-backend/package.json | 4 +- .../src/next/CatalogProcessingEngineImpl.ts | 125 ++++++++++++++ .../next/CatalogProcessingOrchestratorImpl.ts | 153 +++++++++++++++++ .../src/next/DatabaseLocationProvider.ts | 65 ++++++++ .../src/next/LocationServiceImpl.ts | 70 ++++++++ .../src/next/LocationStoreImpl.ts | 114 +++++++++++++ .../src/next/LocationToEntity.ts | 39 +++++ .../src/next/ProcessingStateManagerImpl.ts | 65 ++++++++ .../next/database/ProcessingDatabaseImpl.ts | 156 ++++++++++++++++++ .../src/next/database/types.ts | 68 ++++++++ plugins/catalog-backend/src/next/types.ts | 103 ++++++++++++ 12 files changed, 1036 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend/migrations/20210302150147_refresh_state.js create mode 100644 plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts create mode 100644 plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts create mode 100644 plugins/catalog-backend/src/next/DatabaseLocationProvider.ts create mode 100644 plugins/catalog-backend/src/next/LocationServiceImpl.ts create mode 100644 plugins/catalog-backend/src/next/LocationStoreImpl.ts create mode 100644 plugins/catalog-backend/src/next/LocationToEntity.ts create mode 100644 plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts create mode 100644 plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts create mode 100644 plugins/catalog-backend/src/next/database/types.ts create mode 100644 plugins/catalog-backend/src/next/types.ts diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js new file mode 100644 index 0000000000..ae8a02c785 --- /dev/null +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('refresh_state', table => { + table.comment('Location Refresh states'); + table.text('id').primary().notNullable().comment('Primary id'); + table + .text('entity_ref') + .primary() + .notNullable() + .comment('A reference to the entity that the refresh state is tied to'); + table + .text('unprocessed_entity') + .notNullable() + .comment( + 'The unprocessed entity (in its source form, before being run through all of the processors) as JSON', + ); + table + .text('processed_entity') + .nullable() + .comment( + 'The processed entity (after running through all processors, but before being stitched together with state and relations) as JSON', + ); + table + .text('cache') + .nullable() + .comment( + 'Cache information tied to the refreshing of this entity, such as etag information or actual response caching', + ); + table + .text('errors') + .notNullable() + .comment('JSON array containing all errors related to entity'); + table + .dateTime('next_update_at') + .notNullable() + .comment('Timestamp of when entity should be updated'); + table + .dateTime('last_discovery_at') + .notNullable() + .comment('The last timestamp of which this entity was discovered'); + table.index('entity_ref', 'entity_ref_idx'); + table.index('next_update_at', 'next_update_at_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_state', table => { + table.dropIndex([], 'entity_ref_idx'); + table.dropIndex([], 'next_update_at_idx'); + }); + await knex.schema.dropTable('refresh_state'); +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 531caa9ea5..2661025000 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -33,6 +33,7 @@ "@backstage/backend-common": "^0.6.1", "@backstage/catalog-model": "^0.7.5", "@backstage/config": "^0.1.4", + "@backstage/core": "^0.7.3", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", "@backstage/plugin-search-backend-node": "^0.1.2", @@ -59,7 +60,8 @@ "winston": "^3.2.1", "yaml": "^1.9.2", "yn": "^4.0.0", - "yup": "^0.29.3" + "yup": "^0.29.3", + "zen-observable": "^0.8.15" }, "devDependencies": { "@backstage/cli": "^0.6.6", diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts new file mode 100644 index 0000000000..e94cdf2f31 --- /dev/null +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Subscription } from '@backstage/core'; +import { + CatalogProcessingEngine, + EntityProvider, + EntityMessage, + ProcessingStateManager, + CatalogProcessingOrchestrator, +} from './types'; + +import { JsonObject } from '@backstage/config'; +import { EntitiesCatalog } from '../catalog/types'; +import { Logger } from 'winston'; + +export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { + private subscriptions: Subscription[] = []; + private running: boolean = false; + + constructor( + private readonly logger: Logger, + private readonly entityProviders: EntityProvider[], + private readonly stateManager: ProcessingStateManager, + private readonly orchestrator: CatalogProcessingOrchestrator, + private readonly entitiesCatalog: EntitiesCatalog, + ) {} + + async start() { + for (const provider of this.entityProviders) { + const subscription = provider + .entityChange$() + .subscribe({ next: m => this.onNext(m) }); + this.subscriptions.push(subscription); + } + + this.running = true; + + while (this.running) { + const { entity, state, eager } = await this.stateManager.pop(); + + const { + completeEntities, + deferredEntites, + errors, + } = await this.orchestrator.process({ + entity, + state, + eager, + }); + + for (const error of errors) { + this.logger.warn(error.message); + } + + for (const deferred of deferredEntites) { + this.stateManager.setResult({ + request: { entity: deferred, state: new Map() }, + nextRefresh: 'now()', + }); + } + + if (completeEntities.length) { + await this.entitiesCatalog.batchAddOrUpdateEntities( + completeEntities.map(e => ({ + entity: e, + relations: [], + })), + { + locationId: 'xyz', + dryRun: false, + outputEntities: true, + }, + ); + } + } + } + + async stop() { + this.running = false; + + for (const subscription of this.subscriptions) { + subscription.unsubscribe(); + } + } + private async onNext(message: EntityMessage) { + if ('all' in message) { + // TODO unhandled rejection + await Promise.all( + message.all.map(entity => + this.stateManager.setResult({ + request: { entity, state: new Map() }, + nextRefresh: 'now()', + }), + ), + ); + } + + if ('added' in message) { + await Promise.all( + message.added.map(entity => + this.stateManager.setResult({ + request: { entity, state: new Map() }, + nextRefresh: 'now()', + }), + ), + ); + + // TODO deletions of message.removed + } + } +} diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts new file mode 100644 index 0000000000..f78ebfbef2 --- /dev/null +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -0,0 +1,153 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + LocationSpec, + stringifyLocationReference, +} from '@backstage/catalog-model'; +import { JsonObject } from '@backstage/config'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, + CatalogProcessorResult, +} from '../ingestion/processors'; +import { CatalogProcessingOrchestrator, EntityProcessingError } from './types'; +import { Logger } from 'winston'; +import * as result from '../ingestion/processors/results'; +import { locationToEntity } from './LocationToEntity'; + +export class CatalogProcessingOrchestratorImpl + implements CatalogProcessingOrchestrator { + constructor( + private readonly options: { + processors: CatalogProcessor[]; + logger: Logger; + parser: CatalogProcessorParser; + }, + ) {} + + async process(request: { + entity: Entity; + eager?: boolean | undefined; + state: Map; + }): Promise<{ + state: Map; + completeEntities: Entity[]; + deferredEntites: Entity[]; + errors: EntityProcessingError[]; + }> { + const { entity, eager, state } = request; + const completedEntities: Entity[] = []; + const deferredEntites: Entity[] = []; + const errors: EntityProcessingError[] = []; + + if (eager) { + const stack = [entity]; + const emit = (i: CatalogProcessorResult) => stack.push(i); + while (stack.length) { + const item = stack.pop(); + stack.push(); + } + } else { + const emit = (i: CatalogProcessorResult) => { + if (i.type === 'entity') { + deferredEntites.push(i.entity); + } + if (i.type === 'location') { + deferredEntites.push( + locationToEntity(i.location.type, i.location.type), + ); + } + }; + if (entity.spec) { + if (entity.spec.kind === 'Location') { + this.handleLocation( + { + type: (entity.spec.type as unknown) as string, + target: (entity.spec.target as unknown) as string, + }, + emit, + ); + } + } + } + + return { + deferredEntites, + completeEntities: completedEntities, + errors, + state, + }; + } + + private async handleLocation( + location: LocationSpec, + emit: CatalogProcessorEmit, + ) { + const { processors, logger } = this.options; + + const validatedEmit: CatalogProcessorEmit = emitResult => { + if (emitResult.type === 'relation') { + throw new Error('readLocation may not emit entity relations'); + } + if ( + emitResult.type === 'location' && + emitResult.location.type === location.type && + emitResult.location.target === location.target + ) { + // Ignore self-referential locations silently (this can happen for + // example if you use a glob target like "**/*.yaml" in a Location + // entity) + return; + } + emit(emitResult); + }; + + for (const processor of processors) { + if (processor.readLocation) { + try { + if ( + await processor.readLocation( + location, + // TODO: figure out how this will work in the new processing system - locations from PRs might be optional + location.presence !== 'required', + validatedEmit, + this.options.parser, + ) + ) { + return; + } + } catch (e) { + const message = `Processor ${ + processor.constructor.name + } threw an error while reading location ${stringifyLocationReference( + location, + )}, ${e}`; + emit(result.generalError(location, message)); + logger.warn(message); + } + } + } + + const message = `No processor was able to read location ${stringifyLocationReference( + location, + )}`; + emit(result.inputError(location, message)); + logger.warn(message); + } +} diff --git a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts new file mode 100644 index 0000000000..b56dfa1158 --- /dev/null +++ b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable } from '@backstage/core'; +import { EntityProvider, LocationStore, EntityMessage } from './types'; +import ObservableImpl from 'zen-observable'; +import { locationToEntity, locationToEntityName } from './LocationToEntity'; + +export class DatabaseLocationProvider implements EntityProvider { + private subscribers = new Set< + ZenObservable.SubscriptionObserver + >(); + + constructor(private readonly store: LocationStore) { + store.location$().subscribe({ + next: locations => { + if ('all' in locations) { + this.notify({ + all: locations.all.map(l => locationToEntity(l.type, l.target)), + }); + } else { + this.notify({ + added: locations.added.map(l => locationToEntity(l.type, l.target)), + removed: locations.removed.map(l => + locationToEntityName(l.type, l.target), + ), + }); + } + }, + }); + } + + private notify(message: EntityMessage) { + for (const subscriber of this.subscribers) { + subscriber.next(message); + } + } + + entityChange$(): Observable { + return new ObservableImpl(subscriber => { + this.store.listLocations().then(locations => { + subscriber.next({ + all: locations.map(l => locationToEntity(l.type, l.target)), + }); + this.subscribers.add(subscriber); + }); + return () => { + this.subscribers.delete(subscriber); + }; + }); + } +} diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/LocationServiceImpl.ts new file mode 100644 index 0000000000..29b41e0a09 --- /dev/null +++ b/plugins/catalog-backend/src/next/LocationServiceImpl.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LocationSpec, Location, Entity } from '@backstage/catalog-model'; +import { + LocationService, + LocationStore, + CatalogProcessingOrchestrator, +} from './types'; + +export class LocationServiceImpl implements LocationService { + constructor( + private readonly store: LocationStore, + private readonly orchestrator: CatalogProcessingOrchestrator, + ) {} + + async createLocation( + spec: LocationSpec, + dryRun: boolean, + ): Promise<{ location: Location; entities: Entity[] }> { + if (dryRun) { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: `${spec.type}:${spec.target}`, + namespace: 'default', + }, + spec: { + location: { type: spec.type, target: spec.target }, + }, + }; + const processed = await this.orchestrator.process({ + entity, + eager: true, + state: new Map(), + }); + + return { + location: { ...spec, id: `${spec.type}:${spec.target}` }, + entities: processed.completeEntities, + }; + } + + const location = await this.store.createLocation(spec); + return { location, entities: [] }; + } + + listLocations(): Promise { + return this.store.listLocations(); + } + getLocation(id: string): Promise { + return this.store.getLocation(id); + } + deleteLocation(id: string): Promise { + return this.store.deleteLocation(id); + } +} diff --git a/plugins/catalog-backend/src/next/LocationStoreImpl.ts b/plugins/catalog-backend/src/next/LocationStoreImpl.ts new file mode 100644 index 0000000000..89247056d6 --- /dev/null +++ b/plugins/catalog-backend/src/next/LocationStoreImpl.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec, Location } from '@backstage/catalog-model'; +import { CommonDatabase } from '../database'; +import { LocationStore } from './types'; +import { v4 as uuidv4 } from 'uuid'; +import { ConflictError } from '@backstage/errors'; +import { Observable } from '@backstage/core'; +import ObservableImpl from 'zen-observable'; + +export type LocationMessage = + | { all: Location[] } + | { added: Location[]; removed: Location[] }; + +export class LocationStoreImpl implements LocationStore { + private subscribers = new Set< + ZenObservable.SubscriptionObserver + >(); + + constructor(private readonly db: CommonDatabase) {} + + createLocation(spec: LocationSpec): Promise { + return this.db.transaction(async tx => { + // TODO: id should really be type and target combined and not a uuid. + + // Attempt to find a previous location matching the spec + const previousLocations = await this.listLocations(); + const previousLocation = previousLocations.some( + l => spec.type === l.type && spec.target === l.target, + ); + + if (previousLocation) { + throw new ConflictError( + `Location ${spec.type}:${spec.target} already exists`, + ); + } + + const location = await this.db.addLocation(tx, { + id: uuidv4(), + type: spec.type, + target: spec.target, + }); + + this.notifyAddition(location); + + return location; + }); + } + + async listLocations(): Promise { + const dbLocations = await this.db.locations(); + return dbLocations.map(item => ({ + id: item.id, + target: item.target, + type: item.type, + })); + } + + getLocation(id: string): Promise { + return this.db.location(id); + } + + deleteLocation(id: string): Promise { + return this.db.transaction(async tx => { + const location = await this.db.location(id); + if (!location) { + throw new ConflictError(`No location found with with id: ${id}`); + } + await this.db.removeLocation(tx, id); + this.notifyDeletion(location); + }); + } + + private notifyAddition(location: Location) { + for (const subscriber of this.subscribers) { + subscriber.next({ + added: [location], + removed: [], + }); + } + } + + private notifyDeletion(location: Location) { + for (const subscriber of this.subscribers) { + subscriber.next({ + added: [], + removed: [location], + }); + } + } + + location$(): Observable { + return new ObservableImpl(subscriber => { + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); + } +} diff --git a/plugins/catalog-backend/src/next/LocationToEntity.ts b/plugins/catalog-backend/src/next/LocationToEntity.ts new file mode 100644 index 0000000000..28695e14af --- /dev/null +++ b/plugins/catalog-backend/src/next/LocationToEntity.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, EntityName } from '@backstage/catalog-model'; + +export function locationToEntity(type: string, target: string): Entity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: `${type}:${target}`, + namespace: 'default', + }, + spec: { + location: { type, target }, + }, + }; +} + +export function locationToEntityName(type: string, target: string): EntityName { + return { + kind: 'Location', + namespace: 'default', + name: `${type}:${target}`, + }; +} diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts new file mode 100644 index 0000000000..d8736cfa77 --- /dev/null +++ b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { JsonObject } from '@backstage/config'; +import { DbRefreshStateRow, ProcessingDatabase } from './database/types'; +import { ProcessingResult, ProcessingStateManager } from './types'; + +export class ProcessingStateManagerImpl implements ProcessingStateManager { + constructor(private readonly db: ProcessingDatabase) {} + + async setResult(result: ProcessingResult) { + await this.db.transaction(async tx => { + this.db.addEntityRefreshState(tx, [ + { + entity: result.request.entity, + nextRefresh: result.nextRefresh, + }, + ]); + }); + } + + async pop(): Promise<{ + entity: Entity; + eager?: boolean | undefined; + state: Map; + }> { + const entities = await new Promise(resolve => + this.popFromQueue(resolve), + ); + const result = entities[0]; + return { + entity: JSON.parse(result.unproccessed_entity) as Entity, + state: new Map(JSON.parse(result.cache)), + }; + } + + async popFromQueue(resolve: (rows: DbRefreshStateRow[]) => void) { + const entities = await this.db.transaction(async tx => { + return this.db.getProcessableEntities(tx, { + processBatchSize: 1, + }); + }); + + if (!entities.length) { + setTimeout(() => this.popFromQueue(resolve), 1000); + return; + } + + resolve(entities); + } +} diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts new file mode 100644 index 0000000000..af0d331199 --- /dev/null +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, NotFoundError } from '@backstage/errors'; +import { Knex } from 'knex'; +import { Transaction } from '../../database'; +import { + ProcessingDatabase, + AddUnprocessedEntitiesOptions, + UpdateProcessedEntityOptions, + GetProcessedEntitiesResult, +} from './types'; +import type { Logger } from 'winston'; +import { Entity } from '@backstage/catalog-model'; +import { v4 as uuid } from 'uuid'; + +export type DbRefreshStateRequest = { + entity: Entity; + nextRefresh: string; // TODO dateTime/ Date? +}; + +export type DbRefreshStateRow = { + id: string; + entity_ref: string; + unprocessed_entity: string; + processed_entity: string; + cache: string; + next_update_at: string; + last_discovery_at: string; // remove? + errors: string; +}; + +class ProcessingDatabaseImpl implements ProcessingDatabase { + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} + + async updateProcessedEntity( + txOpaque: Transaction, + options: UpdateProcessedEntityOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { id, processedEntity, cache, errors } = options; + const result = await tx('refresh_state') + .update({ + processed_entity: processedEntity, + cache, + errors, + }) + .where('id', id); + if (result === 0) { + throw new NotFoundError(`Processing state not found for ${id}`); + } + + // Update refs table + // Update fragments + } + + async addUnprocessedEntities( + txOpaque: Transaction, + options: AddUnprocessedEntitiesOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + for (const entity of options.unprocessedEntities) { + await tx('refresh_state') + .insert({ + id: uuid(), + entity_ref: stringifyEntityRef(entity), + unprocessed_entity: JSON.stringify(entity), + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }) + .onConflict('entity_ref') + .merge(['unprocessed_entity', 'last_discovery_at']); + } + } + + async getProcessableEntities( + txOpaque: Transaction, + request: { processBatchSize: number }, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + const items = await tx('refresh_state') + .select() + .where('next_update_at', '<=', tx.fn.now()) + .limit(request.processBatchSize) + .orderBy('next_update_at', 'desc'); + + await tx('refresh_state') + .whereIn( + 'entity_ref', + items.map(i => i.entity_ref), + ) + .update({ + next_update_at: + tx.client.config.client === 'sqlite3' + ? tx.raw(`datetime('now', ?)`, [`10 seconds`]) // TODO: test this in sqlite3 + : tx.raw(`now() + interval '30 seconds'`), + }); + + return items; + } + + async transaction(fn: (tx: Transaction) => Promise): Promise { + try { + let result: T | undefined = undefined; + + await this.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.logger.debug(`Error during transaction, ${e}`); + + if ( + /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || + /unique constraint/.test(e.message) + ) { + throw new ConflictError(`Rejected due to a conflicting entity`, e); + } + + throw e; + } + } +} + +function stringifyEntityRef( + entity: Entity, +): Knex.MaybeRawColumn | undefined { + throw new Error('Function not implemented.'); +} diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts new file mode 100644 index 0000000000..310b6b4b7e --- /dev/null +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { JsonObject } from '@backstage/config'; +import { Transaction } from '../../database/types'; + +export type AddUnprocessedEntitiesOptions = { + unprocessedEntities: Entity[]; +}; + +export type AddUnprocessedEntitiesResult = {}; + +export type UpdateProcessedEntityOptions = { + id: string; + processedEntity?: string; + cache?: string; + errors?: string; +}; + +export type RefreshStateItem = { + id: string; + entityRef: string; + unprocessedEntity: string; + processedEntity: string; + nextUpdateAt: string; + lastDiscoveryAt: string; // remove? + cache: JsonObject; + errors: string; +}; + +export type GetProcessedEntitiesResult = { + items: RefreshStateItem; +}; + +export interface ProcessingDatabase { + transaction(fn: (tx: Transaction) => Promise): Promise; + addUnprocessedEntities( + tx: Transaction, + options: AddUnprocessedEntitiesOptions, + ): Promise; + + getProcessableEntities( + txOpaque: Transaction, + request: { processBatchSize: number }, + ): Promise; + + /** + * Updates the + */ + updateProcessedEntity( + txOpaque: Transaction, + options: UpdateProcessedEntityOptions, + ): Promise; +} diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts new file mode 100644 index 0000000000..c7f1e370e0 --- /dev/null +++ b/plugins/catalog-backend/src/next/types.ts @@ -0,0 +1,103 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + EntityName, + LocationSpec, + Location, +} from '@backstage/catalog-model'; +import { JsonObject } from '@backstage/config'; +import { Observable } from '@backstage/core'; // << nooo + +export interface LocationEntity { + apiVersion: 'backstage.io/v1alpha1'; + kind: 'Location'; + metadata: { + name: string; // type:target + namespace: 'default'; + }; + spec: { + location: { type: string; target: string }; + }; +} + +export interface LocationService { + createLocation( + spec: LocationSpec, + dryRun: boolean, + ): Promise<{ location: Location; entities: Entity[] }>; + listLocations(): Promise; + getLocation(id: string): Promise; + deleteLocation(id: string): Promise; +} + +export type EntityMessage = + | { all: Entity[] } + | { added: Entity[]; removed: EntityName[] }; + +export interface LocationStore { + // extends EntityProvider + createLocation(spec: LocationSpec): Promise; + listLocations(): Promise; + getLocation(id: string): Promise; + deleteLocation(id: string): Promise; + + location$(): Observable< + { all: Location[] } | { added: Location[]; removed: Location[] } + >; +} + +export interface CatalogProcessingEngine { + start(): Promise; + stop(): Promise; +} + +export interface EntityProvider { + entityChange$(): Observable; +} + +// interface CatalogProcessor {} +type EntityProcessingRequest = { + entity: Entity; + eager?: boolean; + state: Map; // Versions for multiple deployments etc +}; + +export type EntityProcessingError = { + // some error stuff here + message: String; +}; + +type EntityProcessingResult = { + state: Map; + completeEntities: Entity[]; + deferredEntites: Entity[]; + errors: EntityProcessingError[]; +}; + +export interface CatalogProcessingOrchestrator { + process(request: EntityProcessingRequest): Promise; +} + +export type ProcessingResult = { + nextRefresh: string; + request: EntityProcessingRequest; +}; + +export interface ProcessingStateManager { + setResult(result: ProcessingResult): Promise; + pop(): Promise; +} From aaf8233a5ef944c390f45ad8fdefd77f52abf70d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 8 Apr 2021 16:48:29 +0200 Subject: [PATCH 02/23] Refactor types and interfaces Signed-off-by: Johan Haals --- .../src/next/CatalogProcessingEngineImpl.ts | 63 +++++++------------ .../next/CatalogProcessingOrchestratorImpl.ts | 24 +++---- .../src/next/ProcessingStateManagerImpl.ts | 57 ++++++++++------- .../next/database/ProcessingDatabaseImpl.ts | 31 +++++---- .../src/next/database/types.ts | 16 ++--- plugins/catalog-backend/src/next/types.ts | 29 ++++++--- 6 files changed, 111 insertions(+), 109 deletions(-) diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts index e94cdf2f31..8a2a0349d6 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -23,7 +23,6 @@ import { CatalogProcessingOrchestrator, } from './types'; -import { JsonObject } from '@backstage/config'; import { EntitiesCatalog } from '../catalog/types'; import { Logger } from 'winston'; @@ -50,42 +49,33 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { this.running = true; while (this.running) { - const { entity, state, eager } = await this.stateManager.pop(); + const { + id, + entity, + state: intialState, + } = await this.stateManager.getNextProccessingItem(); const { - completeEntities, + completedEntity, deferredEntites, errors, + state, } = await this.orchestrator.process({ entity, - state, - eager, + state: intialState, }); for (const error of errors) { this.logger.warn(error.message); } - for (const deferred of deferredEntites) { - this.stateManager.setResult({ - request: { entity: deferred, state: new Map() }, - nextRefresh: 'now()', - }); - } - - if (completeEntities.length) { - await this.entitiesCatalog.batchAddOrUpdateEntities( - completeEntities.map(e => ({ - entity: e, - relations: [], - })), - { - locationId: 'xyz', - dryRun: false, - outputEntities: true, - }, - ); - } + await this.stateManager.setProcessingItemResult({ + id, + entity: completedEntity, + state, + errors, + }); + await this.stateManager.addProcessingItems({ entities: deferredEntites }); } } @@ -96,28 +86,19 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { subscription.unsubscribe(); } } + private async onNext(message: EntityMessage) { if ('all' in message) { // TODO unhandled rejection - await Promise.all( - message.all.map(entity => - this.stateManager.setResult({ - request: { entity, state: new Map() }, - nextRefresh: 'now()', - }), - ), - ); + await this.stateManager.addProcessingItems({ + entities: message.all, + }); } if ('added' in message) { - await Promise.all( - message.added.map(entity => - this.stateManager.setResult({ - request: { entity, state: new Map() }, - nextRefresh: 'now()', - }), - ), - ); + await this.stateManager.addProcessingItems({ + entities: message.added, + }); // TODO deletions of message.removed } diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index f78ebfbef2..add7db53b3 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -19,14 +19,18 @@ import { LocationSpec, stringifyLocationReference, } from '@backstage/catalog-model'; -import { JsonObject } from '@backstage/config'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, CatalogProcessorResult, } from '../ingestion/processors'; -import { CatalogProcessingOrchestrator, EntityProcessingError } from './types'; +import { + CatalogProcessingOrchestrator, + EntityProcessingError, + EntityProcessingRequest, + EntityProcessingResult, +} from './types'; import { Logger } from 'winston'; import * as result from '../ingestion/processors/results'; import { locationToEntity } from './LocationToEntity'; @@ -41,18 +45,10 @@ export class CatalogProcessingOrchestratorImpl }, ) {} - async process(request: { - entity: Entity; - eager?: boolean | undefined; - state: Map; - }): Promise<{ - state: Map; - completeEntities: Entity[]; - deferredEntites: Entity[]; - errors: EntityProcessingError[]; - }> { + async process( + request: EntityProcessingRequest, + ): Promise { const { entity, eager, state } = request; - const completedEntities: Entity[] = []; const deferredEntites: Entity[] = []; const errors: EntityProcessingError[] = []; @@ -89,7 +85,7 @@ export class CatalogProcessingOrchestratorImpl return { deferredEntites, - completeEntities: completedEntities, + completedEntity, errors, state, }; diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts index d8736cfa77..872ee78fdb 100644 --- a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts +++ b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts @@ -14,52 +14,61 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; -import { JsonObject } from '@backstage/config'; -import { DbRefreshStateRow, ProcessingDatabase } from './database/types'; -import { ProcessingResult, ProcessingStateManager } from './types'; +import { ProcessingDatabase, RefreshStateItem } from './database/types'; +import { + AddProcessingItemRequest, + ProccessingItem, + ProcessingItemResult, + ProcessingStateManager, +} from './types'; export class ProcessingStateManagerImpl implements ProcessingStateManager { constructor(private readonly db: ProcessingDatabase) {} - async setResult(result: ProcessingResult) { - await this.db.transaction(async tx => { - this.db.addEntityRefreshState(tx, [ - { - entity: result.request.entity, - nextRefresh: result.nextRefresh, - }, - ]); + async setProcessingItemResult(result: ProcessingItemResult) { + return this.db.transaction(async tx => { + await this.db.updateProcessedEntity(tx, { + id: result.id, + processedEntity: result.entity, + errors: JSON.stringify(result.errors), + state: result.state, + }); }); } - async pop(): Promise<{ - entity: Entity; - eager?: boolean | undefined; - state: Map; - }> { - const entities = await new Promise(resolve => + async addProcessingItems(request: AddProcessingItemRequest) { + return this.db.transaction(async tx => { + await this.db.addUnprocessedEntities(tx, { + unprocessedEntities: request.entities, + }); + }); + } + + async getNextProccessingItem(): Promise { + const entities = await new Promise(resolve => this.popFromQueue(resolve), ); - const result = entities[0]; + const { id, state, unprocessedEntity } = entities[0]; return { - entity: JSON.parse(result.unproccessed_entity) as Entity, - state: new Map(JSON.parse(result.cache)), + id, + entity: unprocessedEntity, + state, }; } - async popFromQueue(resolve: (rows: DbRefreshStateRow[]) => void) { + async popFromQueue(resolve: (rows: RefreshStateItem[]) => void) { const entities = await this.db.transaction(async tx => { return this.db.getProcessableEntities(tx, { processBatchSize: 1, }); }); - if (!entities.length) { + // No entities require refresh, wait and try again. + if (!entities.items.length) { setTimeout(() => this.popFromQueue(resolve), 1000); return; } - resolve(entities); + resolve(entities.items); } } diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index af0d331199..3fe25c1eae 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -21,10 +21,10 @@ import { ProcessingDatabase, AddUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, - GetProcessedEntitiesResult, + GetProcessableEntitiesResult, } from './types'; import type { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { v4 as uuid } from 'uuid'; export type DbRefreshStateRequest = { @@ -54,11 +54,11 @@ class ProcessingDatabaseImpl implements ProcessingDatabase { options: UpdateProcessedEntityOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { id, processedEntity, cache, errors } = options; + const { id, processedEntity, state, errors } = options; const result = await tx('refresh_state') .update({ - processed_entity: processedEntity, - cache, + processed_entity: JSON.stringify(processedEntity), + cache: JSON.stringify(state), errors, }) .where('id', id); @@ -93,7 +93,7 @@ class ProcessingDatabaseImpl implements ProcessingDatabase { async getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, - ): Promise { + ): Promise { const tx = txOpaque as Knex.Transaction; const items = await tx('refresh_state') @@ -114,7 +114,18 @@ class ProcessingDatabaseImpl implements ProcessingDatabase { : tx.raw(`now() + interval '30 seconds'`), }); - return items; + return { + items: items.map(i => ({ + id: i.id, + entityRef: i.entity_ref, + unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, + processedEntity: JSON.parse(i.processed_entity) as Entity, + nextUpdateAt: i.next_update_at, + lastDiscoveryAt: i.last_discovery_at, + state: JSON.parse(i.cache), + errors: i.errors, + })), + }; } async transaction(fn: (tx: Transaction) => Promise): Promise { @@ -148,9 +159,3 @@ class ProcessingDatabaseImpl implements ProcessingDatabase { } } } - -function stringifyEntityRef( - entity: Entity, -): Knex.MaybeRawColumn | undefined { - throw new Error('Function not implemented.'); -} diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 310b6b4b7e..6cd42e690f 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -26,24 +26,24 @@ export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { id: string; - processedEntity?: string; - cache?: string; + processedEntity?: Entity; + state?: Map; errors?: string; }; export type RefreshStateItem = { id: string; entityRef: string; - unprocessedEntity: string; - processedEntity: string; + unprocessedEntity: Entity; + processedEntity: Entity; nextUpdateAt: string; lastDiscoveryAt: string; // remove? - cache: JsonObject; + state: Map; errors: string; }; -export type GetProcessedEntitiesResult = { - items: RefreshStateItem; +export type GetProcessableEntitiesResult = { + items: RefreshStateItem[]; }; export interface ProcessingDatabase { @@ -56,7 +56,7 @@ export interface ProcessingDatabase { getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, - ): Promise; + ): Promise; /** * Updates the diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index c7f1e370e0..58d88636f8 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -69,8 +69,7 @@ export interface EntityProvider { entityChange$(): Observable; } -// interface CatalogProcessor {} -type EntityProcessingRequest = { +export type EntityProcessingRequest = { entity: Entity; eager?: boolean; state: Map; // Versions for multiple deployments etc @@ -81,9 +80,9 @@ export type EntityProcessingError = { message: String; }; -type EntityProcessingResult = { +export type EntityProcessingResult = { state: Map; - completeEntities: Entity[]; + completedEntity: Entity; deferredEntites: Entity[]; errors: EntityProcessingError[]; }; @@ -92,12 +91,24 @@ export interface CatalogProcessingOrchestrator { process(request: EntityProcessingRequest): Promise; } -export type ProcessingResult = { - nextRefresh: string; - request: EntityProcessingRequest; +export type ProcessingItemResult = { + id: string; + entity: Entity; + state: Map; + errors: EntityProcessingError[]; }; +export type AddProcessingItemRequest = { + entities: Entity[]; +}; + +export type ProccessingItem = { + id: string; + entity: Entity; + state: Map; +}; export interface ProcessingStateManager { - setResult(result: ProcessingResult): Promise; - pop(): Promise; + setProcessingItemResult(result: ProcessingItemResult): Promise; + getNextProccessingItem(): Promise; + addProcessingItems(request: AddProcessingItemRequest): Promise; } From abfbacba46aae31105164ff5a570f81cc78b013b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 9 Apr 2021 16:22:18 +0200 Subject: [PATCH 03/23] Make entity ref unique Signed-off-by: Johan Haals --- .../catalog-backend/migrations/20210302150147_refresh_state.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index ae8a02c785..6ee10531eb 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -25,7 +25,7 @@ exports.up = async function up(knex) { table.text('id').primary().notNullable().comment('Primary id'); table .text('entity_ref') - .primary() + .unique() .notNullable() .comment('A reference to the entity that the refresh state is tied to'); table From 7f2a9b0d079885d6728c520575677b8f8a8db9a7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 9 Apr 2021 16:23:46 +0200 Subject: [PATCH 04/23] Use Database instead of CommonDatabase Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/LocationStoreImpl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/next/LocationStoreImpl.ts b/plugins/catalog-backend/src/next/LocationStoreImpl.ts index 89247056d6..1b9e9bd92c 100644 --- a/plugins/catalog-backend/src/next/LocationStoreImpl.ts +++ b/plugins/catalog-backend/src/next/LocationStoreImpl.ts @@ -15,7 +15,7 @@ */ import { LocationSpec, Location } from '@backstage/catalog-model'; -import { CommonDatabase } from '../database'; +import { Database } from '../database'; import { LocationStore } from './types'; import { v4 as uuidv4 } from 'uuid'; import { ConflictError } from '@backstage/errors'; @@ -31,7 +31,7 @@ export class LocationStoreImpl implements LocationStore { ZenObservable.SubscriptionObserver >(); - constructor(private readonly db: CommonDatabase) {} + constructor(private readonly db: Database) {} createLocation(spec: LocationSpec): Promise { return this.db.transaction(async tx => { From bfa51b286f052a77b20682c4f5d72db4abf0441a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 9 Apr 2021 16:25:20 +0200 Subject: [PATCH 05/23] Add entity processing logic Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../src/next/CatalogProcessingEngineImpl.ts | 23 +- .../next/CatalogProcessingOrchestratorImpl.ts | 227 +++++++++++------- .../src/next/LocationServiceImpl.ts | 2 +- .../next/database/ProcessingDatabaseImpl.ts | 3 +- plugins/catalog-backend/src/next/types.ts | 25 +- 5 files changed, 163 insertions(+), 117 deletions(-) diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts index 8a2a0349d6..93616b6399 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -55,27 +55,28 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { state: intialState, } = await this.stateManager.getNextProccessingItem(); - const { - completedEntity, - deferredEntites, - errors, - state, - } = await this.orchestrator.process({ + const result = await this.orchestrator.process({ entity, state: intialState, }); - for (const error of errors) { + for (const error of result.errors) { this.logger.warn(error.message); } + if (!result.ok) { + return; + } + await this.stateManager.setProcessingItemResult({ id, - entity: completedEntity, - state, - errors, + entity: result.completedEntity, + state: result.state, + errors: result.errors, + }); + await this.stateManager.addProcessingItems({ + entities: result.deferredEntites, }); - await this.stateManager.addProcessingItems({ entities: deferredEntites }); } } diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index add7db53b3..1288561b07 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -16,24 +16,23 @@ import { Entity, - LocationSpec, - stringifyLocationReference, + EntityRelationSpec, + stringifyEntityRef, + LOCATION_ANNOTATION, + EntityPolicy, } from '@backstage/catalog-model'; import { CatalogProcessor, - CatalogProcessorEmit, CatalogProcessorParser, CatalogProcessorResult, } from '../ingestion/processors'; import { CatalogProcessingOrchestrator, - EntityProcessingError, EntityProcessingRequest, EntityProcessingResult, } from './types'; import { Logger } from 'winston'; -import * as result from '../ingestion/processors/results'; -import { locationToEntity } from './LocationToEntity'; +import { InputError } from '@backstage/errors'; export class CatalogProcessingOrchestratorImpl implements CatalogProcessingOrchestrator { @@ -42,6 +41,7 @@ export class CatalogProcessingOrchestratorImpl processors: CatalogProcessor[]; logger: Logger; parser: CatalogProcessorParser; + policy: EntityPolicy; }, ) {} @@ -49,101 +49,144 @@ export class CatalogProcessingOrchestratorImpl request: EntityProcessingRequest, ): Promise { const { entity, eager, state } = request; - const deferredEntites: Entity[] = []; - const errors: EntityProcessingError[] = []; - if (eager) { - const stack = [entity]; - const emit = (i: CatalogProcessorResult) => stack.push(i); - while (stack.length) { - const item = stack.pop(); - stack.push(); - } - } else { - const emit = (i: CatalogProcessorResult) => { - if (i.type === 'entity') { - deferredEntites.push(i.entity); - } - if (i.type === 'location') { - deferredEntites.push( - locationToEntity(i.location.type, i.location.type), - ); - } - }; - if (entity.spec) { - if (entity.spec.kind === 'Location') { - this.handleLocation( - { - type: (entity.spec.type as unknown) as string, - target: (entity.spec.target as unknown) as string, - }, - emit, - ); - } - } - } + const result = await this.processSingleEntity(entity); - return { - deferredEntites, - completedEntity, - errors, - state, - }; + console.log('result', JSON.stringify(result)); + return result; } - private async handleLocation( - location: LocationSpec, - emit: CatalogProcessorEmit, - ) { - const { processors, logger } = this.options; + private async processSingleEntity( + unprocessedEntity: Entity, + ): Promise { + // TODO: validate that this doesn't change during processing + const entityRef = stringifyEntityRef(unprocessedEntity); + // TODO: which one do we actually use here? source-location? - maybe probably doesn't exist yet? + const locationRef = + unprocessedEntity.metadata?.annotations?.[LOCATION_ANNOTATION]; - const validatedEmit: CatalogProcessorEmit = emitResult => { - if (emitResult.type === 'relation') { - throw new Error('readLocation may not emit entity relations'); - } - if ( - emitResult.type === 'location' && - emitResult.location.type === location.type && - emitResult.location.target === location.target - ) { - // Ignore self-referential locations silently (this can happen for - // example if you use a glob target like "**/*.yaml" in a Location - // entity) - return; - } - emit(emitResult); - }; + const emitter = createEmitter(this.options.logger); - for (const processor of processors) { - if (processor.readLocation) { - try { - if ( - await processor.readLocation( - location, - // TODO: figure out how this will work in the new processing system - locations from PRs might be optional - location.presence !== 'required', - validatedEmit, - this.options.parser, - ) - ) { - return; + try { + // Pre-process phase, used to populate entities with data that is required during main processing step + let entity = unprocessedEntity; + for (const processor of this.options.processors) { + if (processor.preProcessEntity) { + try { + entity = await processor.preProcessEntity({ + entity, + emit: emitter.emit, + }); + } catch (e) { + throw new Error( + `Processor ${processor.constructor.name} threw an error while preprocessing entity ${entityRef} at ${locationRef}, ${e}`, + ); } - } catch (e) { - const message = `Processor ${ - processor.constructor.name - } threw an error while reading location ${stringifyLocationReference( - location, - )}, ${e}`; - emit(result.generalError(location, message)); - logger.warn(message); } } - } - const message = `No processor was able to read location ${stringifyLocationReference( - location, - )}`; - emit(result.inputError(location, message)); - logger.warn(message); + // Enforce entity policies making sure that entities conform to a general schema + let policyEnforcedEntity; + try { + policyEnforcedEntity = await this.options.policy.enforce(entity); + } catch (e) { + throw new InputError( + `Policy check failed while analyzing entity ${entityRef} at ${locationRef}, ${e}`, + ); + } + if (!policyEnforcedEntity) { + throw new Error( + `Policy unexpectedly returned no data while analyzing entity ${entityRef} at ${locationRef}`, + ); + } + entity = policyEnforcedEntity; + + // Validate the given entity kind against its schema + let handled = false; + for (const processor of this.options.processors) { + if (processor.validateEntityKind) { + try { + handled = await processor.validateEntityKind(entity); + if (handled) { + break; + } + } catch (e) { + throw new InputError( + `Processor ${processor.constructor.name} threw an error while validating the entity ${entityRef} at ${locationRef}, ${e}`, + ); + } + } + } + if (!handled) { + throw new InputError( + `No processor recognized the entity ${entityRef} at ${locationRef}`, + ); + } + + // Main processing step of the entity + for (const processor of this.options.processors) { + if (processor.processEntity) { + try { + entity = await processor.processEntity({ + entity, + emit: emitter.emit, + }); + } catch (e) { + throw new Error( + `Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`, + ); + } + } + } + + return { + ...emitter.results(), + completedEntity: entity, + state: new Map(), + ok: true, + }; + } catch (error) { + this.options.logger.warn(error.message); + return { ok: false, errors: emitter.results().errors.concat(error) }; + } } } + +function createEmitter(logger: Logger) { + let done = false; + + const errors = new Array(); + const relations = new Array(); + const deferredEntites = new Array(); + + const emit = (i: CatalogProcessorResult) => { + console.log('CatalogProcessorResult', i); + if (done) { + logger.warn( + `Item if type ${i.type} was emitted after processing had completed at ${ + new Error().stack + }`, + ); + return; + } + if (i.type === 'entity') { + deferredEntites.push(i.entity); + } else if (i.type === 'relation') { + relations.push(i.relation); + } else if (i.type === 'error') { + errors.push(i.error); + } + }; + + return { + emit, + results() { + done = true; + return { + errors, + relations, + deferredEntites, + }; + }, + }; +} diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/LocationServiceImpl.ts index 29b41e0a09..e8e8745f01 100644 --- a/plugins/catalog-backend/src/next/LocationServiceImpl.ts +++ b/plugins/catalog-backend/src/next/LocationServiceImpl.ts @@ -50,7 +50,7 @@ export class LocationServiceImpl implements LocationService { return { location: { ...spec, id: `${spec.type}:${spec.target}` }, - entities: processed.completeEntities, + entities: [processed.completedEntity], }; } diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index 3fe25c1eae..25181335c4 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -43,7 +43,7 @@ export type DbRefreshStateRow = { errors: string; }; -class ProcessingDatabaseImpl implements ProcessingDatabase { +export class ProcessingDatabaseImpl implements ProcessingDatabase { constructor( private readonly database: Knex, private readonly logger: Logger, @@ -82,6 +82,7 @@ class ProcessingDatabaseImpl implements ProcessingDatabase { id: uuid(), entity_ref: stringifyEntityRef(entity), unprocessed_entity: JSON.stringify(entity), + errors: '', next_update_at: tx.fn.now(), last_discovery_at: tx.fn.now(), }) diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 58d88636f8..091ddc8bcd 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -75,17 +75,18 @@ export type EntityProcessingRequest = { state: Map; // Versions for multiple deployments etc }; -export type EntityProcessingError = { - // some error stuff here - message: String; -}; - -export type EntityProcessingResult = { - state: Map; - completedEntity: Entity; - deferredEntites: Entity[]; - errors: EntityProcessingError[]; -}; +export type EntityProcessingResult = + | { + ok: true; + state: Map; + completedEntity: Entity; + deferredEntites: Entity[]; + errors: Error[]; + } + | { + ok: false; + errors: Error[]; + }; export interface CatalogProcessingOrchestrator { process(request: EntityProcessingRequest): Promise; @@ -95,7 +96,7 @@ export type ProcessingItemResult = { id: string; entity: Entity; state: Map; - errors: EntityProcessingError[]; + errors: Error[]; }; export type AddProcessingItemRequest = { From 07de96af3be4ecd6406193014faa885bb4fa3ee0 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Apr 2021 11:25:48 +0200 Subject: [PATCH 06/23] Backwards compatible processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- .../next/CatalogProcessingOrchestratorImpl.ts | 152 ++++++++++++++++-- .../src/next/DatabaseLocationProvider.ts | 20 ++- .../src/next/LocationServiceImpl.ts | 12 +- .../src/next/LocationToEntity.ts | 39 ----- plugins/catalog-backend/src/next/util.ts | 90 +++++++++++ 5 files changed, 254 insertions(+), 59 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/LocationToEntity.ts create mode 100644 plugins/catalog-backend/src/next/util.ts diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index 1288561b07..9aaa2ffa74 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -19,7 +19,12 @@ import { EntityRelationSpec, stringifyEntityRef, LOCATION_ANNOTATION, + LocationSpec, + LocationEntity, EntityPolicy, + ORIGIN_LOCATION_ANNOTATION, + stringifyLocationReference, + parseLocationReference, } from '@backstage/catalog-model'; import { CatalogProcessor, @@ -33,12 +38,56 @@ import { } from './types'; import { Logger } from 'winston'; import { InputError } from '@backstage/errors'; +import { locationSpecToLocationEntity } from './util'; +import path from 'path'; +import * as results from '../ingestion/processors/results'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +function isLocationEntity(entity: Entity): entity is LocationEntity { + return entity.kind === 'Location'; +} + +function getEntityOriginLocationRef(entity: Entity): string { + const ref = entity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION]; + if (!ref) { + const entityRef = stringifyEntityRef(entity); + throw new InputError( + `Entity '${entityRef}' does not have an origin location`, + ); + } + return ref; +} + +function toAbsoluteUrl( + integrations: ScmIntegrationRegistry, + base: LocationSpec, + type: string, + target: string, +): string { + if (base.type !== type) { + return target; + } + try { + if (type === 'file') { + if (target.startsWith('.')) { + return path.join(path.dirname(base.target), target); + } + return target; + } else if (type === 'url') { + return integrations.resolveUrl({ url: target, base: base.target }); + } + return target; + } catch (e) { + return target; + } +} export class CatalogProcessingOrchestratorImpl implements CatalogProcessingOrchestrator { constructor( private readonly options: { processors: CatalogProcessor[]; + integrations: ScmIntegrationRegistry; logger: Logger; parser: CatalogProcessorParser; policy: EntityPolicy; @@ -52,7 +101,7 @@ export class CatalogProcessingOrchestratorImpl const result = await this.processSingleEntity(entity); - console.log('result', JSON.stringify(result)); + console.log('result', JSON.stringify(result, undefined, 2)); return result; } @@ -64,19 +113,27 @@ export class CatalogProcessingOrchestratorImpl // TODO: which one do we actually use here? source-location? - maybe probably doesn't exist yet? const locationRef = unprocessedEntity.metadata?.annotations?.[LOCATION_ANNOTATION]; + if (!locationRef) { + throw new InputError(`Entity '${entityRef}' does not have a location`); + } + const location = parseLocationReference(locationRef); + const originLocation = parseLocationReference( + getEntityOriginLocationRef(unprocessedEntity), + ); - const emitter = createEmitter(this.options.logger); - + const emitter = createEmitter(this.options.logger, unprocessedEntity); try { // Pre-process phase, used to populate entities with data that is required during main processing step let entity = unprocessedEntity; for (const processor of this.options.processors) { if (processor.preProcessEntity) { try { - entity = await processor.preProcessEntity({ + entity = await processor.preProcessEntity( entity, - emit: emitter.emit, - }); + location, + emitter.emit, + originLocation, + ); } catch (e) { throw new Error( `Processor ${processor.constructor.name} threw an error while preprocessing entity ${entityRef} at ${locationRef}, ${e}`, @@ -123,14 +180,69 @@ export class CatalogProcessingOrchestratorImpl ); } + // Backwards compatible processing of location entites + if (isLocationEntity(entity)) { + const { type = location.type, optional = false } = entity.spec; + const targets = new Array(); + if (entity.spec.target) { + targets.push(entity.spec.target); + } + if (entity.spec.targets) { + targets.push(...entity.spec.targets); + } + + for (const maybeRelativeTarget of targets) { + if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { + emitter.emit( + results.inputError( + location, + `LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, + ), + ); + continue; + } + const target = toAbsoluteUrl( + this.options.integrations, + location, + type, + maybeRelativeTarget, + ); + + for (const processor of this.options.processors) { + if (processor.readLocation) { + try { + handled = await processor.readLocation( + { + type, + target, + presence: optional ? 'optional' : 'required', + }, + Boolean(entity.spec?.optional), + emitter.emit, + this.options.parser, + ); + if (handled) { + break; + } + } catch (e) { + throw new Error( + `Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`, + ); + } + } + } + } + } + // Main processing step of the entity for (const processor of this.options.processors) { - if (processor.processEntity) { + if (processor.postProcessEntity) { try { - entity = await processor.processEntity({ + entity = await processor.postProcessEntity( entity, - emit: emitter.emit, - }); + location, + emitter.emit, + ); } catch (e) { throw new Error( `Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`, @@ -152,7 +264,7 @@ export class CatalogProcessingOrchestratorImpl } } -function createEmitter(logger: Logger) { +function createEmitter(logger: Logger, parentEntity: Entity) { let done = false; const errors = new Array(); @@ -170,7 +282,23 @@ function createEmitter(logger: Logger) { return; } if (i.type === 'entity') { - deferredEntites.push(i.entity); + const originLocation = getEntityOriginLocationRef(parentEntity); + + deferredEntites.push({ + ...i.entity, + metadata: { + ...i.entity.metadata, + annotations: { + ...i.entity.metadata.annotations, + [ORIGIN_LOCATION_ANNOTATION]: originLocation, + [LOCATION_ANNOTATION]: stringifyLocationReference(i.location), + }, + }, + }); + } else if (i.type === 'location') { + deferredEntites.push( + locationSpecToLocationEntity(i.location, parentEntity), + ); } else if (i.type === 'relation') { relations.push(i.relation); } else if (i.type === 'error') { diff --git a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts index b56dfa1158..eaf5a78556 100644 --- a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts +++ b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts @@ -15,9 +15,13 @@ */ import { Observable } from '@backstage/core'; +import { ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { EntityProvider, LocationStore, EntityMessage } from './types'; import ObservableImpl from 'zen-observable'; -import { locationToEntity, locationToEntityName } from './LocationToEntity'; +import { + locationSpecToLocationEntity, + locationSpecToMetadataName, +} from './util'; export class DatabaseLocationProvider implements EntityProvider { private subscribers = new Set< @@ -29,14 +33,16 @@ export class DatabaseLocationProvider implements EntityProvider { next: locations => { if ('all' in locations) { this.notify({ - all: locations.all.map(l => locationToEntity(l.type, l.target)), + all: locations.all.map(l => locationSpecToLocationEntity(l)), }); } else { this.notify({ - added: locations.added.map(l => locationToEntity(l.type, l.target)), - removed: locations.removed.map(l => - locationToEntityName(l.type, l.target), - ), + added: locations.added.map(l => locationSpecToLocationEntity(l)), + removed: locations.removed.map(l => ({ + kind: 'Location', + namespace: ENTITY_DEFAULT_NAMESPACE, + name: locationSpecToMetadataName(l), + })), }); } }, @@ -53,7 +59,7 @@ export class DatabaseLocationProvider implements EntityProvider { return new ObservableImpl(subscriber => { this.store.listLocations().then(locations => { subscriber.next({ - all: locations.map(l => locationToEntity(l.type, l.target)), + all: locations.map(l => locationSpecToLocationEntity(l)), }); this.subscribers.add(subscriber); }); diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/LocationServiceImpl.ts index e8e8745f01..02a98fb2b9 100644 --- a/plugins/catalog-backend/src/next/LocationServiceImpl.ts +++ b/plugins/catalog-backend/src/next/LocationServiceImpl.ts @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { LocationSpec, Location, Entity } from '@backstage/catalog-model'; +import { + LocationSpec, + Location, + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; import { LocationService, LocationStore, @@ -37,6 +43,10 @@ export class LocationServiceImpl implements LocationService { metadata: { name: `${spec.type}:${spec.target}`, namespace: 'default', + annotations: { + [LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`, + [ORIGIN_LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`, + }, }, spec: { location: { type: spec.type, target: spec.target }, diff --git a/plugins/catalog-backend/src/next/LocationToEntity.ts b/plugins/catalog-backend/src/next/LocationToEntity.ts deleted file mode 100644 index 28695e14af..0000000000 --- a/plugins/catalog-backend/src/next/LocationToEntity.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, EntityName } from '@backstage/catalog-model'; - -export function locationToEntity(type: string, target: string): Entity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - name: `${type}:${target}`, - namespace: 'default', - }, - spec: { - location: { type, target }, - }, - }; -} - -export function locationToEntityName(type: string, target: string): EntityName { - return { - kind: 'Location', - namespace: 'default', - name: `${type}:${target}`, - }; -} diff --git a/plugins/catalog-backend/src/next/util.ts b/plugins/catalog-backend/src/next/util.ts new file mode 100644 index 0000000000..6ff1cde2f0 --- /dev/null +++ b/plugins/catalog-backend/src/next/util.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + LocationSpec, + LocationEntityV1alpha1, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, + stringifyEntityRef, + stringifyLocationReference, +} from '@backstage/catalog-model'; +import { createHash } from 'crypto'; + +export function locationSpecToMetadataName(location: LocationSpec) { + const hash = createHash('sha1') + .update(`${location.type}:${location.target}`) + .digest('hex'); + + return `generated-${hash}`; +} + +export function locationSpecToLocationEntity( + location: LocationSpec, + parentEntity?: Entity, +): LocationEntityV1alpha1 { + let ownLocation: string; + let originLocation: string; + if (parentEntity) { + const maybeOwnLocation = + parentEntity.metadata.annotations?.[LOCATION_ANNOTATION]; + if (!maybeOwnLocation) { + throw new Error( + `Parent entity '${stringifyEntityRef( + parentEntity, + )}' of location '${stringifyLocationReference( + location, + )}' does not have a location annotation`, + ); + } + ownLocation = maybeOwnLocation; + const maybeOriginLocation = + parentEntity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION]; + if (!maybeOriginLocation) { + throw new Error( + `Parent entity '${stringifyEntityRef( + parentEntity, + )}' of location '${stringifyLocationReference( + location, + )}' does not have an origin location annotation`, + ); + } + originLocation = maybeOriginLocation; + } else { + ownLocation = stringifyLocationReference(location); + originLocation = ownLocation; + } + + const result: LocationEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: locationSpecToMetadataName(location), + annotations: { + [LOCATION_ANNOTATION]: ownLocation, + [ORIGIN_LOCATION_ANNOTATION]: originLocation, + }, + }, + spec: { + type: location.type, + target: location.target, + optional: location.presence === 'optional' ? true : undefined, + }, + }; + + return result; +} From 77b9fa5e07647fe3c60c117df9222eea6cc3d0db Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Apr 2021 13:35:28 +0200 Subject: [PATCH 07/23] Add NextCatalogBuilder, fix typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- plugins/catalog-backend/src/index.ts | 1 + .../src/next/CatalogProcessingEngineImpl.ts | 2 +- .../next/CatalogProcessingOrchestratorImpl.ts | 1 - .../src/next/LocationServiceImpl.ts | 11 +- .../src/next/NextCatalogBuilder.ts | 364 ++++++++++++++++++ .../src/next/ProcessingStateManagerImpl.ts | 2 +- plugins/catalog-backend/src/next/index.ts | 17 + plugins/catalog-backend/src/next/types.ts | 2 +- 8 files changed, 392 insertions(+), 8 deletions(-) create mode 100644 plugins/catalog-backend/src/next/NextCatalogBuilder.ts create mode 100644 plugins/catalog-backend/src/next/index.ts diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 51df909d05..058ca728a1 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -20,3 +20,4 @@ export * from './ingestion'; export * from './search'; export * from './service'; export * from './util'; +export * from './next'; diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts index 93616b6399..46c6fb0c1b 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -53,7 +53,7 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { id, entity, state: intialState, - } = await this.stateManager.getNextProccessingItem(); + } = await this.stateManager.getNextProcessingItem(); const result = await this.orchestrator.process({ entity, diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index 9aaa2ffa74..8706abc162 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -272,7 +272,6 @@ function createEmitter(logger: Logger, parentEntity: Entity) { const deferredEntites = new Array(); const emit = (i: CatalogProcessorResult) => { - console.log('CatalogProcessorResult', i); if (done) { logger.warn( `Item if type ${i.type} was emitted after processing had completed at ${ diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/LocationServiceImpl.ts index 02a98fb2b9..4125f96ed1 100644 --- a/plugins/catalog-backend/src/next/LocationServiceImpl.ts +++ b/plugins/catalog-backend/src/next/LocationServiceImpl.ts @@ -57,11 +57,14 @@ export class LocationServiceImpl implements LocationService { eager: true, state: new Map(), }); + if (processed.ok) { + return { + location: { ...spec, id: `${spec.type}:${spec.target}` }, + entities: [processed.completedEntity], + }; + } - return { - location: { ...spec, id: `${spec.type}:${spec.target}` }, - entities: [processed.completedEntity], - }; + throw Error('error handling not implemented.'); } const location = await this.store.createLocation(spec); diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts new file mode 100644 index 0000000000..5525e35f7f --- /dev/null +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -0,0 +1,364 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; +import { + DefaultNamespaceEntityPolicy, + EntityPolicies, + EntityPolicy, + FieldFormatEntityPolicy, + makeValidator, + NoForeignRootFieldsEntityPolicy, + SchemaValidEntityPolicy, + Validators, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import lodash from 'lodash'; +import { Logger } from 'winston'; +import { + DatabaseEntitiesCatalog, + DatabaseLocationsCatalog, + EntitiesCatalog, + LocationsCatalog, +} from '../catalog'; +import { DatabaseManager } from '../database'; +import { + AnnotateLocationEntityProcessor, + BitbucketDiscoveryProcessor, + BuiltinKindsEntityProcessor, + CatalogProcessor, + CatalogProcessorParser, + CodeOwnersProcessor, + FileReaderProcessor, + GithubDiscoveryProcessor, + GithubOrgReaderProcessor, + HigherOrderOperation, + HigherOrderOperations, + LdapOrgReaderProcessor, + LocationEntityProcessor, + LocationReaders, + MicrosoftGraphOrgReaderProcessor, + PlaceholderProcessor, + PlaceholderResolver, + StaticLocationProcessor, + UrlReaderProcessor, +} from '../ingestion'; +import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; +import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; +import { + jsonPlaceholderResolver, + textPlaceholderResolver, + yamlPlaceholderResolver, +} from '../ingestion/processors/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +import { LocationAnalyzer } from '../ingestion/types'; +import { CatalogProcessingEngineImpl } from '../next/CatalogProcessingEngineImpl'; +import { CatalogProcessingOrchestratorImpl } from '../next/CatalogProcessingOrchestratorImpl'; +import { ProcessingDatabaseImpl } from '../next/database/ProcessingDatabaseImpl'; +import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; +import { LocationStoreImpl } from '../next/LocationStoreImpl'; +import { ProcessingStateManagerImpl } from '../next/ProcessingStateManagerImpl'; +import { CatalogProcessingEngine } from '../next/types'; + +export type CatalogEnvironment = { + logger: Logger; + database: PluginDatabaseManager; + config: Config; + reader: UrlReader; +}; + +/** + * A builder that helps wire up all of the component parts of the catalog. + * + * The touch points where you can replace or extend behavior are as follows: + * + * - Entity policies can be added or replaced. These are automatically run + * after the processors' pre-processing steps. All policies are given the + * chance to inspect the entity, and all of them have to pass in order for + * the entity to be considered valid from an overall point of view. + * - Placeholder resolvers can be replaced or added. These run on the raw + * structured data between the parsing and pre-processing steps, to replace + * dollar-prefixed entries with their actual values (like $file). + * - Field format validators can be replaced. These check the format of + * individual core fields such as metadata.name, to ensure that they adhere + * to certain rules. + * - Processors can be added or replaced. These implement the functionality of + * reading, parsing, validating, and processing the entity data before it is + * persisted in the catalog. + */ +export class NextCatalogBuilder { + private readonly env: CatalogEnvironment; + private entityPolicies: EntityPolicy[]; + private entityPoliciesReplace: boolean; + private placeholderResolvers: Record; + private fieldFormatValidators: Partial; + private processors: CatalogProcessor[]; + private processorsReplace: boolean; + private parser: CatalogProcessorParser | undefined; + + constructor(env: CatalogEnvironment) { + this.env = env; + this.entityPolicies = []; + this.entityPoliciesReplace = false; + this.placeholderResolvers = {}; + this.fieldFormatValidators = {}; + this.processors = []; + this.processorsReplace = false; + this.parser = undefined; + } + + /** + * Adds policies that are used to validate entities between the pre- + * processing and post-processing stages. All such policies must pass for the + * entity to be considered valid. + * + * If what you want to do is to replace the rules for what format is allowed + * in various core entity fields (such as metadata.name), you may want to use + * {@link NextCatalogBuilder#setFieldFormatValidators} instead. + * + * @param policies One or more policies + */ + addEntityPolicy(...policies: EntityPolicy[]): NextCatalogBuilder { + this.entityPolicies.push(...policies); + return this; + } + + /** + * Sets what policies to use for validation of entities between the pre- + * processing and post-processing stages. All such policies must pass for the + * entity to be considered valid. + * + * If what you want to do is to replace the rules for what format is allowed + * in various core entity fields (such as metadata.name), you may want to use + * {@link NextCatalogBuilder#setFieldFormatValidators} instead. + * + * This function replaces the default set of policies; use with care. + * + * @param policies One or more policies + */ + replaceEntityPolicies(policies: EntityPolicy[]): NextCatalogBuilder { + this.entityPolicies = [...policies]; + this.entityPoliciesReplace = true; + return this; + } + + /** + * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity + * definition files. + * + * @param key The key that identifies the placeholder, e.g. "file" + * @param resolver The resolver that gets values for this placeholder + */ + setPlaceholderResolver( + key: string, + resolver: PlaceholderResolver, + ): NextCatalogBuilder { + this.placeholderResolvers[key] = resolver; + return this; + } + + /** + * Sets the validator function to use for one or more special fields of an + * entity. This is useful if the default rules for formatting of fields are + * not sufficient. + * + * This function has no effect if used together with + * {@link NextCatalogBuilder#replaceEntityPolicies}. + * + * @param validators The (subset of) validators to set + */ + setFieldFormatValidators( + validators: Partial, + ): NextCatalogBuilder { + lodash.merge(this.fieldFormatValidators, validators); + return this; + } + + /** + * Adds entity processors. These are responsible for reading, parsing, and + * processing entities before they are persisted in the catalog. + * + * @param processors One or more processors + */ + addProcessor(...processors: CatalogProcessor[]): NextCatalogBuilder { + this.processors.push(...processors); + return this; + } + + /** + * Sets what entity processors to use. These are responsible for reading, + * parsing, and processing entities before they are persisted in the catalog. + * + * This function replaces the default set of processors; use with care. + * + * @param processors One or more processors + */ + replaceProcessors(processors: CatalogProcessor[]): NextCatalogBuilder { + this.processors = [...processors]; + this.processorsReplace = true; + return this; + } + + /** + * Sets up the catalog to use a custom parser for entity data. + * + * This is the function that gets called immediately after some raw entity + * specification data has been read from a remote source, and needs to be + * parsed and emitted as structured data. + * + * @param parser The custom parser + */ + setEntityDataParser(parser: CatalogProcessorParser): NextCatalogBuilder { + this.parser = parser; + return this; + } + + /** + * Wires up and returns all of the component parts of the catalog + */ + async build(): Promise<{ + entitiesCatalog: EntitiesCatalog; + locationsCatalog: LocationsCatalog; + locationAnalyzer: LocationAnalyzer; + processingEngine: CatalogProcessingEngine; + }> { + const { config, database, logger } = this.env; + + const policy = this.buildEntityPolicy(); + const processors = this.buildProcessors(); + const parser = this.parser || defaultEntityDataParser; + + const dbClient = await database.getClient(); + const db = await DatabaseManager.createDatabase(dbClient, { logger }); + + const processingDatabase = new ProcessingDatabaseImpl(dbClient, logger); + const stateManager = new ProcessingStateManagerImpl(processingDatabase); + const integrations = ScmIntegrations.fromConfig(config); + const orchestrator = new CatalogProcessingOrchestratorImpl({ + processors, + integrations, + logger, + parser, + policy, + }); + const entitiesCatalog = new DatabaseEntitiesCatalog(db, this.env.logger); + + const locationStore = new LocationStoreImpl(db); + const dbLocationProvider = new DatabaseLocationProvider(locationStore); + const processingEngine = new CatalogProcessingEngineImpl( + logger, + [dbLocationProvider], // entityproviders + stateManager, + orchestrator, + entitiesCatalog, + ); + + const locationsCatalog = new DatabaseLocationsCatalog(db); + const locationAnalyzer = new RepoLocationAnalyzer(logger); + + return { + entitiesCatalog, + locationsCatalog, + locationAnalyzer, + processingEngine, + }; + } + + private buildEntityPolicy(): EntityPolicy { + const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace + ? [new SchemaValidEntityPolicy(), ...this.entityPolicies] + : [ + new SchemaValidEntityPolicy(), + new DefaultNamespaceEntityPolicy(), + new NoForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy( + makeValidator(this.fieldFormatValidators), + ), + ...this.entityPolicies, + ]; + + return EntityPolicies.allOf(entityPolicies); + } + + private buildProcessors(): CatalogProcessor[] { + const { config, logger, reader } = this.env; + const integrations = ScmIntegrations.fromConfig(config); + + this.checkDeprecatedReaderProcessors(); + + const placeholderResolvers: Record = { + json: jsonPlaceholderResolver, + yaml: yamlPlaceholderResolver, + text: textPlaceholderResolver, + ...this.placeholderResolvers, + }; + + // These are always there no matter what + const processors: CatalogProcessor[] = [ + StaticLocationProcessor.fromConfig(config), + new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), + new BuiltinKindsEntityProcessor(), + ]; + + // These are only added unless the user replaced them all + if (!this.processorsReplace) { + processors.push( + new FileReaderProcessor(), + BitbucketDiscoveryProcessor.fromConfig(config, { logger }), + GithubDiscoveryProcessor.fromConfig(config, { logger }), + GithubOrgReaderProcessor.fromConfig(config, { logger }), + LdapOrgReaderProcessor.fromConfig(config, { logger }), + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), + new UrlReaderProcessor({ reader, logger }), + CodeOwnersProcessor.fromConfig(config, { logger, reader }), + // new LocationEntityProcessor({ integrations }), + new AnnotateLocationEntityProcessor({ integrations }), + ); + } + + // Add the ones (if any) that the user added + processors.push(...this.processors); + + return processors; + } + + // TODO(Rugvip): These old processors are removed, for a while we'll be throwing + // errors here to make sure people know where to move the config + private checkDeprecatedReaderProcessors() { + const pc = this.env.config.getOptionalConfig('catalog.processors'); + if (pc?.has('github')) { + throw new Error( + `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, + ); + } + if (pc?.has('gitlabApi')) { + throw new Error( + `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, + ); + } + if (pc?.has('bitbucketApi')) { + throw new Error( + `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, + ); + } + if (pc?.has('azureApi')) { + throw new Error( + `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, + ); + } + } +} diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts index 872ee78fdb..bb35da6c37 100644 --- a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts +++ b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts @@ -44,7 +44,7 @@ export class ProcessingStateManagerImpl implements ProcessingStateManager { }); } - async getNextProccessingItem(): Promise { + async getNextProcessingItem(): Promise { const entities = await new Promise(resolve => this.popFromQueue(resolve), ); diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts new file mode 100644 index 0000000000..d4eeca4c81 --- /dev/null +++ b/plugins/catalog-backend/src/next/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { NextCatalogBuilder } from './NextCatalogBuilder'; diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index 091ddc8bcd..ddd2d6ff7c 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -110,6 +110,6 @@ export type ProccessingItem = { }; export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; - getNextProccessingItem(): Promise; + getNextProcessingItem(): Promise; addProcessingItems(request: AddProcessingItemRequest): Promise; } From 5dafa268f7b180796cbd1168e9a233351eb5ee41 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Apr 2021 13:35:55 +0200 Subject: [PATCH 08/23] Add optional flag to LocationEntityV1Alpha Signed-off-by: Johan Haals --- packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index fb452b6ac7..20f99f15b3 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -31,6 +31,7 @@ export interface LocationEntityV1alpha1 extends Entity { type?: string; target?: string; targets?: string[]; + optional?: boolean; }; } From 8f6a17c04c37748d07553003d075fb4d939737e9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 14 Apr 2021 09:53:16 +0200 Subject: [PATCH 09/23] Add experimental option to start the new catalog Signed-off-by: Johan Haals --- packages/backend/src/plugins/catalog.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 076840944f..99ba769736 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -18,6 +18,7 @@ import { useHotCleanup } from '@backstage/backend-common'; import { CatalogBuilder, createRouter, + NextCatalogBuilder, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; @@ -26,6 +27,28 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin( env: PluginEnvironment, ): Promise { + // HIGHLY experimental rework of the software catalog + if (process.env.EXPERIMENTAL_CATALOG === '1') { + const builder = new NextCatalogBuilder(env); + const { + entitiesCatalog, + locationsCatalog, + locationAnalyzer, + processingEngine, + } = await builder.build(); + + // TODO(jhaals): run and manage in background. + processingEngine.start(); + + return await createRouter({ + entitiesCatalog, + locationsCatalog, + locationAnalyzer, + logger: env.logger, + config: env.config, + }); + } + const builder = new CatalogBuilder(env); const { entitiesCatalog, From 4ba6ed45e1a8baf4389ceed54aa685e52817cf42 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 14 Apr 2021 09:56:23 +0200 Subject: [PATCH 10/23] Add relations and stitcher class Signed-off-by: Johan Haals --- plugins/catalog-backend/package.json | 1 + .../src/next/CatalogProcessingEngineImpl.ts | 17 ++++- .../src/next/NextCatalogBuilder.ts | 11 ++-- plugins/catalog-backend/src/next/Stitcher.ts | 65 +++++++++++++++++++ .../next/database/ProcessingDatabaseImpl.ts | 46 ++++++++++++- .../src/next/database/types.ts | 11 ++++ plugins/catalog-backend/src/next/types.ts | 2 + yarn.lock | 2 +- 8 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 plugins/catalog-backend/src/next/Stitcher.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 2661025000..6011562078 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -47,6 +47,7 @@ "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "fast-json-stable-stringify": "^2.1.0", "fs-extra": "^9.0.0", "git-url-parse": "^11.4.4", "glob": "^7.1.6", diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts index 46c6fb0c1b..99b2b19091 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -23,8 +23,13 @@ import { CatalogProcessingOrchestrator, } from './types'; -import { EntitiesCatalog } from '../catalog/types'; import { Logger } from 'winston'; +import { + Entity, + stringifyEntityRef, + EntityRelationSpec, +} from '@backstage/catalog-model'; +import { Stitcher } from './Stitcher'; export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { private subscriptions: Subscription[] = []; @@ -35,7 +40,7 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { private readonly entityProviders: EntityProvider[], private readonly stateManager: ProcessingStateManager, private readonly orchestrator: CatalogProcessingOrchestrator, - private readonly entitiesCatalog: EntitiesCatalog, + private readonly stitcher: Stitcher, ) {} async start() { @@ -77,6 +82,14 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { await this.stateManager.addProcessingItems({ entities: result.deferredEntites, }); + + const setOfThingsToStitch = new Set([ + stringifyEntityRef(result.completedEntity), + ...result.relations.map(relation => + stringifyEntityRef(relation.source), + ), + ]); + await this.stitcher.stitch(setOfThingsToStitch); } } diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 5525e35f7f..95c767407e 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -46,18 +46,13 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, - HigherOrderOperation, - HigherOrderOperations, LdapOrgReaderProcessor, - LocationEntityProcessor, - LocationReaders, MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, StaticLocationProcessor, UrlReaderProcessor, } from '../ingestion'; -import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; import { jsonPlaceholderResolver, @@ -73,6 +68,7 @@ import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; import { LocationStoreImpl } from '../next/LocationStoreImpl'; import { ProcessingStateManagerImpl } from '../next/ProcessingStateManagerImpl'; import { CatalogProcessingEngine } from '../next/types'; +import { Stitcher } from './Stitcher'; export type CatalogEnvironment = { logger: Logger; @@ -255,16 +251,17 @@ export class NextCatalogBuilder { parser, policy, }); - const entitiesCatalog = new DatabaseEntitiesCatalog(db, this.env.logger); + const entitiesCatalog = new DatabaseEntitiesCatalog(db, logger); const locationStore = new LocationStoreImpl(db); const dbLocationProvider = new DatabaseLocationProvider(locationStore); + const stitcher = new Stitcher(dbClient, logger); const processingEngine = new CatalogProcessingEngineImpl( logger, [dbLocationProvider], // entityproviders stateManager, orchestrator, - entitiesCatalog, + stitcher, ); const locationsCatalog = new DatabaseLocationsCatalog(db); diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts new file mode 100644 index 0000000000..8adc93bfad --- /dev/null +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -0,0 +1,65 @@ +import { Transaction } from '../database'; + +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { ConflictError } from '@backstage/errors'; + +export class Stitcher { + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} + + async stitch(entityRefs: Set) { + console.log(entityRefs); + } + + private async transaction( + fn: (tx: Transaction) => Promise, + ): Promise { + try { + let result: T | undefined = undefined; + + await this.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.logger.debug(`Error during transaction, ${e}`); + + if ( + /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || + /unique constraint/.test(e.message) + ) { + throw new ConflictError(`Rejected due to a conflicting entity`, e); + } + + throw e; + } + } +} diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index 25181335c4..bc2d913a73 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -14,17 +14,21 @@ * limitations under the License. */ -import { ConflictError, NotFoundError } from '@backstage/errors'; +import { ConflictError, NotFoundError, InputError } from '@backstage/errors'; import { Knex } from 'knex'; import { Transaction } from '../../database'; +import { DbEntitiesRow } from '../../database/types'; import { ProcessingDatabase, AddUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, GetProcessableEntitiesResult, + UpdateFinalEntityOptions, } from './types'; import type { Logger } from 'winston'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { createHash } from 'crypto'; +import stringify from 'fast-json-stable-stringify'; import { v4 as uuid } from 'uuid'; export type DbRefreshStateRequest = { @@ -43,12 +47,52 @@ export type DbRefreshStateRow = { errors: string; }; +function generateEntityEtag(entity: Entity) { + return createHash('sha1') + .update(stringify({ ...entity })) + .digest('hex'); +} + export class ProcessingDatabaseImpl implements ProcessingDatabase { constructor( private readonly database: Knex, private readonly logger: Logger, ) {} + async updateFinalEntity( + txOpaque: Transaction, + options: UpdateFinalEntityOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + const { finalEntity } = options; + const { relations } = finalEntity; + const { uid, etag, generation } = finalEntity.metadata; + + if (uid === undefined || etag === undefined || generation === undefined) { + throw new InputError( + 'One of the metadata fields "uid", "etag", or "generation" was missing', + ); + } else if (relations === undefined) { + throw new InputError('The field "relations" was missing'); + } + // TODO(freben): state/errors? + + const fullName = stringifyEntityRef(finalEntity); + + const result = await tx('entities') + .insert({ + id: uid, + location_id: null, + etag, + generation, + full_name: fullName, + data: JSON.stringify(finalEntity), + }) + .onConflict('full_name') + .merge(['etag', 'generation', 'data']); + } + async updateProcessedEntity( txOpaque: Transaction, options: UpdateProcessedEntityOptions, diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 6cd42e690f..83483693c4 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -46,8 +46,19 @@ export type GetProcessableEntitiesResult = { items: RefreshStateItem[]; }; +export type UpdateFinalEntityOptions = { + finalEntity: Entity; + // TODO(freben): search +}; + export interface ProcessingDatabase { transaction(fn: (tx: Transaction) => Promise): Promise; + + updateFinalEntity( + txOpaque: Transaction, + options: UpdateFinalEntityOptions, + ): Promise; + addUnprocessedEntities( tx: Transaction, options: AddUnprocessedEntitiesOptions, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index ddd2d6ff7c..cd3beb4e4c 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -18,6 +18,7 @@ import { EntityName, LocationSpec, Location, + EntityRelationSpec, } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Observable } from '@backstage/core'; // << nooo @@ -81,6 +82,7 @@ export type EntityProcessingResult = state: Map; completedEntity: Entity; deferredEntites: Entity[]; + relations: EntityRelationSpec[]; errors: Error[]; } | { diff --git a/yarn.lock b/yarn.lock index 6f7e171b2c..46f4e73e2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13043,7 +13043,7 @@ fast-json-patch@^3.0.0-1: resolved "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.0.0-1.tgz#4c68f2e7acfbab6d29d1719c44be51899c93dabb" integrity sha512-6pdFb07cknxvPzCeLsFHStEy+MysPJPgZQ9LbQ/2O67unQF93SNqfdSqnPPl71YMHX+AD8gbl7iuoGFzHEdDuw== -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: +fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== From e4e6ae88b1862de8ac6a2d2ae66718a7072cd15d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 16 Apr 2021 09:26:30 +0200 Subject: [PATCH 11/23] Add tables for references and relations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../20210302150147_refresh_state.js | 112 +++++++++++++++++- 1 file changed, 106 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index 6ee10531eb..a09d892146 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -20,9 +20,63 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { + await knex.schema.createTable('relations', table => { + table.comment('All relations between entities in the catalog'); + table + .text('originating_entity_id') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .text('source_entity_ref') + .notNullable() + .comment('The entity reference of the source entity of the relation'); + table + .text('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .text('target_entity_ref') + .notNullable() + .comment('The entity reference of the target entity of the relation'); + + table.index( + ['source_entity_ref', 'type', 'target_entity_ref'], + 'relations_full_ref_idx', + ); + }); + + await knex.schema.createTable('final_entities', table => { + table.comment( + 'This table contains the final entity result after processing and stitching', + ); + table + .text('entity_id') + .primary() + .notNullable() + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment( + 'Entity ID which correspond to the ID in the refresh_state table', + ); + + table.text('finalized_entity').notNullable().comment('The final entity'); + }); + await knex.schema.createTable('refresh_state', table => { - table.comment('Location Refresh states'); - table.text('id').primary().notNullable().comment('Primary id'); + table.comment( + 'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.', + ); + table + .text('entity_id') + .primary() + .notNullable() + .comment( + 'Primary ID, which will also be used as the uid of the resulting entity', + ); table .text('entity_ref') .unique() @@ -58,8 +112,48 @@ exports.up = async function up(knex) { .dateTime('last_discovery_at') .notNullable() .comment('The last timestamp of which this entity was discovered'); - table.index('entity_ref', 'entity_ref_idx'); - table.index('next_update_at', 'next_update_at_idx'); + table.index('entity_ref', 'refresh_state_entity_ref_idx'); + table.index('next_update_at', 'refresh_state_next_update_at_idx'); + }); + + await knex.schema.createTable('refresh_state_references', table => { + table.comment( + 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', + ); + table + .text('source_special_key') + .nullable() + .comment( + 'When the reference source is not an entity, this is an opaque identifier for that source.', + ); + table + .text('source_entity_id') + .nullable() + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment( + 'When the reference source is an entity, this is the ID of the source entity.', + ); + table + .text('target_entity_id') + .notNullable() + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment('The ID of the target entity.'); + table.index( + 'source_special_key', + 'refresh_state_references_source_special_key_idx', + ); + table.index( + 'source_entity_id', + 'refresh_state_references_source_entity_id_idx', + ); + table.index( + 'target_entity_id', + 'refresh_state_references_target_entity_id_idx', + ); }); }; @@ -67,9 +161,15 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_state_references', table => { + table.dropIndex([], 'refresh_state_references_source_special_key_idx'); + table.dropIndex([], 'refresh_state_references_source_entity_id_idx'); + table.dropIndex([], 'refresh_state_references_target_entity_id_idx'); + }); await knex.schema.alterTable('refresh_state', table => { - table.dropIndex([], 'entity_ref_idx'); - table.dropIndex([], 'next_update_at_idx'); + table.dropIndex([], 'refresh_state_entity_ref_idx'); + table.dropIndex([], 'refresh_state_next_update_at_idx'); }); await knex.schema.dropTable('refresh_state'); + await knex.schema.dropTable('references'); }; From dbbc5af9681ee5daf80bfaa804466c7d92549539 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 16 Apr 2021 09:29:05 +0200 Subject: [PATCH 12/23] Add processing type Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../src/next/CatalogProcessingEngineImpl.ts | 21 +-- .../next/CatalogProcessingOrchestratorImpl.ts | 1 - .../src/next/ProcessingStateManagerImpl.ts | 6 +- .../next/database/ProcessingDatabaseImpl.ts | 157 ++++++++++++------ .../src/next/database/types.ts | 20 +-- plugins/catalog-backend/src/next/types.ts | 5 + 6 files changed, 129 insertions(+), 81 deletions(-) diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts index 99b2b19091..41ad2bef8e 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts @@ -24,11 +24,7 @@ import { } from './types'; import { Logger } from 'winston'; -import { - Entity, - stringifyEntityRef, - EntityRelationSpec, -} from '@backstage/catalog-model'; +import { stringifyEntityRef } from '@backstage/catalog-model'; import { Stitcher } from './Stitcher'; export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { @@ -45,9 +41,10 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { async start() { for (const provider of this.entityProviders) { + const id = 'databaseProvider'; const subscription = provider .entityChange$() - .subscribe({ next: m => this.onNext(m) }); + .subscribe({ next: m => this.onNext(id, m) }); this.subscriptions.push(subscription); } @@ -73,14 +70,14 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { return; } + result.completedEntity.metadata.uid = id; await this.stateManager.setProcessingItemResult({ id, entity: result.completedEntity, state: result.state, errors: result.errors, - }); - await this.stateManager.addProcessingItems({ - entities: result.deferredEntites, + relations: result.relations, + deferredEntities: result.deferredEntites, }); const setOfThingsToStitch = new Set([ @@ -101,16 +98,20 @@ export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { } } - private async onNext(message: EntityMessage) { + private async onNext(id: string, message: EntityMessage) { if ('all' in message) { // TODO unhandled rejection await this.stateManager.addProcessingItems({ + id, + type: 'provider', entities: message.all, }); } if ('added' in message) { await this.stateManager.addProcessingItems({ + id, + type: 'provider', entities: message.added, }); diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index 8706abc162..fee61cb366 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -101,7 +101,6 @@ export class CatalogProcessingOrchestratorImpl const result = await this.processSingleEntity(entity); - console.log('result', JSON.stringify(result, undefined, 2)); return result; } diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts index bb35da6c37..6cd846d791 100644 --- a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts +++ b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts @@ -32,15 +32,15 @@ export class ProcessingStateManagerImpl implements ProcessingStateManager { processedEntity: result.entity, errors: JSON.stringify(result.errors), state: result.state, + relations: result.relations, + deferedEntities: result.deferredEntities, }); }); } async addProcessingItems(request: AddProcessingItemRequest) { return this.db.transaction(async tx => { - await this.db.addUnprocessedEntities(tx, { - unprocessedEntities: request.entities, - }); + await this.db.addUnprocessedEntities(tx, request); }); } diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index bc2d913a73..f479c9a4f8 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -14,30 +14,25 @@ * limitations under the License. */ -import { ConflictError, NotFoundError, InputError } from '@backstage/errors'; +import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { Transaction } from '../../database'; -import { DbEntitiesRow } from '../../database/types'; +import lodash from 'lodash'; + import { ProcessingDatabase, AddUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, GetProcessableEntitiesResult, - UpdateFinalEntityOptions, } from './types'; import type { Logger } from 'winston'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { createHash } from 'crypto'; -import stringify from 'fast-json-stable-stringify'; +import stableStringify from 'fast-json-stable-stringify'; import { v4 as uuid } from 'uuid'; -export type DbRefreshStateRequest = { - entity: Entity; - nextRefresh: string; // TODO dateTime/ Date? -}; - export type DbRefreshStateRow = { - id: string; + entity_id: string; entity_ref: string; unprocessed_entity: string; processed_entity: string; @@ -47,71 +42,101 @@ export type DbRefreshStateRow = { errors: string; }; +export type DbRelationsRow = { + originating_entity_id: string; + source_entity_ref: string; + target_entity_ref: string; + type: string; +}; + +export type DbRefreshStateReferences = { + source_special_key?: string; + source_entity_id?: string; + target_entity_id: string; +}; + function generateEntityEtag(entity: Entity) { return createHash('sha1') - .update(stringify({ ...entity })) + .update(stableStringify({ ...entity })) .digest('hex'); } +// The number of items that are sent per batch to the database layer, when +// doing .batchInsert calls to knex. This needs to be low enough to not cause +// errors in the underlying engine due to exceeding query limits, but large +// enough to get the speed benefits. +const BATCH_SIZE = 50; + export class ProcessingDatabaseImpl implements ProcessingDatabase { constructor( private readonly database: Knex, private readonly logger: Logger, ) {} - async updateFinalEntity( - txOpaque: Transaction, - options: UpdateFinalEntityOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const { finalEntity } = options; - const { relations } = finalEntity; - const { uid, etag, generation } = finalEntity.metadata; - - if (uid === undefined || etag === undefined || generation === undefined) { - throw new InputError( - 'One of the metadata fields "uid", "etag", or "generation" was missing', - ); - } else if (relations === undefined) { - throw new InputError('The field "relations" was missing'); - } - // TODO(freben): state/errors? - - const fullName = stringifyEntityRef(finalEntity); - - const result = await tx('entities') - .insert({ - id: uid, - location_id: null, - etag, - generation, - full_name: fullName, - data: JSON.stringify(finalEntity), - }) - .onConflict('full_name') - .merge(['etag', 'generation', 'data']); - } - async updateProcessedEntity( txOpaque: Transaction, options: UpdateProcessedEntityOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { id, processedEntity, state, errors } = options; - const result = await tx('refresh_state') + const { + id, + processedEntity, + state, + errors, + relations, + deferedEntities, + } = options; + + const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), cache: JSON.stringify(state), errors, }) - .where('id', id); - if (result === 0) { + .where('entity_id', id); + + if (refreshResult === 0) { throw new NotFoundError(`Processing state not found for ${id}`); } - // Update refs table + // Schedule all deferred entities for future processing. + await this.addUnprocessedEntities(tx, { + entities: deferedEntities, + id, + type: 'entity', + }); + // Update fragments + + // Update relations + const entityRef = stringifyEntityRef(processedEntity); + + // Delete old relations + await tx('relations') + .where({ originating_entity_id: id }) + .delete(); + + // Batch insert new relations + const relationRows: DbRelationsRow[] = relations.map( + ({ source, target, type }) => ({ + originating_entity_id: id, + source_entity_ref: stringifyEntityRef(source), + target_entity_ref: stringifyEntityRef(target), + type, + }), + ); + await tx.batchInsert( + 'relations', + this.deduplicateRelations(relationRows), + BATCH_SIZE, + ); + } + + private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] { + return lodash.uniqBy( + rows, + r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`, + ); } async addUnprocessedEntities( @@ -119,12 +144,14 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { options: AddUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; + const entityIds = new Array(); - for (const entity of options.unprocessedEntities) { + for (const entity of options.entities) { + const entityRef = stringifyEntityRef(entity); await tx('refresh_state') .insert({ - id: uuid(), - entity_ref: stringifyEntityRef(entity), + entity_id: uuid(), + entity_ref: entityRef, unprocessed_entity: JSON.stringify(entity), errors: '', next_update_at: tx.fn.now(), @@ -132,7 +159,29 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { }) .onConflict('entity_ref') .merge(['unprocessed_entity', 'last_discovery_at']); + + const [{ entity_id: entityId }] = await tx( + 'refresh_state', + ).where({ entity_ref: entityRef }); + entityIds.push(entityId); } + + const key = + options.type === 'provider' + ? { source_special_key: options.id } + : { source_entity_id: options.id }; + // copied from update refs + await tx('refresh_state_references') + .where(key) + .delete(); + + const referenceRows: DbRefreshStateReferences[] = entityIds.map( + entityId => ({ + ...key, + target_entity_id: entityId, + }), + ); + await tx.batchInsert('refresh_state_references', referenceRows, BATCH_SIZE); } async getProcessableEntities( @@ -161,7 +210,7 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { return { items: items.map(i => ({ - id: i.id, + id: i.entity_id, entityRef: i.entity_ref, unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity, processedEntity: JSON.parse(i.processed_entity) as Entity, diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 83483693c4..d89a3b23bf 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -14,21 +14,25 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { Entity, EntityRelationSpec } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Transaction } from '../../database/types'; export type AddUnprocessedEntitiesOptions = { - unprocessedEntities: Entity[]; + type: 'entity' | 'provider'; + id: string; + entities: Entity[]; }; export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { id: string; - processedEntity?: Entity; + processedEntity: Entity; state?: Map; errors?: string; + relations: EntityRelationSpec[]; + deferedEntities: Entity[]; }; export type RefreshStateItem = { @@ -46,19 +50,9 @@ export type GetProcessableEntitiesResult = { items: RefreshStateItem[]; }; -export type UpdateFinalEntityOptions = { - finalEntity: Entity; - // TODO(freben): search -}; - export interface ProcessingDatabase { transaction(fn: (tx: Transaction) => Promise): Promise; - updateFinalEntity( - txOpaque: Transaction, - options: UpdateFinalEntityOptions, - ): Promise; - addUnprocessedEntities( tx: Transaction, options: AddUnprocessedEntitiesOptions, diff --git a/plugins/catalog-backend/src/next/types.ts b/plugins/catalog-backend/src/next/types.ts index cd3beb4e4c..0aeb290224 100644 --- a/plugins/catalog-backend/src/next/types.ts +++ b/plugins/catalog-backend/src/next/types.ts @@ -99,9 +99,13 @@ export type ProcessingItemResult = { entity: Entity; state: Map; errors: Error[]; + relations: EntityRelationSpec[]; + deferredEntities: Entity[]; }; export type AddProcessingItemRequest = { + type: 'entity' | 'provider'; + id: string; entities: Entity[]; }; @@ -110,6 +114,7 @@ export type ProccessingItem = { entity: Entity; state: Map; }; + export interface ProcessingStateManager { setProcessingItemResult(result: ProcessingItemResult): Promise; getNextProcessingItem(): Promise; From ec43f706c71f728dfd57dfec20c65b004a71dbf6 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 19 Apr 2021 10:53:57 +0200 Subject: [PATCH 13/23] Copy migrations to migrationsv2 before migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- packages/backend/src/plugins/catalog.ts | 6 +- .../20210302150147_refresh_state.js | 92 +++++++++--------- plugins/catalog-backend/package.json | 1 + .../src/next/NextCatalogBuilder.ts | 28 +++++- .../src/next/NextEntitiesCatalog.ts | 61 ++++++++++++ plugins/catalog-backend/src/next/Stitcher.ts | 96 ++++++++++++++++++- .../next/database/ProcessingDatabaseImpl.ts | 11 --- tsconfig.json | 3 +- 8 files changed, 231 insertions(+), 67 deletions(-) rename plugins/catalog-backend/{migrations => migrationsv2}/20210302150147_refresh_state.js (98%) create mode 100644 plugins/catalog-backend/src/next/NextEntitiesCatalog.ts diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 99ba769736..e58d8cd3eb 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -27,7 +27,11 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin( env: PluginEnvironment, ): Promise { - // HIGHLY experimental rework of the software catalog + /* + * ** WARNING ** + * DO NOT enable the experimental catalog, it will brick your database migrations. + * This is solely for internal backstage development. + */ if (process.env.EXPERIMENTAL_CATALOG === '1') { const builder = new NextCatalogBuilder(env); const { diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js similarity index 98% rename from plugins/catalog-backend/migrations/20210302150147_refresh_state.js rename to plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index a09d892146..56c594ddf2 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -20,52 +20,6 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - await knex.schema.createTable('relations', table => { - table.comment('All relations between entities in the catalog'); - table - .text('originating_entity_id') - .references('entity_id') - .inTable('refresh_state') - .onDelete('CASCADE') - .notNullable() - .comment('The entity that provided the relation'); - table - .text('source_entity_ref') - .notNullable() - .comment('The entity reference of the source entity of the relation'); - table - .text('type') - .notNullable() - .comment('The type of the relation between the entities'); - table - .text('target_entity_ref') - .notNullable() - .comment('The entity reference of the target entity of the relation'); - - table.index( - ['source_entity_ref', 'type', 'target_entity_ref'], - 'relations_full_ref_idx', - ); - }); - - await knex.schema.createTable('final_entities', table => { - table.comment( - 'This table contains the final entity result after processing and stitching', - ); - table - .text('entity_id') - .primary() - .notNullable() - .references('entity_id') - .inTable('refresh_state') - .onDelete('CASCADE') - .comment( - 'Entity ID which correspond to the ID in the refresh_state table', - ); - - table.text('finalized_entity').notNullable().comment('The final entity'); - }); - await knex.schema.createTable('refresh_state', table => { table.comment( 'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.', @@ -116,6 +70,24 @@ exports.up = async function up(knex) { table.index('next_update_at', 'refresh_state_next_update_at_idx'); }); + await knex.schema.createTable('final_entities', table => { + table.comment( + 'This table contains the final entity result after processing and stitching', + ); + table + .text('entity_id') + .primary() + .notNullable() + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment( + 'Entity ID which correspond to the ID in the refresh_state table', + ); + table.text('etag').notNullable().comment('Etag to be used for caching'); + table.text('finalized_entity').notNullable().comment('The final entity'); + }); + await knex.schema.createTable('refresh_state_references', table => { table.comment( 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', @@ -155,6 +127,34 @@ exports.up = async function up(knex) { 'refresh_state_references_target_entity_id_idx', ); }); + + await knex.schema.createTable('relations', table => { + table.comment('All relations between entities in the catalog'); + table + .text('originating_entity_id') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .text('source_entity_ref') + .notNullable() + .comment('The entity reference of the source entity of the relation'); + table + .text('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .text('target_entity_ref') + .notNullable() + .comment('The entity reference of the target entity of the relation'); + + table.index( + ['source_entity_ref', 'type', 'target_entity_ref'], + 'relations_full_ref_idx', + ); + }); }; /** diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 6011562078..64879df777 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -80,6 +80,7 @@ "files": [ "dist", "migrations/**/*.{js,d.ts}", + "migrationsv2/**/*.{js,d.ts}", "config.d.ts" ], "configSchema": "config.d.ts" diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 95c767407e..b7e860cce2 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -14,7 +14,12 @@ * limitations under the License. */ -import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; +import { + PluginDatabaseManager, + resolvePackagePath, + UrlReader, +} from '@backstage/backend-common'; +import fs from 'fs-extra'; import { DefaultNamespaceEntityPolicy, EntityPolicies, @@ -30,12 +35,10 @@ import { ScmIntegrations } from '@backstage/integration'; import lodash from 'lodash'; import { Logger } from 'winston'; import { - DatabaseEntitiesCatalog, DatabaseLocationsCatalog, EntitiesCatalog, LocationsCatalog, } from '../catalog'; -import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, @@ -68,7 +71,9 @@ import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; import { LocationStoreImpl } from '../next/LocationStoreImpl'; import { ProcessingStateManagerImpl } from '../next/ProcessingStateManagerImpl'; import { CatalogProcessingEngine } from '../next/types'; +import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { Stitcher } from './Stitcher'; +import { CommonDatabase } from '../database/CommonDatabase'; export type CatalogEnvironment = { logger: Logger; @@ -239,7 +244,20 @@ export class NextCatalogBuilder { const parser = this.parser || defaultEntityDataParser; const dbClient = await database.getClient(); - const db = await DatabaseManager.createDatabase(dbClient, { logger }); + const allMigrations = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrations', + ); + + const migrationsDir = resolvePackagePath( + '@backstage/plugin-catalog-backend', + 'migrationsv2', + ); + await fs.copy(allMigrations, migrationsDir); + await dbClient.migrate.latest({ + directory: migrationsDir, + }); + const db = new CommonDatabase(dbClient, logger); const processingDatabase = new ProcessingDatabaseImpl(dbClient, logger); const stateManager = new ProcessingStateManagerImpl(processingDatabase); @@ -251,7 +269,7 @@ export class NextCatalogBuilder { parser, policy, }); - const entitiesCatalog = new DatabaseEntitiesCatalog(db, logger); + const entitiesCatalog = new NextEntitiesCatalog(dbClient, logger); const locationStore = new LocationStoreImpl(db); const dbLocationProvider = new DatabaseLocationProvider(locationStore); diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts new file mode 100644 index 0000000000..39d3bf2a63 --- /dev/null +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Knex } from 'knex'; +import { DbFinalEntitiesRow } from './Stitcher'; +import { EntitiesCatalog } from '../catalog'; +import { EntitiesRequest, EntitiesResponse } from '../catalog/types'; + +export class NextEntitiesCatalog implements EntitiesCatalog { + constructor( + private readonly database: Knex, + private readonly logger: Logger, + ) {} + + async entities(request?: EntitiesRequest): Promise { + if (request?.fields) { + throw new Error('Fields extraction is not implemented'); + } + if (request?.pagination) { + throw new Error('Pagination is not implemented'); + } + if (request?.filter) { + throw new Error('Filters are not implemented'); + } + + const dbResponse = await this.database( + 'final_entities', + ).select(); + + const entities = dbResponse.map(e => JSON.parse(e.finalized_entity)); + + return { + entities, + pageInfo: { + hasNextPage: false, + }, + }; + } + + async removeEntityByUid(_uid: string): Promise { + throw new Error('Not implemented'); + } + + async batchAddOrUpdateEntities(): Promise { + throw new Error('Not implemented'); + } +} diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 8adc93bfad..29e7363b80 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -1,5 +1,3 @@ -import { Transaction } from '../database'; - /* * Copyright 2021 Spotify AB * @@ -18,7 +16,28 @@ import { Transaction } from '../database'; import { Knex } from 'knex'; import { Logger } from 'winston'; +import { Transaction } from '../database'; import { ConflictError } from '@backstage/errors'; +import { + DbRefreshStateReferences, + DbRefreshStateRow, + DbRelationsRow, +} from './database/ProcessingDatabaseImpl'; +import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { createHash } from 'crypto'; +import stableStringify from 'fast-json-stable-stringify'; + +export type DbFinalEntitiesRow = { + entity_id: string; + etag: string; + finalized_entity: string; +}; + +function generateEntityEtag(entity: Entity) { + return createHash('sha1') + .update(stableStringify({ ...entity })) + .digest('hex'); +} export class Stitcher { constructor( @@ -27,7 +46,74 @@ export class Stitcher { ) {} async stitch(entityRefs: Set) { - console.log(entityRefs); + for (const entityRef of entityRefs) { + await this.transaction(async txOpaque => { + const tx = txOpaque as Knex.Transaction; + const [result] = await tx('refresh_state') + .select('entity_id', 'processed_entity') + .where({ entity_ref: entityRef }); + + if (!result) { + this.logger.debug( + `Unable to stitch ${entityRef}, item does not exist in refresh state table`, + ); + return; + } else if (!result.processed_entity) { + this.logger.debug( + `Unable to stitch ${entityRef}, the entity has not yet been processed`, + ); + return; + } + + const entity: Entity = JSON.parse(result.processed_entity); + + const entityId = entity?.metadata?.uid; + if (!entityId) { + this.logger.error(`missing ID in entity ${JSON.stringify(entity)}`); + return; + } + + const [reference_count_result] = await tx( + 'refresh_state_references', + ) + .where({ target_entity_id: entity.metadata.uid }) + .count({ reference_count: 'target_entity_id' }); + console.log( + 'DEBUG: reference_count_result =', + reference_count_result, + entityId, + ); + if (Number(reference_count_result.reference_count) === 0) { + this.logger.debug(`${entityRef} is orphan`); + entity.metadata.annotations = { + ...entity.metadata.annotations, + ['backstage.io/orphan']: 'true', + }; + } + + const relationResults = await tx('relations') + .where({ originating_entity_id: entityId }) + .select(); + + // TODO: entityRef is lower case and should be uppercase in the final result. + entity.relations = relationResults.map(relation => ({ + type: relation.type, + target: parseEntityRef(relation.target_entity_ref), + })); + entity.metadata.generation = 1; + const etag = generateEntityEtag(entity); + entity.metadata.etag = etag; + console.log(JSON.stringify(entity, null, 2)); + await tx('final_entities') + .insert({ + finalized_entity: JSON.stringify(entity), + entity_id: entityId, + etag, + }) + .onConflict('entity_id') + .merge(['finalized_entity', 'etag']); + }); + } } private async transaction( @@ -45,6 +131,10 @@ export class Stitcher { { // If we explicitly trigger a rollback, don't fail. doNotRejectOnRollback: true, + isolationLevel: + this.database.client.config.client === 'sqlite3' + ? undefined // sqlite3 only supports serializable transactions, ignoring the isolation level param + : 'serializable', }, ); diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index f479c9a4f8..7c56fcd259 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -27,8 +27,6 @@ import { } from './types'; import type { Logger } from 'winston'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; -import { createHash } from 'crypto'; -import stableStringify from 'fast-json-stable-stringify'; import { v4 as uuid } from 'uuid'; export type DbRefreshStateRow = { @@ -55,12 +53,6 @@ export type DbRefreshStateReferences = { target_entity_id: string; }; -function generateEntityEtag(entity: Entity) { - return createHash('sha1') - .update(stableStringify({ ...entity })) - .digest('hex'); -} - // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause // errors in the underlying engine due to exceeding query limits, but large @@ -108,9 +100,6 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { // Update fragments - // Update relations - const entityRef = stringifyEntityRef(processedEntity); - // Delete old relations await tx('relations') .where({ originating_entity_id: id }) diff --git a/tsconfig.json b/tsconfig.json index d6cb6f4c96..6139c4a7f0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,8 @@ "packages/*/src", "plugins/*/src", "plugins/*/dev", - "plugins/*/migrations" + "plugins/*/migrations", + "plugins/catalog-backend/migrationsv2" ], "compilerOptions": { "outDir": "dist-types", From 33acc81f9079b7cfd996f3b8e78b4acdf96adb8c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 19 Apr 2021 11:06:15 +0200 Subject: [PATCH 14/23] Remove debug logs and unused variables Signed-off-by: Johan Haals --- .../src/next/CatalogProcessingOrchestratorImpl.ts | 2 +- plugins/catalog-backend/src/next/NextCatalogBuilder.ts | 2 +- plugins/catalog-backend/src/next/NextEntitiesCatalog.ts | 6 +----- plugins/catalog-backend/src/next/Stitcher.ts | 7 +------ 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index fee61cb366..0107c5cd8d 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -97,7 +97,7 @@ export class CatalogProcessingOrchestratorImpl async process( request: EntityProcessingRequest, ): Promise { - const { entity, eager, state } = request; + const { entity } = request; const result = await this.processSingleEntity(entity); diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index b7e860cce2..b196d63897 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -269,7 +269,7 @@ export class NextCatalogBuilder { parser, policy, }); - const entitiesCatalog = new NextEntitiesCatalog(dbClient, logger); + const entitiesCatalog = new NextEntitiesCatalog(dbClient); const locationStore = new LocationStoreImpl(db); const dbLocationProvider = new DatabaseLocationProvider(locationStore); diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index 39d3bf2a63..5bddd47f41 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -14,17 +14,13 @@ * limitations under the License. */ -import { Logger } from 'winston'; import { Knex } from 'knex'; import { DbFinalEntitiesRow } from './Stitcher'; import { EntitiesCatalog } from '../catalog'; import { EntitiesRequest, EntitiesResponse } from '../catalog/types'; export class NextEntitiesCatalog implements EntitiesCatalog { - constructor( - private readonly database: Knex, - private readonly logger: Logger, - ) {} + constructor(private readonly database: Knex) {} async entities(request?: EntitiesRequest): Promise { if (request?.fields) { diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index 29e7363b80..f79a247b91 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -78,11 +78,7 @@ export class Stitcher { ) .where({ target_entity_id: entity.metadata.uid }) .count({ reference_count: 'target_entity_id' }); - console.log( - 'DEBUG: reference_count_result =', - reference_count_result, - entityId, - ); + if (Number(reference_count_result.reference_count) === 0) { this.logger.debug(`${entityRef} is orphan`); entity.metadata.annotations = { @@ -103,7 +99,6 @@ export class Stitcher { entity.metadata.generation = 1; const etag = generateEntityEtag(entity); entity.metadata.etag = etag; - console.log(JSON.stringify(entity, null, 2)); await tx('final_entities') .insert({ finalized_entity: JSON.stringify(entity), From ab5ad7ecc3d13abf2552e2a7648a81aef314c0fe Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 19 Apr 2021 11:52:12 +0200 Subject: [PATCH 15/23] Create missing indexes, cleanup in down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- .../20210302150147_refresh_state.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 56c594ddf2..6abab6270b 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -86,6 +86,7 @@ exports.up = async function up(knex) { ); table.text('etag').notNullable().comment('Etag to be used for caching'); table.text('finalized_entity').notNullable().comment('The final entity'); + table.index('entity_id', 'final_entities_entity_id_idx'); }); await knex.schema.createTable('refresh_state_references', table => { @@ -149,11 +150,8 @@ exports.up = async function up(knex) { .text('target_entity_ref') .notNullable() .comment('The entity reference of the target entity of the relation'); - - table.index( - ['source_entity_ref', 'type', 'target_entity_ref'], - 'relations_full_ref_idx', - ); + table.index('source_entity_ref', 'relations_source_entity_ref_idx'); + table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); }; @@ -170,6 +168,15 @@ exports.down = async function down(knex) { table.dropIndex([], 'refresh_state_entity_ref_idx'); table.dropIndex([], 'refresh_state_next_update_at_idx'); }); - await knex.schema.dropTable('refresh_state'); + await knex.schema.alterTable('final_entities', table => { + table.dropIndex([], 'final_entities_entity_id_idx'); + }); + await knex.schema.alterTable('relations', table => { + table.index('source_entity_ref', 'relations_source_entity_ref_idx'); + table.index('originating_entity_id', 'relations_source_entity_id_idx'); + }); + await knex.schema.dropTable('final_entities'); + await knex.schema.dropTable('relations'); await knex.schema.dropTable('references'); + await knex.schema.dropTable('refresh_state'); }; From d126628a9b4dcd40d3923a25484b65b7db8e8ecb Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 19 Apr 2021 15:24:42 +0200 Subject: [PATCH 16/23] Remove optional entity field Signed-off-by: Johan Haals --- packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts | 1 - .../src/next/CatalogProcessingOrchestratorImpl.ts | 6 +++--- plugins/catalog-backend/src/next/util.ts | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index 20f99f15b3..fb452b6ac7 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -31,7 +31,6 @@ export interface LocationEntityV1alpha1 extends Entity { type?: string; target?: string; targets?: string[]; - optional?: boolean; }; } diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index 0107c5cd8d..a8e9b924d0 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -181,7 +181,7 @@ export class CatalogProcessingOrchestratorImpl // Backwards compatible processing of location entites if (isLocationEntity(entity)) { - const { type = location.type, optional = false } = entity.spec; + const { type = location.type } = entity.spec; const targets = new Array(); if (entity.spec.target) { targets.push(entity.spec.target); @@ -214,9 +214,9 @@ export class CatalogProcessingOrchestratorImpl { type, target, - presence: optional ? 'optional' : 'required', + presence: 'required', }, - Boolean(entity.spec?.optional), + false, emitter.emit, this.options.parser, ); diff --git a/plugins/catalog-backend/src/next/util.ts b/plugins/catalog-backend/src/next/util.ts index 6ff1cde2f0..f8e6f1f978 100644 --- a/plugins/catalog-backend/src/next/util.ts +++ b/plugins/catalog-backend/src/next/util.ts @@ -82,7 +82,6 @@ export function locationSpecToLocationEntity( spec: { type: location.type, target: location.target, - optional: location.presence === 'optional' ? true : undefined, }, }; From 572b078563e82d379dfd9b42da21cea5e2b6f792 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 19 Apr 2021 15:27:59 +0200 Subject: [PATCH 17/23] Find relations by source_entity_ref Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/Stitcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index f79a247b91..fe7a378338 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -88,7 +88,7 @@ export class Stitcher { } const relationResults = await tx('relations') - .where({ originating_entity_id: entityId }) + .where({ source_entity_ref: entityRef }) .select(); // TODO: entityRef is lower case and should be uppercase in the final result. From e7532b96ed30ea25cce34402b857c7ab87587140 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 20 Apr 2021 10:11:57 +0200 Subject: [PATCH 18/23] Fix typos Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/LocationStoreImpl.ts | 2 +- .../catalog-backend/src/next/ProcessingStateManagerImpl.ts | 2 +- .../src/next/database/ProcessingDatabaseImpl.ts | 6 +++--- plugins/catalog-backend/src/next/database/types.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/next/LocationStoreImpl.ts b/plugins/catalog-backend/src/next/LocationStoreImpl.ts index 1b9e9bd92c..f970234f09 100644 --- a/plugins/catalog-backend/src/next/LocationStoreImpl.ts +++ b/plugins/catalog-backend/src/next/LocationStoreImpl.ts @@ -78,7 +78,7 @@ export class LocationStoreImpl implements LocationStore { return this.db.transaction(async tx => { const location = await this.db.location(id); if (!location) { - throw new ConflictError(`No location found with with id: ${id}`); + throw new ConflictError(`No location found with id: ${id}`); } await this.db.removeLocation(tx, id); this.notifyDeletion(location); diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts index 6cd846d791..522a7a327c 100644 --- a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts +++ b/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts @@ -33,7 +33,7 @@ export class ProcessingStateManagerImpl implements ProcessingStateManager { errors: JSON.stringify(result.errors), state: result.state, relations: result.relations, - deferedEntities: result.deferredEntities, + deferredEntities: result.deferredEntities, }); }); } diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts index 7c56fcd259..7e9bf78691 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts @@ -76,7 +76,7 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { state, errors, relations, - deferedEntities, + deferredEntities, } = options; const refreshResult = await tx('refresh_state') @@ -93,7 +93,7 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { - entities: deferedEntities, + entities: deferredEntities, id, type: 'entity', }); @@ -183,7 +183,7 @@ export class ProcessingDatabaseImpl implements ProcessingDatabase { .select() .where('next_update_at', '<=', tx.fn.now()) .limit(request.processBatchSize) - .orderBy('next_update_at', 'desc'); + .orderBy('next_update_at', 'asc'); await tx('refresh_state') .whereIn( diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index d89a3b23bf..22e9e90c8e 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -32,7 +32,7 @@ export type UpdateProcessedEntityOptions = { state?: Map; errors?: string; relations: EntityRelationSpec[]; - deferedEntities: Entity[]; + deferredEntities: Entity[]; }; export type RefreshStateItem = { From 6c4aaa849508e2a1fdf05862737bb3e3d5769e2f Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 20 Apr 2021 10:56:47 +0200 Subject: [PATCH 19/23] use Default instead of Impl Signed-off-by: Johan Haals --- ...l.ts => DefaultCatalogProcessingEngine.ts} | 2 +- ...> DefaultCatalogProcessingOrchestrator.ts} | 2 +- ...onStoreImpl.ts => DefaultLocationStore.ts} | 2 +- ...pl.ts => DefaultProcessingStateManager.ts} | 2 +- ...cationServiceImpl.ts => DefaultService.ts} | 2 +- .../src/next/NextCatalogBuilder.ts | 20 +++++++++---------- plugins/catalog-backend/src/next/Stitcher.ts | 2 +- ...seImpl.ts => DefaultProcessingDatabase.ts} | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) rename plugins/catalog-backend/src/next/{CatalogProcessingEngineImpl.ts => DefaultCatalogProcessingEngine.ts} (97%) rename plugins/catalog-backend/src/next/{CatalogProcessingOrchestratorImpl.ts => DefaultCatalogProcessingOrchestrator.ts} (99%) rename plugins/catalog-backend/src/next/{LocationStoreImpl.ts => DefaultLocationStore.ts} (98%) rename plugins/catalog-backend/src/next/{ProcessingStateManagerImpl.ts => DefaultProcessingStateManager.ts} (96%) rename plugins/catalog-backend/src/next/{LocationServiceImpl.ts => DefaultService.ts} (97%) rename plugins/catalog-backend/src/next/database/{ProcessingDatabaseImpl.ts => DefaultProcessingDatabase.ts} (99%) diff --git a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts similarity index 97% rename from plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts rename to plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 41ad2bef8e..934ab3a1cc 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingEngineImpl.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -27,7 +27,7 @@ import { Logger } from 'winston'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { Stitcher } from './Stitcher'; -export class CatalogProcessingEngineImpl implements CatalogProcessingEngine { +export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { private subscriptions: Subscription[] = []; private running: boolean = false; diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts similarity index 99% rename from plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts rename to plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts index a8e9b924d0..f97d86a75a 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts @@ -82,7 +82,7 @@ function toAbsoluteUrl( } } -export class CatalogProcessingOrchestratorImpl +export class DefaultCatalogProcessingOrchestrator implements CatalogProcessingOrchestrator { constructor( private readonly options: { diff --git a/plugins/catalog-backend/src/next/LocationStoreImpl.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts similarity index 98% rename from plugins/catalog-backend/src/next/LocationStoreImpl.ts rename to plugins/catalog-backend/src/next/DefaultLocationStore.ts index f970234f09..b21fe9da02 100644 --- a/plugins/catalog-backend/src/next/LocationStoreImpl.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -26,7 +26,7 @@ export type LocationMessage = | { all: Location[] } | { added: Location[]; removed: Location[] }; -export class LocationStoreImpl implements LocationStore { +export class DefaultLocationStore implements LocationStore { private subscribers = new Set< ZenObservable.SubscriptionObserver >(); diff --git a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts similarity index 96% rename from plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts rename to plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts index 522a7a327c..91090bc139 100644 --- a/plugins/catalog-backend/src/next/ProcessingStateManagerImpl.ts +++ b/plugins/catalog-backend/src/next/DefaultProcessingStateManager.ts @@ -22,7 +22,7 @@ import { ProcessingStateManager, } from './types'; -export class ProcessingStateManagerImpl implements ProcessingStateManager { +export class DefaultProcessingStateManager implements ProcessingStateManager { constructor(private readonly db: ProcessingDatabase) {} async setProcessingItemResult(result: ProcessingItemResult) { diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/DefaultService.ts similarity index 97% rename from plugins/catalog-backend/src/next/LocationServiceImpl.ts rename to plugins/catalog-backend/src/next/DefaultService.ts index 4125f96ed1..3935f6e4d0 100644 --- a/plugins/catalog-backend/src/next/LocationServiceImpl.ts +++ b/plugins/catalog-backend/src/next/DefaultService.ts @@ -26,7 +26,7 @@ import { CatalogProcessingOrchestrator, } from './types'; -export class LocationServiceImpl implements LocationService { +export class DefaultLocationService implements LocationService { constructor( private readonly store: LocationStore, private readonly orchestrator: CatalogProcessingOrchestrator, diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index b196d63897..c3e0a6bd9c 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -64,12 +64,12 @@ import { } from '../ingestion/processors/PlaceholderProcessor'; import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; -import { CatalogProcessingEngineImpl } from '../next/CatalogProcessingEngineImpl'; -import { CatalogProcessingOrchestratorImpl } from '../next/CatalogProcessingOrchestratorImpl'; -import { ProcessingDatabaseImpl } from '../next/database/ProcessingDatabaseImpl'; +import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; +import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; +import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { DatabaseLocationProvider } from '../next/DatabaseLocationProvider'; -import { LocationStoreImpl } from '../next/LocationStoreImpl'; -import { ProcessingStateManagerImpl } from '../next/ProcessingStateManagerImpl'; +import { DefaultLocationStore } from './DefaultLocationStore'; +import { DefaultProcessingStateManager } from './DefaultProcessingStateManager'; import { CatalogProcessingEngine } from '../next/types'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { Stitcher } from './Stitcher'; @@ -259,10 +259,10 @@ export class NextCatalogBuilder { }); const db = new CommonDatabase(dbClient, logger); - const processingDatabase = new ProcessingDatabaseImpl(dbClient, logger); - const stateManager = new ProcessingStateManagerImpl(processingDatabase); + const processingDatabase = new DefaultProcessingDatabase(dbClient, logger); + const stateManager = new DefaultProcessingStateManager(processingDatabase); const integrations = ScmIntegrations.fromConfig(config); - const orchestrator = new CatalogProcessingOrchestratorImpl({ + const orchestrator = new DefaultCatalogProcessingOrchestrator({ processors, integrations, logger, @@ -271,10 +271,10 @@ export class NextCatalogBuilder { }); const entitiesCatalog = new NextEntitiesCatalog(dbClient); - const locationStore = new LocationStoreImpl(db); + const locationStore = new DefaultLocationStore(db); const dbLocationProvider = new DatabaseLocationProvider(locationStore); const stitcher = new Stitcher(dbClient, logger); - const processingEngine = new CatalogProcessingEngineImpl( + const processingEngine = new DefaultCatalogProcessingEngine( logger, [dbLocationProvider], // entityproviders stateManager, diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index fe7a378338..3e5e3ac24b 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -22,7 +22,7 @@ import { DbRefreshStateReferences, DbRefreshStateRow, DbRelationsRow, -} from './database/ProcessingDatabaseImpl'; +} from './database/DefaultProcessingDatabase'; import { Entity, parseEntityRef } from '@backstage/catalog-model'; import { createHash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; diff --git a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts similarity index 99% rename from plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts rename to plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 7e9bf78691..cf2d4e121c 100644 --- a/plugins/catalog-backend/src/next/database/ProcessingDatabaseImpl.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -59,7 +59,7 @@ export type DbRefreshStateReferences = { // enough to get the speed benefits. const BATCH_SIZE = 50; -export class ProcessingDatabaseImpl implements ProcessingDatabase { +export class DefaultProcessingDatabase implements ProcessingDatabase { constructor( private readonly database: Knex, private readonly logger: Logger, From cf101c539c0fabbc22e0c6605402d929c520e352 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 20 Apr 2021 11:15:25 +0200 Subject: [PATCH 20/23] tsconfig: skip adding catalog v2 migrations Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- tsconfig.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tsconfig.json b/tsconfig.json index 6139c4a7f0..d6cb6f4c96 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,8 +4,7 @@ "packages/*/src", "plugins/*/src", "plugins/*/dev", - "plugins/*/migrations", - "plugins/catalog-backend/migrationsv2" + "plugins/*/migrations" ], "compilerOptions": { "outDir": "dist-types", From b2adf03b584f8d690a0e1b0db5b558943d1c6cda Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 20 Apr 2021 11:15:47 +0200 Subject: [PATCH 21/23] catalog-backend: review feedback, reduce scope of `handled` Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .../src/next/DefaultCatalogProcessingOrchestrator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts index f97d86a75a..1880b5d17e 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingOrchestrator.ts @@ -210,7 +210,7 @@ export class DefaultCatalogProcessingOrchestrator for (const processor of this.options.processors) { if (processor.readLocation) { try { - handled = await processor.readLocation( + const read = await processor.readLocation( { type, target, @@ -220,7 +220,7 @@ export class DefaultCatalogProcessingOrchestrator emitter.emit, this.options.parser, ); - if (handled) { + if (read) { break; } } catch (e) { From bd6b2eefb5798d04c281c9f89922e33d326ffb0b Mon Sep 17 00:00:00 2001 From: David Tuite Date: Tue, 20 Apr 2021 11:10:33 +0100 Subject: [PATCH 22/23] Add Datadog plugin to marketplace Signed-off-by: David Tuite --- microsite/data/plugins/datadog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 microsite/data/plugins/datadog.yaml diff --git a/microsite/data/plugins/datadog.yaml b/microsite/data/plugins/datadog.yaml new file mode 100644 index 0000000000..7d47906a68 --- /dev/null +++ b/microsite/data/plugins/datadog.yaml @@ -0,0 +1,9 @@ +--- +title: Datadog +author: roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=datadog +category: Monitoring +description: Embed Datadog graphs and dashboards in Backstage. +documentation: https://roadie.io/backstage/plugins/datadog +iconUrl: https://roadie.io/images/logos/datadog-white-background.png +npmPackageName: '@roadiehq/backstage-plugin-datadog' From 9fba7470d793bb249e067ea89b328677750663b1 Mon Sep 17 00:00:00 2001 From: David Tuite Date: Tue, 20 Apr 2021 11:24:39 +0100 Subject: [PATCH 23/23] Add utm params to roadie marketplace links Signed-off-by: David Tuite --- microsite/data/plugins/argo-cd.yaml | 4 ++-- microsite/data/plugins/aws-lambda.yaml | 4 ++-- microsite/data/plugins/buildkite.yaml | 4 ++-- microsite/data/plugins/datadog.yaml | 2 +- microsite/data/plugins/firebase-functions.yaml | 4 ++-- microsite/data/plugins/github-insights.yaml | 4 ++-- microsite/data/plugins/github-pull-requests.yaml | 4 ++-- microsite/data/plugins/jira.yaml | 4 ++-- microsite/data/plugins/security-insights.yaml | 4 ++-- microsite/data/plugins/travis-ci.yaml | 4 ++-- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/microsite/data/plugins/argo-cd.yaml b/microsite/data/plugins/argo-cd.yaml index ca753f682a..121c833cb4 100644 --- a/microsite/data/plugins/argo-cd.yaml +++ b/microsite/data/plugins/argo-cd.yaml @@ -1,10 +1,10 @@ --- title: Argo CD author: roadie.io -authorUrl: https://roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=argo-cd category: CI/CD description: View Argo CD status for your projects in Backstage. -documentation: https://roadie.io/backstage/plugins/argo-cd +documentation: https://roadie.io/backstage/plugins/argo-cd/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=argo-cd iconUrl: https://roadie.io/images/logos/argo.png npmPackageName: '@roadiehq/backstage-plugin-argo-cd' tags: diff --git a/microsite/data/plugins/aws-lambda.yaml b/microsite/data/plugins/aws-lambda.yaml index cc3d59d188..f20cc0d393 100644 --- a/microsite/data/plugins/aws-lambda.yaml +++ b/microsite/data/plugins/aws-lambda.yaml @@ -1,9 +1,9 @@ --- title: AWS Lambda author: roadie.io -authorUrl: https://roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=aws-lambda category: Infrastructure description: View AWS Lambda functions for your components in Backstage. -documentation: https://roadie.io/backstage/plugins/aws-lambda +documentation: https://roadie.io/backstage/plugins/aws-lambda/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=aws-lambda iconUrl: https://roadie.io/images/logos/lambda.png npmPackageName: '@roadiehq/backstage-plugin-aws-lambda' diff --git a/microsite/data/plugins/buildkite.yaml b/microsite/data/plugins/buildkite.yaml index 16c6960898..4e9b1594d3 100644 --- a/microsite/data/plugins/buildkite.yaml +++ b/microsite/data/plugins/buildkite.yaml @@ -1,10 +1,10 @@ --- title: Buildkite author: roadie.io -authorUrl: https://roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=buildkite category: CI/CD description: View Buildkite CI builds for your service in Backstage. -documentation: https://roadie.io/backstage/plugins/buildkite +documentation: https://roadie.io/backstage/plugins/buildkite/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=buildkite iconUrl: https://roadie.io/images/logos/buildkite.png npmPackageName: '@roadiehq/backstage-plugin-buildkite' tags: diff --git a/microsite/data/plugins/datadog.yaml b/microsite/data/plugins/datadog.yaml index 7d47906a68..266627009c 100644 --- a/microsite/data/plugins/datadog.yaml +++ b/microsite/data/plugins/datadog.yaml @@ -4,6 +4,6 @@ author: roadie.io authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=datadog category: Monitoring description: Embed Datadog graphs and dashboards in Backstage. -documentation: https://roadie.io/backstage/plugins/datadog +documentation: https://roadie.io/backstage/plugins/datadog/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=datadog iconUrl: https://roadie.io/images/logos/datadog-white-background.png npmPackageName: '@roadiehq/backstage-plugin-datadog' diff --git a/microsite/data/plugins/firebase-functions.yaml b/microsite/data/plugins/firebase-functions.yaml index 9a1777754a..0b98948f88 100644 --- a/microsite/data/plugins/firebase-functions.yaml +++ b/microsite/data/plugins/firebase-functions.yaml @@ -1,9 +1,9 @@ --- title: Firebase Functions author: roadie.io -authorUrl: https://roadie.io/ +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=firebase-functions category: Infrastructure description: View Firebase Functions details for your service in Backstage. -documentation: https://roadie.io/backstage/plugins/firebase-functions +documentation: https://roadie.io/backstage/plugins/firebase-functions/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=firebase-functions iconUrl: https://roadie.io/images/logos/firebase.png npmPackageName: '@roadiehq/backstage-plugin-firebase-functions' diff --git a/microsite/data/plugins/github-insights.yaml b/microsite/data/plugins/github-insights.yaml index f127f88a3c..3515065916 100644 --- a/microsite/data/plugins/github-insights.yaml +++ b/microsite/data/plugins/github-insights.yaml @@ -1,9 +1,9 @@ --- title: GitHub Insights author: roadie.io -authorUrl: https://roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=github-insights category: Source Control Mgmt description: View GitHub Insights for your components in Backstage. -documentation: https://roadie.io/backstage/plugins/github-insights +documentation: https://roadie.io/backstage/plugins/github-insights/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=github-insights iconUrl: https://roadie.io/images/logos/insights.png npmPackageName: '@roadiehq/backstage-plugin-github-insights' diff --git a/microsite/data/plugins/github-pull-requests.yaml b/microsite/data/plugins/github-pull-requests.yaml index 4172cdff6b..8fc0dd5a6e 100644 --- a/microsite/data/plugins/github-pull-requests.yaml +++ b/microsite/data/plugins/github-pull-requests.yaml @@ -1,9 +1,9 @@ --- title: GitHub Pull Requests author: roadie.io -authorUrl: https://roadie.io/ +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=github-pull-requests category: Source Control Mgmt description: View GitHub pull requests for your service in Backstage. -documentation: https://roadie.io/backstage/plugins/github-pull-requests +documentation: https://roadie.io/backstage/plugins/github-pull-requests/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=github-pull-requests iconUrl: https://roadie.io/images/logos/github.png npmPackageName: '@roadiehq/backstage-plugin-github-pull-requests' diff --git a/microsite/data/plugins/jira.yaml b/microsite/data/plugins/jira.yaml index c32455c7d5..3342a2594c 100644 --- a/microsite/data/plugins/jira.yaml +++ b/microsite/data/plugins/jira.yaml @@ -1,9 +1,9 @@ --- title: Jira author: roadie.io -authorUrl: https://roadie.io +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=jira category: Agile Planning description: View Jira summary for your projects in Backstage. -documentation: https://roadie.io/backstage/plugins/jira +documentation: https://roadie.io/backstage/plugins/jira/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=jira iconUrl: https://roadie.io/images/logos/jira.png npmPackageName: '@roadiehq/backstage-plugin-jira' diff --git a/microsite/data/plugins/security-insights.yaml b/microsite/data/plugins/security-insights.yaml index 4f34f79027..1761bdf30a 100644 --- a/microsite/data/plugins/security-insights.yaml +++ b/microsite/data/plugins/security-insights.yaml @@ -1,9 +1,9 @@ --- title: Security Insights author: roadie.io -authorUrl: https://roadie.io/ +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=security-insights category: Security description: View Security Insights for your components in Backstage. -documentation: https://roadie.io/backstage/plugins/security-insights +documentation: https://roadie.io/backstage/plugins/security-insights/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=security-insights iconUrl: https://roadie.io/images/logos/github.png npmPackageName: '@roadiehq/backstage-plugin-security-insights' diff --git a/microsite/data/plugins/travis-ci.yaml b/microsite/data/plugins/travis-ci.yaml index a6b8c6ebd7..7cb108a5d8 100644 --- a/microsite/data/plugins/travis-ci.yaml +++ b/microsite/data/plugins/travis-ci.yaml @@ -1,9 +1,9 @@ --- title: Travis CI author: roadie.io -authorUrl: https://roadie.io/ +authorUrl: https://roadie.io/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=travis-ci category: CI/CD description: View Travis CI builds for your service in Backstage. -documentation: https://roadie.io/backstage/plugins/travis-ci +documentation: https://roadie.io/backstage/plugins/travis-ci/?utm_source=backstage.io&utm_medium=marketplace&utm_campaign=travis-ci iconUrl: https://roadie.io/images/logos/travis.png npmPackageName: '@roadiehq/backstage-plugin-travis-ci'