From def0eef285491deb3b575f121f24c10eba78a85e Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 20 Jun 2022 11:56:20 +0200 Subject: [PATCH 01/35] add a potential implementation for refreshKeys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 41 +++++++++++++++++++ plugins/catalog-backend/src/api/processor.ts | 10 ++++- .../src/database/DefaultProcessingDatabase.ts | 36 ++++++++++++++++ .../catalog-backend/src/database/tables.ts | 5 +++ plugins/catalog-backend/src/database/types.ts | 13 ++++++ .../src/modules/core/FileReaderProcessor.ts | 7 ++++ .../src/modules/core/PlaceholderProcessor.ts | 8 +++- .../src/modules/core/UrlReaderProcessor.ts | 7 ++++ .../DefaultCatalogProcessingEngine.ts | 6 +++ .../processing/ProcessorOutputCollector.ts | 7 ++++ .../catalog-backend/src/processing/types.ts | 2 +- 11 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 plugins/catalog-backend/migrations/20220616202842_refresh_keys.js diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js new file mode 100644 index 0000000000..dfa0bf5794 --- /dev/null +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -0,0 +1,41 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.createTable('refresh_keys', table => { + table.comment( + 'This table contains relations between entities and keys to trigger refreshes with', + ); + table + .text('entity_ref') + .notNullable() + .comment('A reference to the entity that the refresh key is tied to'); + table + .text('key') + .notNullable() + .comment( + 'A reference to a key which should be used to trigger a refresh on this entity', + ); + table.unique(['entity_ref', 'key']); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('refresh_keys'); +}; diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 3f274d13ea..d5a08edd44 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -169,9 +169,17 @@ export type CatalogProcessorErrorResult = { location: LocationSpec; }; +/** @public */ +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + entity: Entity; + key: String; +}; + /** @public */ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index dd28cf1290..e4a5ab3d85 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,12 +33,15 @@ import { UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, + RefreshKeyOptions, + RefreshByKeyOptions, } from './types'; import { DeferredEntity } from '../processing/types'; import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -515,7 +518,40 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); } } + async refreshByRefreshKey( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ) { + const tx = txOpaque as Knex.Transaction; + const { key } = options; + const rows = await tx('refresh_keys') + .where({ key }) + .select({ + entity_ref: 'refresh_keys.entity_ref', + }); + + await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); + } + async addRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { keys } = options; + + await Promise.all( + keys.map(k => { + return tx('refresh_keys') + .insert({ + entity_ref: stringifyEntityRef(k.entity), + key: k.key, + }) + .onConflict(['entity_ref', 'key']) + .ignore(); + }), + ); + } async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 40b4cb9a12..33c3ab0ef3 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -43,6 +43,11 @@ export type DbRefreshStateRow = { location_key?: string; }; +export type DbRefreshKeysRow = { + entity_ref: string; + key: string; +}; + export type DbRefreshStateReferencesRow = { source_key?: string; source_entity_ref?: string; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ce45a88938..bf3f765f4a 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -81,6 +81,14 @@ export type ReplaceUnprocessedEntitiesOptions = type: 'delta'; }; +export type RefreshKeyOptions = { + keys: { key: String; entity: Entity }[]; +}; + +export type RefreshByKeyOptions = { + key: string; +}; + export type RefreshOptions = { entityRef: string; }; @@ -149,6 +157,11 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + addRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise; + /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 66c6479fa7..768d763a24 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -61,6 +61,13 @@ export class FileReaderProcessor implements CatalogProcessor { }, })) { emit(parseResult); + if (parseResult.type === 'entity') { + emit({ + type: 'refresh', + key: path.normalize(fileMatch), + entity: parseResult.entity, + }); + } } } } else if (!optional) { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index ddd5f951db..5089ee2637 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,11 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from '../../api'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; @@ -66,6 +70,7 @@ export class PlaceholderProcessor implements CatalogProcessor { async preProcessEntity( entity: Entity, location: LocationSpec, + emit: CatalogProcessorEmit, ): Promise { const process = async (data: any): Promise<[any, boolean]> => { if (!data || !(data instanceof Object)) { @@ -102,6 +107,7 @@ export class PlaceholderProcessor implements CatalogProcessor { const resolverKey = keys[0].substr(1); const resolverValue = data[keys[0]]; + emit({ type: 'refresh', key: resolverValue, entity }); const resolver = this.options.resolvers[resolverKey]; if (!resolver || typeof resolverValue !== 'string') { // If there was no such placeholder resolver or if the value was not a diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 7f27bcd19d..a6c5bd74d8 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -83,6 +83,13 @@ export class UrlReaderProcessor implements CatalogProcessor { })) { parseResults.push(parseResult); emit(parseResult); + if (parseResult.type === 'entity') { + emit({ + type: 'refresh', + key: item.url, + entity: parseResult.entity, + }); + } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index b7ef5c9344..d3d679d2b7 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -123,6 +123,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { + await this.processingDatabase.transaction(tx => + this.processingDatabase.addRefreshKeys(tx, { + keys: result.refreshKeys, + }), + ); + const { entityRefs: parents } = await this.processingDatabase.transaction(tx => this.processingDatabase.listParents(tx, { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index db7a050dce..058d04ff2d 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -38,6 +38,10 @@ export class ProcessorOutputCollector { private readonly errors = new Array(); private readonly relations = new Array(); private readonly deferredEntities = new Array(); + private readonly refreshKeys = new Array<{ + key: String; + entity: Entity; + }>(); private done = false; constructor( @@ -54,6 +58,7 @@ export class ProcessorOutputCollector { return { errors: this.errors, relations: this.relations, + refreshKeys: this.refreshKeys, deferredEntities: this.deferredEntities, }; } @@ -116,6 +121,8 @@ export class ProcessorOutputCollector { this.relations.push(i.relation); } else if (i.type === 'error') { this.errors.push(i.error); + } else if (i.type === 'refresh') { + this.refreshKeys.push({ key: i.key, entity: i.entity }); } } } diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b178522c14..b93d57863f 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -26,7 +26,6 @@ export type EntityProcessingRequest = { entity: Entity; state?: JsonObject; // Versions for multiple deployments etc }; - /** * The result of processing an entity. * @public @@ -38,6 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; + refreshKeys: { key: String; entity: Entity }[]; errors: Error[]; } | { From 2aa5bb6c58126b5b2b75de7a64f8220150bc7373 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 00:23:39 +0200 Subject: [PATCH 02/35] add refresh function to processingResult Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 2fc2c9eac7..fdc8280ed0 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -65,4 +65,8 @@ export const processingResult = Object.freeze({ relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; }, + + refresh(entity, key) { + return { type: 'refresh', entity, key }; + }, } as const); From 6a0a4ecbfd17d7e4dda30b1f46f0339e2f1e95e0 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 00:25:32 +0200 Subject: [PATCH 03/35] Add refreshByRefreshKey to refresh service Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 2 +- .../src/service/AuthorizedRefreshService.ts | 9 ++++++++- .../src/service/DefaultRefreshService.ts | 11 ++++++++++- plugins/catalog-backend/src/service/types.ts | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index fdc8280ed0..3f05b30acf 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entity, key) { + refresh(entity: Entity, key: String) { return { type: 'refresh', entity, key }; }, } as const); diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 8634fbf86d..2e48b3d8b6 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -20,7 +20,11 @@ import { AuthorizeResult, PermissionEvaluator, } from '@backstage/plugin-permission-common'; -import { RefreshOptions, RefreshService } from './types'; +import { + RefreshByRefreshKeysOptions, + RefreshOptions, + RefreshService, +} from './types'; export class AuthorizedRefreshService implements RefreshService { constructor( @@ -45,4 +49,7 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } + async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + await this.service.refreshByRefreshKey(options); + } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index 3b982a0e46..d0c2742d97 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -15,7 +15,11 @@ */ import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; -import { RefreshOptions, RefreshService } from './types'; +import { + RefreshByRefreshKeysOptions, + RefreshOptions, + RefreshService, +} from './types'; export class DefaultRefreshService implements RefreshService { private database: DefaultProcessingDatabase; @@ -45,4 +49,9 @@ export class DefaultRefreshService implements RefreshService { }); }); } + async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + await this.database.transaction(async tx => { + await this.database.refreshByRefreshKey(tx, options); + }); + } } diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index f6cfa2e783..cb4debc41d 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -61,6 +61,9 @@ export type RefreshOptions = { authorizationToken?: string; }; +export type RefreshByRefreshKeysOptions = { + key: string; +}; /** * A service that manages refreshes of entities in the catalog. * @@ -71,6 +74,7 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; + refreshByRefreshKey(options: RefreshByRefreshKeysOptions): Promise; } /** From 3744b0c67a6cda52730cce7badd1df30a32c2d4d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 12:43:44 +0200 Subject: [PATCH 04/35] addRefreshKEys -> setRefreshKeys Signed-off-by: Kiss Miklos --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 2 +- .../src/processing/DefaultCatalogProcessingEngine.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index e4a5ab3d85..20ef39f6e8 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -533,7 +533,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); } - async addRefreshKeys( + async setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, ): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index bf3f765f4a..9d4043a340 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -157,7 +157,7 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; - addRefreshKeys( + setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, ): Promise; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index d3d679d2b7..5cb9c76e62 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -124,7 +124,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { await this.processingDatabase.transaction(tx => - this.processingDatabase.addRefreshKeys(tx, { + this.processingDatabase.setRefreshKeys(tx, { keys: result.refreshKeys, }), ); From 2b1ab47a7a4bacea634a09b6a9404dbf2eb79929 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 13:08:16 +0200 Subject: [PATCH 05/35] add deleteRefreshKey Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 20ef39f6e8..a003328244 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -518,6 +518,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); } } + async refreshByRefreshKey( txOpaque: Transaction, options: RefreshByKeyOptions, @@ -533,6 +534,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); } + async setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, @@ -552,6 +554,17 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { }), ); } + + async deleteRefreshKey( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { key } = options; + + await tx('refresh_keys').where({ key }).delete(); + } + async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; From b713a4c8dc9bdd33d4485405b0c75ca8b21c4811 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 18:53:38 +0200 Subject: [PATCH 06/35] use entityRef instead of Entity Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 4 ++-- plugins/catalog-backend/src/api/processor.ts | 2 +- .../src/database/DefaultProcessingDatabase.ts | 6 +++--- plugins/catalog-backend/src/database/types.ts | 2 +- .../src/modules/core/FileReaderProcessor.ts | 12 +++++++----- .../src/modules/core/PlaceholderProcessor.ts | 7 +++++-- .../src/modules/core/UrlReaderProcessor.ts | 13 +++++++------ .../DefaultCatalogProcessingEngine.test.ts | 5 +++++ .../processing/DefaultCatalogProcessingEngine.ts | 2 +- .../src/processing/ProcessorOutputCollector.ts | 4 ++-- plugins/catalog-backend/src/processing/types.ts | 2 +- 11 files changed, 35 insertions(+), 24 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 3f05b30acf..e1727a9f78 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entity: Entity, key: String) { - return { type: 'refresh', entity, key }; + refresh(entityRef: String, key: String): CatalogProcessorResult { + return { type: 'refresh', entityRef, key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index d5a08edd44..8480aec681 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,7 +172,7 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entity: Entity; + entityRef: String; key: String; }; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index a003328244..7cd590127c 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -540,13 +540,13 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: RefreshKeyOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { keys } = options; + const { refreshKeys } = options; await Promise.all( - keys.map(k => { + refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: stringifyEntityRef(k.entity), + entity_ref: k.entityRef, key: k.key, }) .onConflict(['entity_ref', 'key']) diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 9d4043a340..b4ab598f4a 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -82,7 +82,7 @@ export type ReplaceUnprocessedEntitiesOptions = }; export type RefreshKeyOptions = { - keys: { key: String; entity: Entity }[]; + refreshKeys: { key: String; entityRef: String }[]; }; export type RefreshByKeyOptions = { diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 768d763a24..20eabf2d03 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -25,6 +25,7 @@ import { LocationSpec, processingResult, } from '../../api'; +import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); @@ -62,11 +63,12 @@ export class FileReaderProcessor implements CatalogProcessor { })) { emit(parseResult); if (parseResult.type === 'entity') { - emit({ - type: 'refresh', - key: path.normalize(fileMatch), - entity: parseResult.entity, - }); + emit( + processingResult.refresh( + stringifyEntityRef(parseResult.entity), + path.normalize(fileMatch), + ), + ); } } } diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 5089ee2637..f1ad5c66f7 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; @@ -23,6 +23,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, + processingResult, } from '../../api'; /** @public */ @@ -107,7 +108,7 @@ export class PlaceholderProcessor implements CatalogProcessor { const resolverKey = keys[0].substr(1); const resolverValue = data[keys[0]]; - emit({ type: 'refresh', key: resolverValue, entity }); + const resolver = this.options.resolvers[resolverKey]; if (!resolver || typeof resolverValue !== 'string') { // If there was no such placeholder resolver or if the value was not a @@ -133,6 +134,8 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); + emit(processingResult.refresh(stringifyEntityRef(entity), resolverValue)); + return [ await resolver({ key: resolverKey, diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index a6c5bd74d8..ec8f7c11ec 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; @@ -84,11 +84,12 @@ export class UrlReaderProcessor implements CatalogProcessor { parseResults.push(parseResult); emit(parseResult); if (parseResult.type === 'entity') { - emit({ - type: 'refresh', - key: item.url, - entity: parseResult.entity, - }); + emit( + processingResult.refresh( + stringifyEntityRef(parseResult.entity), + item.url, + ), + ); } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index 10de36a1be..dd316583af 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -58,6 +58,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -123,6 +124,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -203,6 +205,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( @@ -413,6 +416,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }) .mockResolvedValueOnce({ ok: true, @@ -432,6 +436,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); await engine.start(); diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 5cb9c76e62..f1c7ba6943 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -125,7 +125,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { if (result.ok) { await this.processingDatabase.transaction(tx => this.processingDatabase.setRefreshKeys(tx, { - keys: result.refreshKeys, + refreshKeys: result.refreshKeys, }), ); diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 058d04ff2d..e4b8ca2e9b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -40,7 +40,7 @@ export class ProcessorOutputCollector { private readonly deferredEntities = new Array(); private readonly refreshKeys = new Array<{ key: String; - entity: Entity; + entityRef: String; }>(); private done = false; @@ -122,7 +122,7 @@ export class ProcessorOutputCollector { } else if (i.type === 'error') { this.errors.push(i.error); } else if (i.type === 'refresh') { - this.refreshKeys.push({ key: i.key, entity: i.entity }); + this.refreshKeys.push({ key: i.key, entityRef: i.entityRef }); } } } diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b93d57863f..78d8b875b2 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -37,7 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; - refreshKeys: { key: String; entity: Entity }[]; + refreshKeys: { key: String; entityRef: String }[]; errors: Error[]; } | { From af0b0314249053fb3a849d2b9460f1824728bfcd Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 27 Jun 2022 16:49:23 +0200 Subject: [PATCH 07/35] fix some tests Signed-off-by: Kiss Miklos --- .../DefaultProcessingDatabase.test.ts | 56 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 15 ++++- .../modules/core/FileReaderProcessor.test.ts | 22 ++++++-- .../modules/core/PlaceholderProcessor.test.ts | 13 ++++- .../modules/core/UrlReaderProcessor.test.ts | 2 +- .../DefaultCatalogProcessingEngine.test.ts | 1 + ...faultCatalogProcessingOrchestrator.test.ts | 2 + .../catalog-backend/src/processing/types.ts | 11 +++- .../src/service/DefaultRefreshService.test.ts | 1 + 10 files changed, 112 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 251388f0d2..703595e2ee 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,6 +24,7 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -67,6 +68,10 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; + const insertRefreshKeysRow = async (db: Knex, ref) => { + await db('refresh_keys').insert(ref); + }; + describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -1397,4 +1402,55 @@ describe('Default Processing Database', () => { }, ); }); + + describe('setRefreshKeys', () => { + it.each(databases.eachSupportedId())( + 'should set keys, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await db.transaction(async tx => + db.setRefreshKeys(tx, { + refreshKeys: [{ entityRef: 'location:default/root-1', key: 'foo' }], + }), + ); + + const rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(1); + expect(rows[0]).toEqual({ + entity_ref: 'location:default/root-1', + key: 'foo', + }); + }, + ); + }); + + describe('deleteRefreshKeys', () => { + it.each(databases.eachSupportedId())( + 'should delete keys, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await knex('refresh_keys').insert({ + entity_ref: 'location:default/root-1', + key: 'foo', + }); + + let rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(1); + + await db.transaction(async tx => + db.deleteRefreshKey(tx, { + key: 'foo', + }), + ); + + rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(0); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 7cd590127c..66b87d6b24 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -562,7 +562,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { key } = options; - await tx('refresh_keys').where({ key }).delete(); + await tx('refresh_keys').where({ key: key }).delete(); } async transaction(fn: (tx: Transaction) => Promise): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index b4ab598f4a..23755ad513 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; import { EntityRelationSpec } from '../api'; -import { DeferredEntity } from '../processing/types'; +import { DeferredEntity, RefreshKeyData } from '../processing/types'; import { DbRelationsRow } from './tables'; /** @@ -82,7 +82,7 @@ export type ReplaceUnprocessedEntitiesOptions = }; export type RefreshKeyOptions = { - refreshKeys: { key: String; entityRef: String }[]; + refreshKeys: RefreshKeyData[]; }; export type RefreshByKeyOptions = { @@ -157,6 +157,17 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + /** + * Schedules a refresh for all the entities that have the given refreshKey + */ + setRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise; + + /** + * Schedules a refresh for all the entities that have the given refreshKey + */ setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index a3eb5e4542..ad47350346 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -43,7 +43,10 @@ describe('FileReaderProcessor', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual(spec); - expect(generated.entity).toEqual({ kind: 'Component' }); + expect(generated.entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); }); it('should fail load from file with error', async () => { @@ -77,14 +80,23 @@ describe('FileReaderProcessor', () => { defaultEntityDataParser, ); - expect(emit).toBeCalledTimes(2); - expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); + expect(emit).toBeCalledTimes(4); + expect(emit.mock.calls[0][0].entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].entity).toEqual({ kind: 'API' }); - expect(emit.mock.calls[1][0].location).toEqual({ + expect(emit.mock.calls[1][0].entityRef).toEqual( + 'component:default/component-test', + ); + expect(emit.mock.calls[2][0].entity).toEqual({ + kind: 'API', + metadata: { name: 'api-test' }, + }); + expect(emit.mock.calls[2][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index adff97336e..3ec699bc04 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -51,7 +51,7 @@ describe('PlaceholderProcessor', () => { integrations, }); await expect( - processor.preProcessEntity(input, { type: 't', target: 'l' }), + processor.preProcessEntity(input, { type: 't', target: 'l' }, () => {}), ).resolves.toBe(input); }); @@ -76,6 +76,7 @@ describe('PlaceholderProcessor', () => { spec: { a: [{ b: { $upper: 'text' } }] }, }, { type: 'fake', target: 'http://example.com' }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -110,7 +111,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -131,7 +132,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -158,6 +159,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -194,6 +196,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -228,6 +231,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -266,6 +270,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -303,6 +308,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -343,6 +349,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).rejects.toThrow( /^Placeholder \$text could not form a URL out of \.\/a\/b\/catalog-info\.yaml and \.\.\/c\/catalog-info\.yaml, TypeError \[ERR_INVALID_URL\]/, diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index dbcb9e34df..ecb0eb30d3 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -74,7 +74,7 @@ describe('UrlReaderProcessor', () => { mockCache, ); - expect(emitted.length).toBe(1); + expect(emitted.length).toBe(2); expect(emitted[0]).toEqual({ type: 'entity', location: spec, diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index dd316583af..3b6f126391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -30,6 +30,7 @@ describe('DefaultCatalogProcessingEngine', () => { updateProcessedEntity: jest.fn(), updateEntityCache: jest.fn(), listParents: jest.fn(), + setRefreshKeys: jest.fn(), } as unknown as jest.Mocked; const orchestrator: jest.Mocked = { process: jest.fn(), diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 6e494d784e..b2d9846391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -102,6 +102,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ok: true, completedEntity: entity, deferredEntities: [], + refreshKeys: [], errors: [], relations: [], state: { @@ -119,6 +120,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ).resolves.toEqual({ ok: true, completedEntity: entity, + refreshKeys: [], deferredEntities: [ { locationKey: 'url:./new-place', diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 78d8b875b2..0bfbbf613b 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -37,7 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; - refreshKeys: { key: String; entityRef: String }[]; + refreshKeys: RefreshKeyData[]; errors: Error[]; } | { @@ -45,6 +45,15 @@ export type EntityProcessingResult = errors: Error[]; }; +/** + * A string to associate to the entity itself. + * @public + */ +export type RefreshKeyData = { + key: String; + entityRef: String; +}; + /** * Responsible for executing the individual processing steps in order to fully process an entity. * @public diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 84bfde1259..419aedc15d 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -138,6 +138,7 @@ describe('Refresh integration', () => { errors: [], deferredEntities, state: {}, + refreshKeys: [], }; }, }, From 1e53a82d52d161668b7ab2926dbc91242ca1b6e3 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 00:52:22 +0200 Subject: [PATCH 08/35] fix existing tests Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.test.ts | 4 ++-- .../core/__fixtures__/fileReaderProcessor/component.yaml | 2 ++ .../core/__fixtures__/fileReaderProcessor/dir/api.yaml | 2 ++ .../src/service/AuthorizedRefreshService.test.ts | 1 + .../src/service/DefaultLocationService.test.ts | 6 ++++++ plugins/catalog-backend/src/service/createRouter.test.ts | 5 +++-- 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 703595e2ee..129e92cc0e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -68,7 +68,7 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; - const insertRefreshKeysRow = async (db: Knex, ref) => { + const insertRefreshKeysRow = async (db: Knex, ref: DbRefreshKeysRow) => { await db('refresh_keys').insert(ref); }; @@ -1432,7 +1432,7 @@ describe('Default Processing Database', () => { async databaseId => { const { knex, db } = await createDatabase(databaseId); - await knex('refresh_keys').insert({ + await insertRefreshKeysRow(knex, { entity_ref: 'location:default/root-1', key: 'foo', }); diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml index 8524aaf14a..b5844bd6ed 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml @@ -1 +1,3 @@ kind: Component +metadata: + name: component-test diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml index a894e34f70..ad33548f13 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml @@ -1 +1,3 @@ kind: API +metadata: + name: api-test diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index 5d0a192470..17a3ef3b45 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,6 +22,7 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), + refreshByRefreshKey: jest.fn(), }; const permissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts index bd166cbd50..8151bbf18d 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts @@ -48,6 +48,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -75,6 +76,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -134,6 +136,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -161,6 +164,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -188,6 +192,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -211,6 +216,7 @@ describe('DefaultLocationServiceTest', () => { name: 'bar', }, }, + refreshKeys: [], deferredEntities: [], relations: [], errors: [], diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 8eafd8bfa3..d652893b55 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -409,6 +409,7 @@ describe('createRouter readonly disabled', () => { state: {}, completedEntity: entity, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -711,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, From d65305574690086a76998d95b7531467c2d4db85 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 00:59:15 +0200 Subject: [PATCH 09/35] fix type from String to string Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 2 +- plugins/catalog-backend/src/api/processor.ts | 4 ++-- .../src/processing/ProcessorOutputCollector.ts | 7 ++----- plugins/catalog-backend/src/processing/types.ts | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index e1727a9f78..4eefca196c 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entityRef: String, key: String): CatalogProcessorResult { + refresh(entityRef: string, key: string): CatalogProcessorResult { return { type: 'refresh', entityRef, key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 8480aec681..196391cf50 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,8 +172,8 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: String; - key: String; + entityRef: string; + key: string; }; /** @public */ diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index e4b8ca2e9b..56988569cf 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -24,7 +24,7 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity } from './types'; +import { DeferredEntity, RefreshKeyData } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, @@ -38,10 +38,7 @@ export class ProcessorOutputCollector { private readonly errors = new Array(); private readonly relations = new Array(); private readonly deferredEntities = new Array(); - private readonly refreshKeys = new Array<{ - key: String; - entityRef: String; - }>(); + private readonly refreshKeys = new Array(); private done = false; constructor( diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 0bfbbf613b..64b1dac16f 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -50,8 +50,8 @@ export type EntityProcessingResult = * @public */ export type RefreshKeyData = { - key: String; - entityRef: String; + key: string; + entityRef: string; }; /** From bb58626b1b5f546b8884b21f86614859c2bf162d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 01:12:05 +0200 Subject: [PATCH 10/35] export CatalogProcessorRefreshKeysResult Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts index 1cb4b71aa3..136c23d20d 100644 --- a/plugins/catalog-backend/src/api/index.ts +++ b/plugins/catalog-backend/src/api/index.ts @@ -26,6 +26,7 @@ export type { CatalogProcessorRelationResult, CatalogProcessorErrorResult, CatalogProcessorResult, + CatalogProcessorRefreshKeysResult, } from './processor'; export type { EntityProvider, From 3117584484a1ed785ae4a7e16cdc6a6afd796c67 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 01:15:49 +0200 Subject: [PATCH 11/35] generate new api-report.md Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index cb8a5abb8a..fbbf2186ca 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -268,6 +268,13 @@ export type CatalogProcessorParser = (options: { location: LocationSpec; }) => AsyncIterable; +// @public (undocumented) +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + entityRef: string; + key: string; +}; + // @public (undocumented) export type CatalogProcessorRelationResult = { type: 'relation'; @@ -279,7 +286,8 @@ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; // @public (undocumented) export class CodeOwnersProcessor implements CatalogProcessor { @@ -545,7 +553,11 @@ export class PlaceholderProcessor implements CatalogProcessor { // (undocumented) getProcessorName(): string; // (undocumented) - preProcessEntity(entity: Entity, location: LocationSpec): Promise; + preProcessEntity( + entity: Entity, + location: LocationSpec, + emit: CatalogProcessorEmit, + ): Promise; } // @public (undocumented) @@ -601,6 +613,7 @@ export const processingResult: Readonly<{ newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; + readonly refresh: (entityRef: string, key: string) => CatalogProcessorResult; }>; // @public (undocumented) From 3125f54ad63f18bd374a0cc47cf304a14fa8fb5f Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 10:14:07 +0200 Subject: [PATCH 12/35] fix UrlReaderProcessor test Signed-off-by: Kiss Miklos --- .../modules/core/UrlReaderProcessor.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index ecb0eb30d3..631f463fd5 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -61,7 +61,13 @@ describe('UrlReaderProcessor', () => { server.use( rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => - res(ctx.set({ ETag: 'my-etag' }), ctx.json({ mock: 'entity' })), + res( + ctx.set({ ETag: 'my-etag' }), + ctx.json({ + kind: 'component', + metadata: { name: 'mock-url-entity' }, + }), + ), ), ); @@ -78,11 +84,17 @@ describe('UrlReaderProcessor', () => { expect(emitted[0]).toEqual({ type: 'entity', location: spec, - entity: { mock: 'entity' }, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, }); expect(mockCache.set).toBeCalledWith('v1', { etag: 'my-etag', - value: [{ type: 'entity', location: spec, entity: { mock: 'entity' } }], + value: [ + { + type: 'entity', + location: spec, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, + }, + ], }); expect(mockCache.set).toBeCalledTimes(1); }); From 137b029a2d3e3954b71cb9187f9949b093a2e613 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 16:12:09 +0200 Subject: [PATCH 13/35] make refreshByRefreshKeys accept an array of keys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 3 ++ .../src/database/DefaultProcessingDatabase.ts | 34 +++++++++---------- plugins/catalog-backend/src/database/types.ts | 10 +----- .../src/service/DefaultRefreshService.ts | 2 +- plugins/catalog-backend/src/service/types.ts | 2 +- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index dfa0bf5794..8358cbb0cd 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -14,6 +14,9 @@ * limitations under the License. */ +/** + * @param { import("knex").Knex } knex + */ exports.up = async function up(knex) { await knex.schema.createTable('refresh_keys', table => { table.comment( diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 66b87d6b24..3b93e9a222 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -519,20 +519,28 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } - async refreshByRefreshKey( + async refreshByRefreshKeys( txOpaque: Transaction, options: RefreshByKeyOptions, ) { const tx = txOpaque as Knex.Transaction; - const { key } = options; + const { keys } = options; - const rows = await tx('refresh_keys') - .where({ key }) - .select({ - entity_ref: 'refresh_keys.entity_ref', - }); + const query = await tx('refresh_state') + .whereIn('entity_ref', function (tx2) { + tx2 + .whereIn('key', keys) + .select({ + entity_ref: 'refresh_keys.entity_ref', + }) + .from('refresh_keys') + .columns('entity_ref'); + }) + .update({ next_update_at: tx.fn.now() }) + .toSQL() + .toNative(); - await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); + console.log(query, '@@@@@@@!!!!!!!@@@@@'); } async setRefreshKeys( @@ -555,16 +563,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } - async deleteRefreshKey( - txOpaque: Transaction, - options: RefreshByKeyOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - const { key } = options; - - await tx('refresh_keys').where({ key: key }).delete(); - } - async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 23755ad513..4d2b64a721 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -86,7 +86,7 @@ export type RefreshKeyOptions = { }; export type RefreshByKeyOptions = { - key: string; + keys: string[]; }; export type RefreshOptions = { @@ -165,14 +165,6 @@ export interface ProcessingDatabase { options: RefreshKeyOptions, ): Promise; - /** - * Schedules a refresh for all the entities that have the given refreshKey - */ - setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise; - /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index d0c2742d97..e5deab1790 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -51,7 +51,7 @@ export class DefaultRefreshService implements RefreshService { } async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { await this.database.transaction(async tx => { - await this.database.refreshByRefreshKey(tx, options); + await this.database.refreshByRefreshKeys(tx, options); }); } } diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index cb4debc41d..1e2142fc3a 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -62,7 +62,7 @@ export type RefreshOptions = { }; export type RefreshByRefreshKeysOptions = { - key: string; + keys: string[]; }; /** * A service that manages refreshes of entities in the catalog. From 8f84695e0ffb9bf2bde1c959fd0dc440e0fa2d52 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 18:45:03 +0200 Subject: [PATCH 14/35] set the refreshKeys in the updateProcessedEntity function Signed-off-by: Kiss Miklos --- .../src/api/processingResult.ts | 4 +- plugins/catalog-backend/src/api/processor.ts | 1 - .../src/database/DefaultProcessingDatabase.ts | 51 +++++++++---------- plugins/catalog-backend/src/database/types.ts | 9 +--- .../src/modules/core/FileReaderProcessor.ts | 19 ++++--- .../src/modules/core/PlaceholderProcessor.ts | 2 +- .../src/modules/core/UrlReaderProcessor.ts | 11 ++-- .../DefaultCatalogProcessingEngine.ts | 7 +-- .../catalog-backend/src/processing/types.ts | 1 - 9 files changed, 40 insertions(+), 65 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 4eefca196c..84f1f4b70d 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entityRef: string, key: string): CatalogProcessorResult { - return { type: 'refresh', entityRef, key }; + refresh(key: string): CatalogProcessorResult { + return { type: 'refresh', key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 196391cf50..44e8b298b0 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,7 +172,6 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: string; key: string; }; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 3b93e9a222..b8d7493360 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -103,11 +103,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { `Conflicting write of processing result for ${id} with location key '${locationKey}'`, ); } + const sourceEntityRef = stringifyEntityRef(processedEntity); // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { entities: deferredEntities, - sourceEntityRef: stringifyEntityRef(processedEntity), + sourceEntityRef, }); // Delete old relations @@ -141,6 +142,19 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); + // Insert the refresh keys for the procssed entity + await Promise.all( + options.refreshKeys.map(k => { + return tx('refresh_keys') + .insert({ + entity_ref: sourceEntityRef, + key: k.key, + }) + .onConflict(['entity_ref', 'key']) + .ignore(); + }), + ); + return { previous: { relations: previousRelationRows, @@ -526,41 +540,22 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { keys } = options; - const query = await tx('refresh_state') + const updateResult = await tx('refresh_state') .whereIn('entity_ref', function (tx2) { tx2 .whereIn('key', keys) .select({ entity_ref: 'refresh_keys.entity_ref', }) - .from('refresh_keys') - .columns('entity_ref'); + .from('refresh_keys'); }) - .update({ next_update_at: tx.fn.now() }) - .toSQL() - .toNative(); + .update({ next_update_at: tx.fn.now() }); - console.log(query, '@@@@@@@!!!!!!!@@@@@'); - } - - async setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - const { refreshKeys } = options; - - await Promise.all( - refreshKeys.map(k => { - return tx('refresh_keys') - .insert({ - entity_ref: k.entityRef, - key: k.key, - }) - .onConflict(['entity_ref', 'key']) - .ignore(); - }), - ); + if (updateResult === 0) { + throw new NotFoundError( + `Failed to schedule ${JSON.stringify(keys)} for keys`, + ); + } } async transaction(fn: (tx: Transaction) => Promise): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 4d2b64a721..da7aa65714 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -38,6 +38,7 @@ export type UpdateProcessedEntityOptions = { relations: EntityRelationSpec[]; deferredEntities: DeferredEntity[]; locationKey?: string; + refreshKeys: RefreshKeyData[]; }; export type UpdateEntityCacheOptions = { @@ -157,14 +158,6 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; - /** - * Schedules a refresh for all the entities that have the given refreshKey - */ - setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise; - /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 20eabf2d03..1763a17acb 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -29,6 +29,8 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); +const LOCATION_TYPE = 'file'; + /** @public */ export class FileReaderProcessor implements CatalogProcessor { getProcessorName(): string { @@ -41,7 +43,7 @@ export class FileReaderProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'file') { + if (location.type !== LOCATION_TYPE) { return false; } @@ -57,19 +59,16 @@ export class FileReaderProcessor implements CatalogProcessor { for await (const parseResult of parser({ data: data, location: { - type: 'file', + type: LOCATION_TYPE, target: path.normalize(fileMatch), }, })) { emit(parseResult); - if (parseResult.type === 'entity') { - emit( - processingResult.refresh( - stringifyEntityRef(parseResult.entity), - path.normalize(fileMatch), - ), - ); - } + emit( + processingResult.refresh( + `${LOCATION_TYPE}:${path.normalize(fileMatch)}`, + ), + ); } } } else if (!optional) { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index f1ad5c66f7..5b6ed31f4d 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -134,7 +134,7 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit(processingResult.refresh(stringifyEntityRef(entity), resolverValue)); + emit(processingResult.refresh(`url:${resolverValue}`)); return [ await resolver({ diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index ec8f7c11ec..61bb8fed99 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -30,6 +30,7 @@ import { LocationSpec, processingResult, } from '../../api'; +import { locationSpecToLocationEntity } from '../../util'; const CACHE_KEY = 'v1'; @@ -83,14 +84,6 @@ export class UrlReaderProcessor implements CatalogProcessor { })) { parseResults.push(parseResult); emit(parseResult); - if (parseResult.type === 'entity') { - emit( - processingResult.refresh( - stringifyEntityRef(parseResult.entity), - item.url, - ), - ); - } } } @@ -101,6 +94,8 @@ export class UrlReaderProcessor implements CatalogProcessor { value: parseResults as CatalogProcessorEntityResult[], }); } + + emit(processingResult.refresh(`${location.type}:${location.target}`)); } catch (error) { assertError(error); const message = `Unable to read ${location.type}, ${error}`; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index f1c7ba6943..10d6fc6975 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -123,12 +123,6 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { - await this.processingDatabase.transaction(tx => - this.processingDatabase.setRefreshKeys(tx, { - refreshKeys: result.refreshKeys, - }), - ); - const { entityRefs: parents } = await this.processingDatabase.transaction(tx => this.processingDatabase.listParents(tx, { @@ -186,6 +180,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { relations: result.relations, deferredEntities: result.deferredEntities, locationKey, + refreshKeys: result.refreshKeys, }); oldRelationSources = new Set( previous.relations.map(r => r.source_entity_ref), diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 64b1dac16f..e94125b6f5 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -51,7 +51,6 @@ export type EntityProcessingResult = */ export type RefreshKeyData = { key: string; - entityRef: string; }; /** From 8de6b8912123cb92fe4f772e31b5d59565e498e3 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 18:56:14 +0200 Subject: [PATCH 15/35] use inices instead of table.unique Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index 8358cbb0cd..4e15a948cc 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -24,6 +24,9 @@ exports.up = async function up(knex) { ); table .text('entity_ref') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') .notNullable() .comment('A reference to the entity that the refresh key is tied to'); table @@ -32,7 +35,8 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); - table.unique(['entity_ref', 'key']); + table.index('entity_ref', 'refresh_keys_entity_ref_idx'); + table.index('key', 'refresh_keys_key_idx'); }); }; @@ -40,5 +44,10 @@ exports.up = async function up(knex) { * @param { import("knex").Knex } knex */ exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_keys', table => { + table.dropIndex([], 'refresh_keys_entity_ref_idx'); + table.dropIndex([], 'refresh_keys_key_idx'); + }); + await knex.schema.dropTable('refresh_keys'); }; From e31cc714c36ef6bf88179c5a6d9fda50a13f38f9 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 19:00:10 +0200 Subject: [PATCH 16/35] move notNullable upper Signed-off-by: Kiss Miklos --- .../catalog-backend/migrations/20220616202842_refresh_keys.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index 4e15a948cc..a4e02545c7 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -24,10 +24,10 @@ exports.up = async function up(knex) { ); table .text('entity_ref') + .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') - .notNullable() .comment('A reference to the entity that the refresh key is tied to'); table .text('key') From 2355a43259182585a5290eddf2c2322e99864e6b Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 10:37:30 +0200 Subject: [PATCH 17/35] name function Signed-off-by: Kiss Miklos --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index b8d7493360..de8d18aad6 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -541,7 +541,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { keys } = options; const updateResult = await tx('refresh_state') - .whereIn('entity_ref', function (tx2) { + .whereIn('entity_ref', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) .select({ From 2eb8c3d71bac3922686897d4b29c27cb0066e20f Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 12:17:53 +0200 Subject: [PATCH 18/35] fix tsc issues Signed-off-by: Kiss Miklos --- .../DefaultProcessingDatabase.test.ts | 64 +++---------------- .../src/database/DefaultProcessingDatabase.ts | 1 - .../src/modules/core/FileReaderProcessor.ts | 1 - .../src/modules/core/PlaceholderProcessor.ts | 2 +- .../src/modules/core/UrlReaderProcessor.ts | 3 +- .../processing/ProcessorOutputCollector.ts | 2 +- 6 files changed, 11 insertions(+), 62 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 129e92cc0e..1e96c13dc5 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,7 +24,6 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { - DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -68,10 +67,6 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; - const insertRefreshKeysRow = async (db: Knex, ref: DbRefreshKeysRow) => { - await db('refresh_keys').insert(ref); - }; - describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -103,6 +98,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities: [], + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'undefined'`, @@ -122,6 +118,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }; const { knex, db } = await createDatabase(databaseId); @@ -148,6 +145,7 @@ describe('Default Processing Database', () => { ...options, resultHash: '', locationKey: 'fail', + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'fail'`, @@ -179,6 +177,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }), ); @@ -233,6 +232,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -255,6 +255,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -314,6 +315,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities, + refreshKeys: [], }), ); @@ -405,6 +407,7 @@ describe('Default Processing Database', () => { processedEntity, resultHash: '', relations: [], + refreshKeys: [], deferredEntities: [ { entity: { @@ -1402,55 +1405,4 @@ describe('Default Processing Database', () => { }, ); }); - - describe('setRefreshKeys', () => { - it.each(databases.eachSupportedId())( - 'should set keys, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await db.transaction(async tx => - db.setRefreshKeys(tx, { - refreshKeys: [{ entityRef: 'location:default/root-1', key: 'foo' }], - }), - ); - - const rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(1); - expect(rows[0]).toEqual({ - entity_ref: 'location:default/root-1', - key: 'foo', - }); - }, - ); - }); - - describe('deleteRefreshKeys', () => { - it.each(databases.eachSupportedId())( - 'should delete keys, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await insertRefreshKeysRow(knex, { - entity_ref: 'location:default/root-1', - key: 'foo', - }); - - let rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(1); - - await db.transaction(async tx => - db.deleteRefreshKey(tx, { - key: 'foo', - }), - ); - - rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(0); - }, - ); - }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index de8d18aad6..11b723a117 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,7 +33,6 @@ import { UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, - RefreshKeyOptions, RefreshByKeyOptions, } from './types'; import { DeferredEntity } from '../processing/types'; diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 1763a17acb..59795589b2 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -25,7 +25,6 @@ import { LocationSpec, processingResult, } from '../../api'; -import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 5b6ed31f4d..8a1891fe0c 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 61bb8fed99..7b62690341 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; @@ -30,7 +30,6 @@ import { LocationSpec, processingResult, } from '../../api'; -import { locationSpecToLocationEntity } from '../../util'; const CACHE_KEY = 'v1'; diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 56988569cf..033dfd4ae3 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -119,7 +119,7 @@ export class ProcessorOutputCollector { } else if (i.type === 'error') { this.errors.push(i.error); } else if (i.type === 'refresh') { - this.refreshKeys.push({ key: i.key, entityRef: i.entityRef }); + this.refreshKeys.push({ key: i.key }); } } } From b79a82028a064e6e331edc0855cfac7ffb9c5b53 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 14:27:49 +0200 Subject: [PATCH 19/35] update api-report.md Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index fbbf2186ca..c410f166cc 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -271,7 +271,6 @@ export type CatalogProcessorParser = (options: { // @public (undocumented) export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: string; key: string; }; @@ -613,7 +612,7 @@ export const processingResult: Readonly<{ newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; - readonly refresh: (entityRef: string, key: string) => CatalogProcessorResult; + readonly refresh: (key: string) => CatalogProcessorResult; }>; // @public (undocumented) From 197ed5b83c91bcfb72cce5f6099d4737621324e4 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 17:21:47 +0200 Subject: [PATCH 20/35] add and fix tests Signed-off-by: Kiss Miklos --- .../modules/core/FileReaderProcessor.test.ts | 5 +-- .../modules/core/PlaceholderProcessor.test.ts | 31 +++++++++++++++++++ .../modules/core/UrlReaderProcessor.test.ts | 4 +++ .../src/service/AuthorizedRefreshService.ts | 4 +-- .../src/service/DefaultRefreshService.ts | 2 +- .../src/service/createRouter.test.ts | 4 +-- plugins/catalog-backend/src/service/types.ts | 2 +- 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index ad47350346..0c114f49a1 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,8 +89,9 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].entityRef).toEqual( - 'component:default/component-test', + expect(emit.mock.calls[1][0].key).toContain('file:'); + expect(emit.mock.calls[1][0].key).toContain( + 'fileReaderProcessor/component.yaml', ); expect(emit.mock.calls[2][0].entity).toEqual({ kind: 'API', diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index 3ec699bc04..aa448dd955 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -17,6 +17,7 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; +import { CatalogProcessorResult } from '../../api'; import { jsonPlaceholderResolver, PlaceholderProcessor, @@ -357,6 +358,36 @@ describe('PlaceholderProcessor', () => { expect(read).not.toBeCalled(); }); + it('should emit the resolverValue as a refreshKey', async () => { + read.mockResolvedValue( + Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + ); + + const processor = new PlaceholderProcessor({ + resolvers: { + json: jsonPlaceholderResolver, + }, + reader, + integrations, + }); + + const emitted = new Array(); + await processor.preProcessEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { a: [{ b: { $json: './path-to-file.json' } }] }, + }, + { type: 'fake', target: 'http://example.com' }, + result => emitted.push(result), + ); + console.log(emitted); + expect(emitted[0]).toEqual({ + type: 'refresh', + key: 'url:./path-to-file.json', + }); + }); }); describe('yamlPlaceholderResolver', () => { diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index 631f463fd5..fb3f48c939 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -86,6 +86,10 @@ describe('UrlReaderProcessor', () => { location: spec, entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, }); + expect(emitted[1]).toEqual({ + type: 'refresh', + key: 'url:http://localhost/component.yaml', + }); expect(mockCache.set).toBeCalledWith('v1', { etag: 'my-etag', value: [ diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 2e48b3d8b6..19f6270da6 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -49,7 +49,7 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } - async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { - await this.service.refreshByRefreshKey(options); + async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { + await this.service.refreshByRefreshKeys(options); } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index e5deab1790..deb69bd041 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -49,7 +49,7 @@ export class DefaultRefreshService implements RefreshService { }); }); } - async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { await this.database.transaction(async tx => { await this.database.refreshByRefreshKeys(tx, options); }); diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index d652893b55..b7eeb6f1bc 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -712,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index 1e2142fc3a..a073e5bb6c 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -74,7 +74,7 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; - refreshByRefreshKey(options: RefreshByRefreshKeysOptions): Promise; + refreshByRefreshKeys(options: RefreshByRefreshKeysOptions): Promise; } /** From 859c547303080451ce34c576a5c0d3a227e7298b Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 17:09:53 +0200 Subject: [PATCH 21/35] add test for storing refreshKeys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 3 +- .../DefaultProcessingDatabase.test.ts | 58 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 16 ++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index a4e02545c7..ded53e4f67 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -25,7 +25,7 @@ exports.up = async function up(knex) { table .text('entity_ref') .notNullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); @@ -35,6 +35,7 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); + table.unique(['entity_ref', 'key']); table.index('entity_ref', 'refresh_keys_entity_ref_idx'); table.index('key', 'refresh_keys_key_idx'); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 1e96c13dc5..f9f51b7a0d 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,6 +24,7 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -473,6 +474,63 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'stores the refresh keys for the entity', + async databaseId => { + const mockLogger = { + debug: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + }; + const { knex, db } = await createDatabase( + databaseId, + mockLogger as unknown as Logger, + ); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const deferredEntities = [ + { + entity: { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'next', + }, + }, + locationKey: 'mock', + }, + ]; + + await db.transaction(tx => + db.updateProcessedEntity(tx, { + id, + processedEntity, + resultHash: '', + relations: [], + deferredEntities, + refreshKeys: [{ key: 'protocol:foo-bar.com' }], + }), + ); + + const refreshKeys = await knex('refresh_keys') + .where({ entity_ref: stringifyEntityRef(processedEntity) }) + .select(); + + expect(refreshKeys[0]).toEqual({ + entity_ref: 'location:default/fakelocation', + key: 'protocol:foo-bar.com', + }); + }, + ); }); describe('updateEntityCache', () => { diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 11b723a117..1f984c1bab 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -141,12 +141,24 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); - // Insert the refresh keys for the procssed entity + // Find the top-level location entity that manages the processedEntity + let entityRefToRefresh = sourceEntityRef; + const { entityRefs } = await this.listAncestors(tx, { + entityRef: sourceEntityRef, + }); + const locationAncestor = entityRefs.find(ref => + ref.startsWith('location:'), + ); + if (locationAncestor) { + entityRefToRefresh = locationAncestor; + } + + // Insert the refresh keys for the processed entity await Promise.all( options.refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: sourceEntityRef, + entity_ref: entityRefToRefresh, key: k.key, }) .onConflict(['entity_ref', 'key']) From aa1e84e78a9b2599dede960a405e282174148e82 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 18:23:28 +0200 Subject: [PATCH 22/35] fix types Signed-off-by: Kiss Miklos --- .../src/service/AuthorizedRefreshService.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index 17a3ef3b45..f22b5b86f4 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,7 +22,7 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), - refreshByRefreshKey: jest.fn(), + refreshByRefreshKeys: jest.fn(), }; const permissionApi = { authorize: jest.fn(), From 61aca33a258b4d37f427666bb9df73f51a5b084a Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 18:35:02 +0200 Subject: [PATCH 23/35] use onConflict Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 1f984c1bab..5b191dce32 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -141,24 +141,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); - // Find the top-level location entity that manages the processedEntity - let entityRefToRefresh = sourceEntityRef; - const { entityRefs } = await this.listAncestors(tx, { - entityRef: sourceEntityRef, - }); - const locationAncestor = entityRefs.find(ref => - ref.startsWith('location:'), - ); - if (locationAncestor) { - entityRefToRefresh = locationAncestor; - } - // Insert the refresh keys for the processed entity await Promise.all( options.refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: entityRefToRefresh, + entity_ref: sourceEntityRef, key: k.key, }) .onConflict(['entity_ref', 'key']) From e75bde1342122eb7885ea91e188d00b8121ffe30 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 14:39:10 +0200 Subject: [PATCH 24/35] use entity_id instead of entity_ref Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 9 +++--- .../DefaultProcessingDatabase.test.ts | 2 +- .../src/database/DefaultProcessingDatabase.ts | 29 ++++++++++--------- .../catalog-backend/src/database/tables.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 4 --- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index ded53e4f67..b1b67f68f2 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -23,9 +23,9 @@ exports.up = async function up(knex) { 'This table contains relations between entities and keys to trigger refreshes with', ); table - .text('entity_ref') + .text('entity_id') .notNullable() - .references('entity_ref') + .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); @@ -35,8 +35,7 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); - table.unique(['entity_ref', 'key']); - table.index('entity_ref', 'refresh_keys_entity_ref_idx'); + table.index('entity_id', 'refresh_keys_entity_id_idx'); table.index('key', 'refresh_keys_key_idx'); }); }; @@ -46,7 +45,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { await knex.schema.alterTable('refresh_keys', table => { - table.dropIndex([], 'refresh_keys_entity_ref_idx'); + table.dropIndex([], 'refresh_keys_entity_id_idx'); table.dropIndex([], 'refresh_keys_key_idx'); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index f9f51b7a0d..d280f3a493 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -522,7 +522,7 @@ describe('Default Processing Database', () => { ); const refreshKeys = await knex('refresh_keys') - .where({ entity_ref: stringifyEntityRef(processedEntity) }) + .where({ entity_id: id }) .select(); expect(refreshKeys[0]).toEqual({ diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 5b191dce32..f0779212be 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -79,6 +79,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, relations, deferredEntities, + refreshKeys, locationKey, } = options; const refreshResult = await tx('refresh_state') @@ -141,17 +142,19 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); + // Delete old refresh keys + await tx('refresh_keys') + .where({ entity_id: id }) + .delete(); + // Insert the refresh keys for the processed entity - await Promise.all( - options.refreshKeys.map(k => { - return tx('refresh_keys') - .insert({ - entity_ref: sourceEntityRef, - key: k.key, - }) - .onConflict(['entity_ref', 'key']) - .ignore(); - }), + await tx.batchInsert( + 'refresh_keys', + refreshKeys.map(k => ({ + entity_id: id, + key: k.key, + })), + BATCH_SIZE, ); return { @@ -540,18 +543,18 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { keys } = options; const updateResult = await tx('refresh_state') - .whereIn('entity_ref', function selectEntityRefs(tx2) { + .whereIn('entity_id', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) .select({ - entity_ref: 'refresh_keys.entity_ref', + entity_id: 'refresh_keys.entity_id', }) .from('refresh_keys'); }) .update({ next_update_at: tx.fn.now() }); if (updateResult === 0) { - throw new NotFoundError( + this.options.logger.info( `Failed to schedule ${JSON.stringify(keys)} for keys`, ); } diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 33c3ab0ef3..b9fe12be11 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -44,7 +44,7 @@ export type DbRefreshStateRow = { }; export type DbRefreshKeysRow = { - entity_ref: string; + entity_id: string; key: string; }; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index da7aa65714..8a839c0650 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -82,10 +82,6 @@ export type ReplaceUnprocessedEntitiesOptions = type: 'delta'; }; -export type RefreshKeyOptions = { - refreshKeys: RefreshKeyData[]; -}; - export type RefreshByKeyOptions = { keys: string[]; }; From 7c85fa3aa35e49a82ae9f7db826dacdbc255c556 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 15:23:27 +0200 Subject: [PATCH 25/35] use absolute urls Signed-off-by: Kiss Miklos --- .../src/modules/core/FileReaderProcessor.test.ts | 2 +- .../src/modules/core/FileReaderProcessor.ts | 5 +++-- .../src/modules/core/PlaceholderProcessor.test.ts | 2 +- .../src/modules/core/PlaceholderProcessor.ts | 12 +++++++++++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index 0c114f49a1..d726edc4db 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,7 +89,7 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].key).toContain('file:'); + expect(emit.mock.calls[1][0].key).toContain('file://'); expect(emit.mock.calls[1][0].key).toContain( 'fileReaderProcessor/component.yaml', ); diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 59795589b2..188bfd0031 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -52,6 +52,7 @@ export class FileReaderProcessor implements CatalogProcessor { if (fileMatches.length > 0) { for (const fileMatch of fileMatches) { const data = await fs.readFile(fileMatch); + const normalizedFilePath = path.normalize(fileMatch); // The normalize converts to native slashes; the glob library returns // forward slashes even on windows @@ -59,13 +60,13 @@ export class FileReaderProcessor implements CatalogProcessor { data: data, location: { type: LOCATION_TYPE, - target: path.normalize(fileMatch), + target: normalizedFilePath, }, })) { emit(parseResult); emit( processingResult.refresh( - `${LOCATION_TYPE}:${path.normalize(fileMatch)}`, + `${LOCATION_TYPE}://${normalizedFilePath}`, ), ); } diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index aa448dd955..02f30c9e69 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -385,7 +385,7 @@ describe('PlaceholderProcessor', () => { console.log(emitted); expect(emitted[0]).toEqual({ type: 'refresh', - key: 'url:./path-to-file.json', + key: 'url:http://example.com/path-to-file.json', }); }); }); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 8a1891fe0c..bcd474016b 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -134,7 +134,17 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit(processingResult.refresh(`url:${resolverValue}`)); + emit( + processingResult.refresh( + `url:${relativeUrl({ + key: resolverKey, + value: resolverValue, + baseUrl: location.target, + read, + resolveUrl, + })}`, + ), + ); return [ await resolver({ From 990d448055396a84e5f2f67e428c001f791020c0 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:21:21 +0200 Subject: [PATCH 26/35] fix tests Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.test.ts | 2 +- .../src/modules/core/PlaceholderProcessor.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index d280f3a493..6a0986b345 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -526,7 +526,7 @@ describe('Default Processing Database', () => { .select(); expect(refreshKeys[0]).toEqual({ - entity_ref: 'location:default/fakelocation', + entity_id: id, key: 'protocol:foo-bar.com', }); }, diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index 02f30c9e69..c8a5a5355a 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -382,7 +382,6 @@ describe('PlaceholderProcessor', () => { { type: 'fake', target: 'http://example.com' }, result => emitted.push(result), ); - console.log(emitted); expect(emitted[0]).toEqual({ type: 'refresh', key: 'url:http://example.com/path-to-file.json', From 04f67153877155023d4fc6528f6f683754bf8808 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:22:47 +0200 Subject: [PATCH 27/35] remove log based on condition Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index f0779212be..903dd59e9e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -542,7 +542,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { keys } = options; - const updateResult = await tx('refresh_state') + await tx('refresh_state') .whereIn('entity_id', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) @@ -552,12 +552,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .from('refresh_keys'); }) .update({ next_update_at: tx.fn.now() }); - - if (updateResult === 0) { - this.options.logger.info( - `Failed to schedule ${JSON.stringify(keys)} for keys`, - ); - } } async transaction(fn: (tx: Transaction) => Promise): Promise { From d2bd28696d749f12c7998470a7b9e6ae865a5a43 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:33:53 +0200 Subject: [PATCH 28/35] add refreshKeys to the hashBuilder Signed-off-by: Kiss Miklos --- .../src/processing/DefaultCatalogProcessingEngine.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 10d6fc6975..82062a8e2c 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -134,6 +134,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { .update(stableStringify({ ...result.completedEntity })) .update(stableStringify([...result.deferredEntities])) .update(stableStringify([...result.relations])) + .update(stableStringify([...result.refreshKeys])) .update(stableStringify([...parents])); } From a35aae43b01127a19c08134e9048e777ccf55f37 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 11:58:43 +0200 Subject: [PATCH 29/35] remove // from refreshKeys Signed-off-by: Kiss Miklos --- .../src/modules/core/FileReaderProcessor.test.ts | 2 +- plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index d726edc4db..0c114f49a1 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,7 +89,7 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].key).toContain('file://'); + expect(emit.mock.calls[1][0].key).toContain('file:'); expect(emit.mock.calls[1][0].key).toContain( 'fileReaderProcessor/component.yaml', ); diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 188bfd0031..8ececed59f 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -66,7 +66,7 @@ export class FileReaderProcessor implements CatalogProcessor { emit(parseResult); emit( processingResult.refresh( - `${LOCATION_TYPE}://${normalizedFilePath}`, + `${LOCATION_TYPE}:${normalizedFilePath}`, ), ); } From 3e3d8282efcca45022abde7b3c4db585fe361e9d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:14:14 +0200 Subject: [PATCH 30/35] pass emit to resolvers Signed-off-by: Kiss Miklos --- .../modules/core/PlaceholderProcessor.test.ts | 2 ++ .../src/modules/core/PlaceholderProcessor.ts | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index c8a5a5355a..9bfa9027d9 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -397,6 +397,7 @@ describe('yamlPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { @@ -442,6 +443,7 @@ describe('jsonPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index bcd474016b..be2e9f9f5b 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -42,6 +42,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; /** @public */ @@ -134,18 +135,6 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit( - processingResult.refresh( - `url:${relativeUrl({ - key: resolverKey, - value: resolverValue, - baseUrl: location.target, - read, - resolveUrl, - })}`, - ), - ); - return [ await resolver({ key: resolverKey, @@ -153,6 +142,7 @@ export class PlaceholderProcessor implements CatalogProcessor { baseUrl: location.target, read, resolveUrl, + emit, }), true, ]; @@ -170,11 +160,13 @@ export class PlaceholderProcessor implements CatalogProcessor { export async function yamlPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); let documents: yaml.Document.Parsed[]; try { - documents = yaml.parseAllDocuments(text).filter(d => d); + documents = yaml.parseAllDocuments(content).filter(d => d); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`, @@ -201,10 +193,12 @@ export async function yamlPlaceholderResolver( export async function jsonPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); try { - return JSON.parse(text); + return JSON.parse(content); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`, @@ -215,7 +209,11 @@ export async function jsonPlaceholderResolver( export async function textPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - return await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + + return content; } /* @@ -224,12 +222,12 @@ export async function textPlaceholderResolver( async function readTextLocation( params: PlaceholderResolverParams, -): Promise { +): Promise<{ content: string; url: string }> { const newUrl = relativeUrl(params); try { const data = await params.read(newUrl); - return data.toString('utf-8'); + return { content: data.toString('utf-8'), url: newUrl }; } catch (e) { throw new Error( `Placeholder \$${params.key} could not read location ${params.value}, ${e}`, From 1dd6c22cc88a49aa8ff9544fe9ec8b9435979b96 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:29:24 +0200 Subject: [PATCH 31/35] add changeset Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/cold-coins-tickle.md diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md new file mode 100644 index 0000000000..7fab336352 --- /dev/null +++ b/.changeset/cold-coins-tickle.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key. + +The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute url of the catalog file as a refresh key. In the format of `:` +The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. From 9a54bb6d2c075c2e714b71c29de9bbf2bb926798 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:34:41 +0200 Subject: [PATCH 32/35] add emit to resolvers Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index c410f166cc..09ca4d8dfe 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -578,6 +578,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; // @public (undocumented) From f3caf2e4b2e6af82690e0a224b531f270669a354 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:46:54 +0200 Subject: [PATCH 33/35] fix url spelling Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md index 7fab336352..80f0958daa 100644 --- a/.changeset/cold-coins-tickle.md +++ b/.changeset/cold-coins-tickle.md @@ -4,5 +4,5 @@ Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key. -The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute url of the catalog file as a refresh key. In the format of `:` +The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `:` The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. From 6714971845c9f331abc6e32266a8fb7240bcba47 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 7 Jul 2022 12:03:37 +0200 Subject: [PATCH 34/35] remove refreshing by keys from refresh service Signed-off-by: Kiss Miklos --- .../src/service/AuthorizedRefreshService.test.ts | 1 - .../src/service/AuthorizedRefreshService.ts | 9 +-------- .../src/service/DefaultRefreshService.ts | 11 +---------- .../catalog-backend/src/service/createRouter.test.ts | 4 ++-- plugins/catalog-backend/src/service/types.ts | 4 ---- 5 files changed, 4 insertions(+), 25 deletions(-) diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index f22b5b86f4..5d0a192470 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,7 +22,6 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), - refreshByRefreshKeys: jest.fn(), }; const permissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 19f6270da6..8634fbf86d 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -20,11 +20,7 @@ import { AuthorizeResult, PermissionEvaluator, } from '@backstage/plugin-permission-common'; -import { - RefreshByRefreshKeysOptions, - RefreshOptions, - RefreshService, -} from './types'; +import { RefreshOptions, RefreshService } from './types'; export class AuthorizedRefreshService implements RefreshService { constructor( @@ -49,7 +45,4 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } - async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { - await this.service.refreshByRefreshKeys(options); - } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index deb69bd041..3b982a0e46 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -15,11 +15,7 @@ */ import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; -import { - RefreshByRefreshKeysOptions, - RefreshOptions, - RefreshService, -} from './types'; +import { RefreshOptions, RefreshService } from './types'; export class DefaultRefreshService implements RefreshService { private database: DefaultProcessingDatabase; @@ -49,9 +45,4 @@ export class DefaultRefreshService implements RefreshService { }); }); } - async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { - await this.database.transaction(async tx => { - await this.database.refreshByRefreshKeys(tx, options); - }); - } } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index b7eeb6f1bc..16a36eb8c4 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; + refreshService = { refresh: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -712,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; + refreshService = { refresh: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index a073e5bb6c..f6cfa2e783 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -61,9 +61,6 @@ export type RefreshOptions = { authorizationToken?: string; }; -export type RefreshByRefreshKeysOptions = { - keys: string[]; -}; /** * A service that manages refreshes of entities in the catalog. * @@ -74,7 +71,6 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; - refreshByRefreshKeys(options: RefreshByRefreshKeysOptions): Promise; } /** From 1c23763ce8de8ced945963818b8c012a90d31147 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 7 Jul 2022 12:09:22 +0200 Subject: [PATCH 35/35] update changeset Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md index 80f0958daa..301a9d0a84 100644 --- a/.changeset/cold-coins-tickle.md +++ b/.changeset/cold-coins-tickle.md @@ -6,3 +6,14 @@ Added an option to be able to trigger refreshes on entities based on a prestored The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `:` The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. + +The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts` + +```ts + // yamlPlaceholderResolver + ... + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + ... +```