From def0eef285491deb3b575f121f24c10eba78a85e Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 20 Jun 2022 11:56:20 +0200 Subject: [PATCH 01/64] add a potential implementation for refreshKeys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 41 +++++++++++++++++++ plugins/catalog-backend/src/api/processor.ts | 10 ++++- .../src/database/DefaultProcessingDatabase.ts | 36 ++++++++++++++++ .../catalog-backend/src/database/tables.ts | 5 +++ plugins/catalog-backend/src/database/types.ts | 13 ++++++ .../src/modules/core/FileReaderProcessor.ts | 7 ++++ .../src/modules/core/PlaceholderProcessor.ts | 8 +++- .../src/modules/core/UrlReaderProcessor.ts | 7 ++++ .../DefaultCatalogProcessingEngine.ts | 6 +++ .../processing/ProcessorOutputCollector.ts | 7 ++++ .../catalog-backend/src/processing/types.ts | 2 +- 11 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 plugins/catalog-backend/migrations/20220616202842_refresh_keys.js diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js new file mode 100644 index 0000000000..dfa0bf5794 --- /dev/null +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -0,0 +1,41 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.createTable('refresh_keys', table => { + table.comment( + 'This table contains relations between entities and keys to trigger refreshes with', + ); + table + .text('entity_ref') + .notNullable() + .comment('A reference to the entity that the refresh key is tied to'); + table + .text('key') + .notNullable() + .comment( + 'A reference to a key which should be used to trigger a refresh on this entity', + ); + table.unique(['entity_ref', 'key']); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('refresh_keys'); +}; diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 3f274d13ea..d5a08edd44 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -169,9 +169,17 @@ export type CatalogProcessorErrorResult = { location: LocationSpec; }; +/** @public */ +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + entity: Entity; + key: String; +}; + /** @public */ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index dd28cf1290..e4a5ab3d85 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,12 +33,15 @@ import { UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, + RefreshKeyOptions, + RefreshByKeyOptions, } from './types'; import { DeferredEntity } from '../processing/types'; import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -515,7 +518,40 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); } } + async refreshByRefreshKey( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ) { + const tx = txOpaque as Knex.Transaction; + const { key } = options; + const rows = await tx('refresh_keys') + .where({ key }) + .select({ + entity_ref: 'refresh_keys.entity_ref', + }); + + await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); + } + async addRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { keys } = options; + + await Promise.all( + keys.map(k => { + return tx('refresh_keys') + .insert({ + entity_ref: stringifyEntityRef(k.entity), + key: k.key, + }) + .onConflict(['entity_ref', 'key']) + .ignore(); + }), + ); + } async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 40b4cb9a12..33c3ab0ef3 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -43,6 +43,11 @@ export type DbRefreshStateRow = { location_key?: string; }; +export type DbRefreshKeysRow = { + entity_ref: string; + key: string; +}; + export type DbRefreshStateReferencesRow = { source_key?: string; source_entity_ref?: string; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ce45a88938..bf3f765f4a 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -81,6 +81,14 @@ export type ReplaceUnprocessedEntitiesOptions = type: 'delta'; }; +export type RefreshKeyOptions = { + keys: { key: String; entity: Entity }[]; +}; + +export type RefreshByKeyOptions = { + key: string; +}; + export type RefreshOptions = { entityRef: string; }; @@ -149,6 +157,11 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + addRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise; + /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 66c6479fa7..768d763a24 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -61,6 +61,13 @@ export class FileReaderProcessor implements CatalogProcessor { }, })) { emit(parseResult); + if (parseResult.type === 'entity') { + emit({ + type: 'refresh', + key: path.normalize(fileMatch), + entity: parseResult.entity, + }); + } } } } else if (!optional) { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index ddd5f951db..5089ee2637 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,11 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from '../../api'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; @@ -66,6 +70,7 @@ export class PlaceholderProcessor implements CatalogProcessor { async preProcessEntity( entity: Entity, location: LocationSpec, + emit: CatalogProcessorEmit, ): Promise { const process = async (data: any): Promise<[any, boolean]> => { if (!data || !(data instanceof Object)) { @@ -102,6 +107,7 @@ export class PlaceholderProcessor implements CatalogProcessor { const resolverKey = keys[0].substr(1); const resolverValue = data[keys[0]]; + emit({ type: 'refresh', key: resolverValue, entity }); const resolver = this.options.resolvers[resolverKey]; if (!resolver || typeof resolverValue !== 'string') { // If there was no such placeholder resolver or if the value was not a diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 7f27bcd19d..a6c5bd74d8 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -83,6 +83,13 @@ export class UrlReaderProcessor implements CatalogProcessor { })) { parseResults.push(parseResult); emit(parseResult); + if (parseResult.type === 'entity') { + emit({ + type: 'refresh', + key: item.url, + entity: parseResult.entity, + }); + } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index b7ef5c9344..d3d679d2b7 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -123,6 +123,12 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { + await this.processingDatabase.transaction(tx => + this.processingDatabase.addRefreshKeys(tx, { + keys: result.refreshKeys, + }), + ); + const { entityRefs: parents } = await this.processingDatabase.transaction(tx => this.processingDatabase.listParents(tx, { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index db7a050dce..058d04ff2d 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -38,6 +38,10 @@ export class ProcessorOutputCollector { private readonly errors = new Array(); private readonly relations = new Array(); private readonly deferredEntities = new Array(); + private readonly refreshKeys = new Array<{ + key: String; + entity: Entity; + }>(); private done = false; constructor( @@ -54,6 +58,7 @@ export class ProcessorOutputCollector { return { errors: this.errors, relations: this.relations, + refreshKeys: this.refreshKeys, deferredEntities: this.deferredEntities, }; } @@ -116,6 +121,8 @@ export class ProcessorOutputCollector { this.relations.push(i.relation); } else if (i.type === 'error') { this.errors.push(i.error); + } else if (i.type === 'refresh') { + this.refreshKeys.push({ key: i.key, entity: i.entity }); } } } diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b178522c14..b93d57863f 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -26,7 +26,6 @@ export type EntityProcessingRequest = { entity: Entity; state?: JsonObject; // Versions for multiple deployments etc }; - /** * The result of processing an entity. * @public @@ -38,6 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; + refreshKeys: { key: String; entity: Entity }[]; errors: Error[]; } | { From 2aa5bb6c58126b5b2b75de7a64f8220150bc7373 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 00:23:39 +0200 Subject: [PATCH 02/64] add refresh function to processingResult Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 2fc2c9eac7..fdc8280ed0 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -65,4 +65,8 @@ export const processingResult = Object.freeze({ relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; }, + + refresh(entity, key) { + return { type: 'refresh', entity, key }; + }, } as const); From 6a0a4ecbfd17d7e4dda30b1f46f0339e2f1e95e0 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 00:25:32 +0200 Subject: [PATCH 03/64] Add refreshByRefreshKey to refresh service Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 2 +- .../src/service/AuthorizedRefreshService.ts | 9 ++++++++- .../src/service/DefaultRefreshService.ts | 11 ++++++++++- plugins/catalog-backend/src/service/types.ts | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index fdc8280ed0..3f05b30acf 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entity, key) { + refresh(entity: Entity, key: String) { return { type: 'refresh', entity, key }; }, } as const); diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 8634fbf86d..2e48b3d8b6 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -20,7 +20,11 @@ import { AuthorizeResult, PermissionEvaluator, } from '@backstage/plugin-permission-common'; -import { RefreshOptions, RefreshService } from './types'; +import { + RefreshByRefreshKeysOptions, + RefreshOptions, + RefreshService, +} from './types'; export class AuthorizedRefreshService implements RefreshService { constructor( @@ -45,4 +49,7 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } + async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + await this.service.refreshByRefreshKey(options); + } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index 3b982a0e46..d0c2742d97 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -15,7 +15,11 @@ */ import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; -import { RefreshOptions, RefreshService } from './types'; +import { + RefreshByRefreshKeysOptions, + RefreshOptions, + RefreshService, +} from './types'; export class DefaultRefreshService implements RefreshService { private database: DefaultProcessingDatabase; @@ -45,4 +49,9 @@ export class DefaultRefreshService implements RefreshService { }); }); } + async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + await this.database.transaction(async tx => { + await this.database.refreshByRefreshKey(tx, options); + }); + } } diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index f6cfa2e783..cb4debc41d 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -61,6 +61,9 @@ export type RefreshOptions = { authorizationToken?: string; }; +export type RefreshByRefreshKeysOptions = { + key: string; +}; /** * A service that manages refreshes of entities in the catalog. * @@ -71,6 +74,7 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; + refreshByRefreshKey(options: RefreshByRefreshKeysOptions): Promise; } /** From 3744b0c67a6cda52730cce7badd1df30a32c2d4d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 12:43:44 +0200 Subject: [PATCH 04/64] addRefreshKEys -> setRefreshKeys Signed-off-by: Kiss Miklos --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 2 +- .../src/processing/DefaultCatalogProcessingEngine.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index e4a5ab3d85..20ef39f6e8 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -533,7 +533,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); } - async addRefreshKeys( + async setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, ): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index bf3f765f4a..9d4043a340 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -157,7 +157,7 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; - addRefreshKeys( + setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, ): Promise; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index d3d679d2b7..5cb9c76e62 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -124,7 +124,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { await this.processingDatabase.transaction(tx => - this.processingDatabase.addRefreshKeys(tx, { + this.processingDatabase.setRefreshKeys(tx, { keys: result.refreshKeys, }), ); From 2b1ab47a7a4bacea634a09b6a9404dbf2eb79929 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 13:08:16 +0200 Subject: [PATCH 05/64] add deleteRefreshKey Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 20ef39f6e8..a003328244 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -518,6 +518,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); } } + async refreshByRefreshKey( txOpaque: Transaction, options: RefreshByKeyOptions, @@ -533,6 +534,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); } + async setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, @@ -552,6 +554,17 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { }), ); } + + async deleteRefreshKey( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { key } = options; + + await tx('refresh_keys').where({ key }).delete(); + } + async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; From b713a4c8dc9bdd33d4485405b0c75ca8b21c4811 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 22 Jun 2022 18:53:38 +0200 Subject: [PATCH 06/64] use entityRef instead of Entity Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 4 ++-- plugins/catalog-backend/src/api/processor.ts | 2 +- .../src/database/DefaultProcessingDatabase.ts | 6 +++--- plugins/catalog-backend/src/database/types.ts | 2 +- .../src/modules/core/FileReaderProcessor.ts | 12 +++++++----- .../src/modules/core/PlaceholderProcessor.ts | 7 +++++-- .../src/modules/core/UrlReaderProcessor.ts | 13 +++++++------ .../DefaultCatalogProcessingEngine.test.ts | 5 +++++ .../processing/DefaultCatalogProcessingEngine.ts | 2 +- .../src/processing/ProcessorOutputCollector.ts | 4 ++-- plugins/catalog-backend/src/processing/types.ts | 2 +- 11 files changed, 35 insertions(+), 24 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 3f05b30acf..e1727a9f78 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entity: Entity, key: String) { - return { type: 'refresh', entity, key }; + refresh(entityRef: String, key: String): CatalogProcessorResult { + return { type: 'refresh', entityRef, key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index d5a08edd44..8480aec681 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,7 +172,7 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entity: Entity; + entityRef: String; key: String; }; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index a003328244..7cd590127c 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -540,13 +540,13 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: RefreshKeyOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { keys } = options; + const { refreshKeys } = options; await Promise.all( - keys.map(k => { + refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: stringifyEntityRef(k.entity), + entity_ref: k.entityRef, key: k.key, }) .onConflict(['entity_ref', 'key']) diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 9d4043a340..b4ab598f4a 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -82,7 +82,7 @@ export type ReplaceUnprocessedEntitiesOptions = }; export type RefreshKeyOptions = { - keys: { key: String; entity: Entity }[]; + refreshKeys: { key: String; entityRef: String }[]; }; export type RefreshByKeyOptions = { diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 768d763a24..20eabf2d03 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -25,6 +25,7 @@ import { LocationSpec, processingResult, } from '../../api'; +import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); @@ -62,11 +63,12 @@ export class FileReaderProcessor implements CatalogProcessor { })) { emit(parseResult); if (parseResult.type === 'entity') { - emit({ - type: 'refresh', - key: path.normalize(fileMatch), - entity: parseResult.entity, - }); + emit( + processingResult.refresh( + stringifyEntityRef(parseResult.entity), + path.normalize(fileMatch), + ), + ); } } } diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 5089ee2637..f1ad5c66f7 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; @@ -23,6 +23,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, + processingResult, } from '../../api'; /** @public */ @@ -107,7 +108,7 @@ export class PlaceholderProcessor implements CatalogProcessor { const resolverKey = keys[0].substr(1); const resolverValue = data[keys[0]]; - emit({ type: 'refresh', key: resolverValue, entity }); + const resolver = this.options.resolvers[resolverKey]; if (!resolver || typeof resolverValue !== 'string') { // If there was no such placeholder resolver or if the value was not a @@ -133,6 +134,8 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); + emit(processingResult.refresh(stringifyEntityRef(entity), resolverValue)); + return [ await resolver({ key: resolverKey, diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index a6c5bd74d8..ec8f7c11ec 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; @@ -84,11 +84,12 @@ export class UrlReaderProcessor implements CatalogProcessor { parseResults.push(parseResult); emit(parseResult); if (parseResult.type === 'entity') { - emit({ - type: 'refresh', - key: item.url, - entity: parseResult.entity, - }); + emit( + processingResult.refresh( + stringifyEntityRef(parseResult.entity), + item.url, + ), + ); } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index 10de36a1be..dd316583af 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -58,6 +58,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -123,6 +124,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -203,6 +205,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( @@ -413,6 +416,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }) .mockResolvedValueOnce({ ok: true, @@ -432,6 +436,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); await engine.start(); diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 5cb9c76e62..f1c7ba6943 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -125,7 +125,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { if (result.ok) { await this.processingDatabase.transaction(tx => this.processingDatabase.setRefreshKeys(tx, { - keys: result.refreshKeys, + refreshKeys: result.refreshKeys, }), ); diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 058d04ff2d..e4b8ca2e9b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -40,7 +40,7 @@ export class ProcessorOutputCollector { private readonly deferredEntities = new Array(); private readonly refreshKeys = new Array<{ key: String; - entity: Entity; + entityRef: String; }>(); private done = false; @@ -122,7 +122,7 @@ export class ProcessorOutputCollector { } else if (i.type === 'error') { this.errors.push(i.error); } else if (i.type === 'refresh') { - this.refreshKeys.push({ key: i.key, entity: i.entity }); + this.refreshKeys.push({ key: i.key, entityRef: i.entityRef }); } } } diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b93d57863f..78d8b875b2 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -37,7 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; - refreshKeys: { key: String; entity: Entity }[]; + refreshKeys: { key: String; entityRef: String }[]; errors: Error[]; } | { From af0b0314249053fb3a849d2b9460f1824728bfcd Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 27 Jun 2022 16:49:23 +0200 Subject: [PATCH 07/64] fix some tests Signed-off-by: Kiss Miklos --- .../DefaultProcessingDatabase.test.ts | 56 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 15 ++++- .../modules/core/FileReaderProcessor.test.ts | 22 ++++++-- .../modules/core/PlaceholderProcessor.test.ts | 13 ++++- .../modules/core/UrlReaderProcessor.test.ts | 2 +- .../DefaultCatalogProcessingEngine.test.ts | 1 + ...faultCatalogProcessingOrchestrator.test.ts | 2 + .../catalog-backend/src/processing/types.ts | 11 +++- .../src/service/DefaultRefreshService.test.ts | 1 + 10 files changed, 112 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 251388f0d2..703595e2ee 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,6 +24,7 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -67,6 +68,10 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; + const insertRefreshKeysRow = async (db: Knex, ref) => { + await db('refresh_keys').insert(ref); + }; + describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -1397,4 +1402,55 @@ describe('Default Processing Database', () => { }, ); }); + + describe('setRefreshKeys', () => { + it.each(databases.eachSupportedId())( + 'should set keys, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await db.transaction(async tx => + db.setRefreshKeys(tx, { + refreshKeys: [{ entityRef: 'location:default/root-1', key: 'foo' }], + }), + ); + + const rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(1); + expect(rows[0]).toEqual({ + entity_ref: 'location:default/root-1', + key: 'foo', + }); + }, + ); + }); + + describe('deleteRefreshKeys', () => { + it.each(databases.eachSupportedId())( + 'should delete keys, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await knex('refresh_keys').insert({ + entity_ref: 'location:default/root-1', + key: 'foo', + }); + + let rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(1); + + await db.transaction(async tx => + db.deleteRefreshKey(tx, { + key: 'foo', + }), + ); + + rows = await knex('refresh_keys').select(); + + expect(rows.length).toBe(0); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 7cd590127c..66b87d6b24 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -562,7 +562,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { key } = options; - await tx('refresh_keys').where({ key }).delete(); + await tx('refresh_keys').where({ key: key }).delete(); } async transaction(fn: (tx: Transaction) => Promise): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index b4ab598f4a..23755ad513 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; import { EntityRelationSpec } from '../api'; -import { DeferredEntity } from '../processing/types'; +import { DeferredEntity, RefreshKeyData } from '../processing/types'; import { DbRelationsRow } from './tables'; /** @@ -82,7 +82,7 @@ export type ReplaceUnprocessedEntitiesOptions = }; export type RefreshKeyOptions = { - refreshKeys: { key: String; entityRef: String }[]; + refreshKeys: RefreshKeyData[]; }; export type RefreshByKeyOptions = { @@ -157,6 +157,17 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + /** + * Schedules a refresh for all the entities that have the given refreshKey + */ + setRefreshKeys( + txOpaque: Transaction, + options: RefreshKeyOptions, + ): Promise; + + /** + * Schedules a refresh for all the entities that have the given refreshKey + */ setRefreshKeys( txOpaque: Transaction, options: RefreshKeyOptions, diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index a3eb5e4542..ad47350346 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -43,7 +43,10 @@ describe('FileReaderProcessor', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual(spec); - expect(generated.entity).toEqual({ kind: 'Component' }); + expect(generated.entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); }); it('should fail load from file with error', async () => { @@ -77,14 +80,23 @@ describe('FileReaderProcessor', () => { defaultEntityDataParser, ); - expect(emit).toBeCalledTimes(2); - expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); + expect(emit).toBeCalledTimes(4); + expect(emit.mock.calls[0][0].entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].entity).toEqual({ kind: 'API' }); - expect(emit.mock.calls[1][0].location).toEqual({ + expect(emit.mock.calls[1][0].entityRef).toEqual( + 'component:default/component-test', + ); + expect(emit.mock.calls[2][0].entity).toEqual({ + kind: 'API', + metadata: { name: 'api-test' }, + }); + expect(emit.mock.calls[2][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index adff97336e..3ec699bc04 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -51,7 +51,7 @@ describe('PlaceholderProcessor', () => { integrations, }); await expect( - processor.preProcessEntity(input, { type: 't', target: 'l' }), + processor.preProcessEntity(input, { type: 't', target: 'l' }, () => {}), ).resolves.toBe(input); }); @@ -76,6 +76,7 @@ describe('PlaceholderProcessor', () => { spec: { a: [{ b: { $upper: 'text' } }] }, }, { type: 'fake', target: 'http://example.com' }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -110,7 +111,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -131,7 +132,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -158,6 +159,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -194,6 +196,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -228,6 +231,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -266,6 +270,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -303,6 +308,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -343,6 +349,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).rejects.toThrow( /^Placeholder \$text could not form a URL out of \.\/a\/b\/catalog-info\.yaml and \.\.\/c\/catalog-info\.yaml, TypeError \[ERR_INVALID_URL\]/, diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index dbcb9e34df..ecb0eb30d3 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -74,7 +74,7 @@ describe('UrlReaderProcessor', () => { mockCache, ); - expect(emitted.length).toBe(1); + expect(emitted.length).toBe(2); expect(emitted[0]).toEqual({ type: 'entity', location: spec, diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index dd316583af..3b6f126391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -30,6 +30,7 @@ describe('DefaultCatalogProcessingEngine', () => { updateProcessedEntity: jest.fn(), updateEntityCache: jest.fn(), listParents: jest.fn(), + setRefreshKeys: jest.fn(), } as unknown as jest.Mocked; const orchestrator: jest.Mocked = { process: jest.fn(), diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 6e494d784e..b2d9846391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -102,6 +102,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ok: true, completedEntity: entity, deferredEntities: [], + refreshKeys: [], errors: [], relations: [], state: { @@ -119,6 +120,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ).resolves.toEqual({ ok: true, completedEntity: entity, + refreshKeys: [], deferredEntities: [ { locationKey: 'url:./new-place', diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 78d8b875b2..0bfbbf613b 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -37,7 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; - refreshKeys: { key: String; entityRef: String }[]; + refreshKeys: RefreshKeyData[]; errors: Error[]; } | { @@ -45,6 +45,15 @@ export type EntityProcessingResult = errors: Error[]; }; +/** + * A string to associate to the entity itself. + * @public + */ +export type RefreshKeyData = { + key: String; + entityRef: String; +}; + /** * Responsible for executing the individual processing steps in order to fully process an entity. * @public diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 84bfde1259..419aedc15d 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -138,6 +138,7 @@ describe('Refresh integration', () => { errors: [], deferredEntities, state: {}, + refreshKeys: [], }; }, }, From 1e53a82d52d161668b7ab2926dbc91242ca1b6e3 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 00:52:22 +0200 Subject: [PATCH 08/64] fix existing tests Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.test.ts | 4 ++-- .../core/__fixtures__/fileReaderProcessor/component.yaml | 2 ++ .../core/__fixtures__/fileReaderProcessor/dir/api.yaml | 2 ++ .../src/service/AuthorizedRefreshService.test.ts | 1 + .../src/service/DefaultLocationService.test.ts | 6 ++++++ plugins/catalog-backend/src/service/createRouter.test.ts | 5 +++-- 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 703595e2ee..129e92cc0e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -68,7 +68,7 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; - const insertRefreshKeysRow = async (db: Knex, ref) => { + const insertRefreshKeysRow = async (db: Knex, ref: DbRefreshKeysRow) => { await db('refresh_keys').insert(ref); }; @@ -1432,7 +1432,7 @@ describe('Default Processing Database', () => { async databaseId => { const { knex, db } = await createDatabase(databaseId); - await knex('refresh_keys').insert({ + await insertRefreshKeysRow(knex, { entity_ref: 'location:default/root-1', key: 'foo', }); diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml index 8524aaf14a..b5844bd6ed 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml @@ -1 +1,3 @@ kind: Component +metadata: + name: component-test diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml index a894e34f70..ad33548f13 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml @@ -1 +1,3 @@ kind: API +metadata: + name: api-test diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index 5d0a192470..17a3ef3b45 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,6 +22,7 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), + refreshByRefreshKey: jest.fn(), }; const permissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts index bd166cbd50..8151bbf18d 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts @@ -48,6 +48,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -75,6 +76,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -134,6 +136,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -161,6 +164,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -188,6 +192,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -211,6 +216,7 @@ describe('DefaultLocationServiceTest', () => { name: 'bar', }, }, + refreshKeys: [], deferredEntities: [], relations: [], errors: [], diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 8eafd8bfa3..d652893b55 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -409,6 +409,7 @@ describe('createRouter readonly disabled', () => { state: {}, completedEntity: entity, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -711,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, From d65305574690086a76998d95b7531467c2d4db85 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 00:59:15 +0200 Subject: [PATCH 09/64] fix type from String to string Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/processingResult.ts | 2 +- plugins/catalog-backend/src/api/processor.ts | 4 ++-- .../src/processing/ProcessorOutputCollector.ts | 7 ++----- plugins/catalog-backend/src/processing/types.ts | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index e1727a9f78..4eefca196c 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entityRef: String, key: String): CatalogProcessorResult { + refresh(entityRef: string, key: string): CatalogProcessorResult { return { type: 'refresh', entityRef, key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 8480aec681..196391cf50 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,8 +172,8 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: String; - key: String; + entityRef: string; + key: string; }; /** @public */ diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index e4b8ca2e9b..56988569cf 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -24,7 +24,7 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity } from './types'; +import { DeferredEntity, RefreshKeyData } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, @@ -38,10 +38,7 @@ export class ProcessorOutputCollector { private readonly errors = new Array(); private readonly relations = new Array(); private readonly deferredEntities = new Array(); - private readonly refreshKeys = new Array<{ - key: String; - entityRef: String; - }>(); + private readonly refreshKeys = new Array(); private done = false; constructor( diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 0bfbbf613b..64b1dac16f 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -50,8 +50,8 @@ export type EntityProcessingResult = * @public */ export type RefreshKeyData = { - key: String; - entityRef: String; + key: string; + entityRef: string; }; /** From bb58626b1b5f546b8884b21f86614859c2bf162d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 01:12:05 +0200 Subject: [PATCH 10/64] export CatalogProcessorRefreshKeysResult Signed-off-by: Kiss Miklos --- plugins/catalog-backend/src/api/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts index 1cb4b71aa3..136c23d20d 100644 --- a/plugins/catalog-backend/src/api/index.ts +++ b/plugins/catalog-backend/src/api/index.ts @@ -26,6 +26,7 @@ export type { CatalogProcessorRelationResult, CatalogProcessorErrorResult, CatalogProcessorResult, + CatalogProcessorRefreshKeysResult, } from './processor'; export type { EntityProvider, From 3117584484a1ed785ae4a7e16cdc6a6afd796c67 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 01:15:49 +0200 Subject: [PATCH 11/64] generate new api-report.md Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index cb8a5abb8a..fbbf2186ca 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -268,6 +268,13 @@ export type CatalogProcessorParser = (options: { location: LocationSpec; }) => AsyncIterable; +// @public (undocumented) +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + entityRef: string; + key: string; +}; + // @public (undocumented) export type CatalogProcessorRelationResult = { type: 'relation'; @@ -279,7 +286,8 @@ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; // @public (undocumented) export class CodeOwnersProcessor implements CatalogProcessor { @@ -545,7 +553,11 @@ export class PlaceholderProcessor implements CatalogProcessor { // (undocumented) getProcessorName(): string; // (undocumented) - preProcessEntity(entity: Entity, location: LocationSpec): Promise; + preProcessEntity( + entity: Entity, + location: LocationSpec, + emit: CatalogProcessorEmit, + ): Promise; } // @public (undocumented) @@ -601,6 +613,7 @@ export const processingResult: Readonly<{ newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; + readonly refresh: (entityRef: string, key: string) => CatalogProcessorResult; }>; // @public (undocumented) From 3125f54ad63f18bd374a0cc47cf304a14fa8fb5f Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 28 Jun 2022 10:14:07 +0200 Subject: [PATCH 12/64] fix UrlReaderProcessor test Signed-off-by: Kiss Miklos --- .../modules/core/UrlReaderProcessor.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index ecb0eb30d3..631f463fd5 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -61,7 +61,13 @@ describe('UrlReaderProcessor', () => { server.use( rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => - res(ctx.set({ ETag: 'my-etag' }), ctx.json({ mock: 'entity' })), + res( + ctx.set({ ETag: 'my-etag' }), + ctx.json({ + kind: 'component', + metadata: { name: 'mock-url-entity' }, + }), + ), ), ); @@ -78,11 +84,17 @@ describe('UrlReaderProcessor', () => { expect(emitted[0]).toEqual({ type: 'entity', location: spec, - entity: { mock: 'entity' }, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, }); expect(mockCache.set).toBeCalledWith('v1', { etag: 'my-etag', - value: [{ type: 'entity', location: spec, entity: { mock: 'entity' } }], + value: [ + { + type: 'entity', + location: spec, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, + }, + ], }); expect(mockCache.set).toBeCalledTimes(1); }); From 137b029a2d3e3954b71cb9187f9949b093a2e613 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 16:12:09 +0200 Subject: [PATCH 13/64] make refreshByRefreshKeys accept an array of keys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 3 ++ .../src/database/DefaultProcessingDatabase.ts | 34 +++++++++---------- plugins/catalog-backend/src/database/types.ts | 10 +----- .../src/service/DefaultRefreshService.ts | 2 +- plugins/catalog-backend/src/service/types.ts | 2 +- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index dfa0bf5794..8358cbb0cd 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -14,6 +14,9 @@ * limitations under the License. */ +/** + * @param { import("knex").Knex } knex + */ exports.up = async function up(knex) { await knex.schema.createTable('refresh_keys', table => { table.comment( diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 66b87d6b24..3b93e9a222 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -519,20 +519,28 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } - async refreshByRefreshKey( + async refreshByRefreshKeys( txOpaque: Transaction, options: RefreshByKeyOptions, ) { const tx = txOpaque as Knex.Transaction; - const { key } = options; + const { keys } = options; - const rows = await tx('refresh_keys') - .where({ key }) - .select({ - entity_ref: 'refresh_keys.entity_ref', - }); + const query = await tx('refresh_state') + .whereIn('entity_ref', function (tx2) { + tx2 + .whereIn('key', keys) + .select({ + entity_ref: 'refresh_keys.entity_ref', + }) + .from('refresh_keys') + .columns('entity_ref'); + }) + .update({ next_update_at: tx.fn.now() }) + .toSQL() + .toNative(); - await Promise.all(rows.map(r => this.refresh(tx, r.entity_ref))); + console.log(query, '@@@@@@@!!!!!!!@@@@@'); } async setRefreshKeys( @@ -555,16 +563,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } - async deleteRefreshKey( - txOpaque: Transaction, - options: RefreshByKeyOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - const { key } = options; - - await tx('refresh_keys').where({ key: key }).delete(); - } - async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 23755ad513..4d2b64a721 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -86,7 +86,7 @@ export type RefreshKeyOptions = { }; export type RefreshByKeyOptions = { - key: string; + keys: string[]; }; export type RefreshOptions = { @@ -165,14 +165,6 @@ export interface ProcessingDatabase { options: RefreshKeyOptions, ): Promise; - /** - * Schedules a refresh for all the entities that have the given refreshKey - */ - setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise; - /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index d0c2742d97..e5deab1790 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -51,7 +51,7 @@ export class DefaultRefreshService implements RefreshService { } async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { await this.database.transaction(async tx => { - await this.database.refreshByRefreshKey(tx, options); + await this.database.refreshByRefreshKeys(tx, options); }); } } diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index cb4debc41d..1e2142fc3a 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -62,7 +62,7 @@ export type RefreshOptions = { }; export type RefreshByRefreshKeysOptions = { - key: string; + keys: string[]; }; /** * A service that manages refreshes of entities in the catalog. From 8f84695e0ffb9bf2bde1c959fd0dc440e0fa2d52 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 18:45:03 +0200 Subject: [PATCH 14/64] set the refreshKeys in the updateProcessedEntity function Signed-off-by: Kiss Miklos --- .../src/api/processingResult.ts | 4 +- plugins/catalog-backend/src/api/processor.ts | 1 - .../src/database/DefaultProcessingDatabase.ts | 51 +++++++++---------- plugins/catalog-backend/src/database/types.ts | 9 +--- .../src/modules/core/FileReaderProcessor.ts | 19 ++++--- .../src/modules/core/PlaceholderProcessor.ts | 2 +- .../src/modules/core/UrlReaderProcessor.ts | 11 ++-- .../DefaultCatalogProcessingEngine.ts | 7 +-- .../catalog-backend/src/processing/types.ts | 1 - 9 files changed, 40 insertions(+), 65 deletions(-) diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 4eefca196c..84f1f4b70d 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -66,7 +66,7 @@ export const processingResult = Object.freeze({ return { type: 'relation', relation: spec }; }, - refresh(entityRef: string, key: string): CatalogProcessorResult { - return { type: 'refresh', entityRef, key }; + refresh(key: string): CatalogProcessorResult { + return { type: 'refresh', key }; }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 196391cf50..44e8b298b0 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -172,7 +172,6 @@ export type CatalogProcessorErrorResult = { /** @public */ export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: string; key: string; }; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 3b93e9a222..b8d7493360 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -103,11 +103,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { `Conflicting write of processing result for ${id} with location key '${locationKey}'`, ); } + const sourceEntityRef = stringifyEntityRef(processedEntity); // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { entities: deferredEntities, - sourceEntityRef: stringifyEntityRef(processedEntity), + sourceEntityRef, }); // Delete old relations @@ -141,6 +142,19 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); + // Insert the refresh keys for the procssed entity + await Promise.all( + options.refreshKeys.map(k => { + return tx('refresh_keys') + .insert({ + entity_ref: sourceEntityRef, + key: k.key, + }) + .onConflict(['entity_ref', 'key']) + .ignore(); + }), + ); + return { previous: { relations: previousRelationRows, @@ -526,41 +540,22 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { keys } = options; - const query = await tx('refresh_state') + const updateResult = await tx('refresh_state') .whereIn('entity_ref', function (tx2) { tx2 .whereIn('key', keys) .select({ entity_ref: 'refresh_keys.entity_ref', }) - .from('refresh_keys') - .columns('entity_ref'); + .from('refresh_keys'); }) - .update({ next_update_at: tx.fn.now() }) - .toSQL() - .toNative(); + .update({ next_update_at: tx.fn.now() }); - console.log(query, '@@@@@@@!!!!!!!@@@@@'); - } - - async setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - const { refreshKeys } = options; - - await Promise.all( - refreshKeys.map(k => { - return tx('refresh_keys') - .insert({ - entity_ref: k.entityRef, - key: k.key, - }) - .onConflict(['entity_ref', 'key']) - .ignore(); - }), - ); + if (updateResult === 0) { + throw new NotFoundError( + `Failed to schedule ${JSON.stringify(keys)} for keys`, + ); + } } async transaction(fn: (tx: Transaction) => Promise): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 4d2b64a721..da7aa65714 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -38,6 +38,7 @@ export type UpdateProcessedEntityOptions = { relations: EntityRelationSpec[]; deferredEntities: DeferredEntity[]; locationKey?: string; + refreshKeys: RefreshKeyData[]; }; export type UpdateEntityCacheOptions = { @@ -157,14 +158,6 @@ export interface ProcessingDatabase { */ refresh(txOpaque: Transaction, options: RefreshOptions): Promise; - /** - * Schedules a refresh for all the entities that have the given refreshKey - */ - setRefreshKeys( - txOpaque: Transaction, - options: RefreshKeyOptions, - ): Promise; - /** * Lists all ancestors of a given entityRef. * diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 20eabf2d03..1763a17acb 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -29,6 +29,8 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); +const LOCATION_TYPE = 'file'; + /** @public */ export class FileReaderProcessor implements CatalogProcessor { getProcessorName(): string { @@ -41,7 +43,7 @@ export class FileReaderProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'file') { + if (location.type !== LOCATION_TYPE) { return false; } @@ -57,19 +59,16 @@ export class FileReaderProcessor implements CatalogProcessor { for await (const parseResult of parser({ data: data, location: { - type: 'file', + type: LOCATION_TYPE, target: path.normalize(fileMatch), }, })) { emit(parseResult); - if (parseResult.type === 'entity') { - emit( - processingResult.refresh( - stringifyEntityRef(parseResult.entity), - path.normalize(fileMatch), - ), - ); - } + emit( + processingResult.refresh( + `${LOCATION_TYPE}:${path.normalize(fileMatch)}`, + ), + ); } } } else if (!optional) { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index f1ad5c66f7..5b6ed31f4d 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -134,7 +134,7 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit(processingResult.refresh(stringifyEntityRef(entity), resolverValue)); + emit(processingResult.refresh(`url:${resolverValue}`)); return [ await resolver({ diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index ec8f7c11ec..61bb8fed99 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -30,6 +30,7 @@ import { LocationSpec, processingResult, } from '../../api'; +import { locationSpecToLocationEntity } from '../../util'; const CACHE_KEY = 'v1'; @@ -83,14 +84,6 @@ export class UrlReaderProcessor implements CatalogProcessor { })) { parseResults.push(parseResult); emit(parseResult); - if (parseResult.type === 'entity') { - emit( - processingResult.refresh( - stringifyEntityRef(parseResult.entity), - item.url, - ), - ); - } } } @@ -101,6 +94,8 @@ export class UrlReaderProcessor implements CatalogProcessor { value: parseResults as CatalogProcessorEntityResult[], }); } + + emit(processingResult.refresh(`${location.type}:${location.target}`)); } catch (error) { assertError(error); const message = `Unable to read ${location.type}, ${error}`; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index f1c7ba6943..10d6fc6975 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -123,12 +123,6 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { let hashBuilder = this.createHash().update(errorsString); if (result.ok) { - await this.processingDatabase.transaction(tx => - this.processingDatabase.setRefreshKeys(tx, { - refreshKeys: result.refreshKeys, - }), - ); - const { entityRefs: parents } = await this.processingDatabase.transaction(tx => this.processingDatabase.listParents(tx, { @@ -186,6 +180,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { relations: result.relations, deferredEntities: result.deferredEntities, locationKey, + refreshKeys: result.refreshKeys, }); oldRelationSources = new Set( previous.relations.map(r => r.source_entity_ref), diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index 64b1dac16f..e94125b6f5 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -51,7 +51,6 @@ export type EntityProcessingResult = */ export type RefreshKeyData = { key: string; - entityRef: string; }; /** From 8de6b8912123cb92fe4f772e31b5d59565e498e3 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 18:56:14 +0200 Subject: [PATCH 15/64] use inices instead of table.unique Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index 8358cbb0cd..4e15a948cc 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -24,6 +24,9 @@ exports.up = async function up(knex) { ); table .text('entity_ref') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') .notNullable() .comment('A reference to the entity that the refresh key is tied to'); table @@ -32,7 +35,8 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); - table.unique(['entity_ref', 'key']); + table.index('entity_ref', 'refresh_keys_entity_ref_idx'); + table.index('key', 'refresh_keys_key_idx'); }); }; @@ -40,5 +44,10 @@ exports.up = async function up(knex) { * @param { import("knex").Knex } knex */ exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_keys', table => { + table.dropIndex([], 'refresh_keys_entity_ref_idx'); + table.dropIndex([], 'refresh_keys_key_idx'); + }); + await knex.schema.dropTable('refresh_keys'); }; From e31cc714c36ef6bf88179c5a6d9fda50a13f38f9 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 30 Jun 2022 19:00:10 +0200 Subject: [PATCH 16/64] move notNullable upper Signed-off-by: Kiss Miklos --- .../catalog-backend/migrations/20220616202842_refresh_keys.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index 4e15a948cc..a4e02545c7 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -24,10 +24,10 @@ exports.up = async function up(knex) { ); table .text('entity_ref') + .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') - .notNullable() .comment('A reference to the entity that the refresh key is tied to'); table .text('key') From 2355a43259182585a5290eddf2c2322e99864e6b Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 10:37:30 +0200 Subject: [PATCH 17/64] name function Signed-off-by: Kiss Miklos --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index b8d7493360..de8d18aad6 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -541,7 +541,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { keys } = options; const updateResult = await tx('refresh_state') - .whereIn('entity_ref', function (tx2) { + .whereIn('entity_ref', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) .select({ From 2eb8c3d71bac3922686897d4b29c27cb0066e20f Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 12:17:53 +0200 Subject: [PATCH 18/64] fix tsc issues Signed-off-by: Kiss Miklos --- .../DefaultProcessingDatabase.test.ts | 64 +++---------------- .../src/database/DefaultProcessingDatabase.ts | 1 - .../src/modules/core/FileReaderProcessor.ts | 1 - .../src/modules/core/PlaceholderProcessor.ts | 2 +- .../src/modules/core/UrlReaderProcessor.ts | 3 +- .../processing/ProcessorOutputCollector.ts | 2 +- 6 files changed, 11 insertions(+), 62 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 129e92cc0e..1e96c13dc5 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,7 +24,6 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { - DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -68,10 +67,6 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; - const insertRefreshKeysRow = async (db: Knex, ref: DbRefreshKeysRow) => { - await db('refresh_keys').insert(ref); - }; - describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -103,6 +98,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities: [], + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'undefined'`, @@ -122,6 +118,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }; const { knex, db } = await createDatabase(databaseId); @@ -148,6 +145,7 @@ describe('Default Processing Database', () => { ...options, resultHash: '', locationKey: 'fail', + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'fail'`, @@ -179,6 +177,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }), ); @@ -233,6 +232,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -255,6 +255,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -314,6 +315,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities, + refreshKeys: [], }), ); @@ -405,6 +407,7 @@ describe('Default Processing Database', () => { processedEntity, resultHash: '', relations: [], + refreshKeys: [], deferredEntities: [ { entity: { @@ -1402,55 +1405,4 @@ describe('Default Processing Database', () => { }, ); }); - - describe('setRefreshKeys', () => { - it.each(databases.eachSupportedId())( - 'should set keys, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await db.transaction(async tx => - db.setRefreshKeys(tx, { - refreshKeys: [{ entityRef: 'location:default/root-1', key: 'foo' }], - }), - ); - - const rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(1); - expect(rows[0]).toEqual({ - entity_ref: 'location:default/root-1', - key: 'foo', - }); - }, - ); - }); - - describe('deleteRefreshKeys', () => { - it.each(databases.eachSupportedId())( - 'should delete keys, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await insertRefreshKeysRow(knex, { - entity_ref: 'location:default/root-1', - key: 'foo', - }); - - let rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(1); - - await db.transaction(async tx => - db.deleteRefreshKey(tx, { - key: 'foo', - }), - ); - - rows = await knex('refresh_keys').select(); - - expect(rows.length).toBe(0); - }, - ); - }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index de8d18aad6..11b723a117 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,7 +33,6 @@ import { UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, - RefreshKeyOptions, RefreshByKeyOptions, } from './types'; import { DeferredEntity } from '../processing/types'; diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 1763a17acb..59795589b2 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -25,7 +25,6 @@ import { LocationSpec, processingResult, } from '../../api'; -import { stringifyEntityRef } from '@backstage/catalog-model'; const glob = promisify(g); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 5b6ed31f4d..8a1891fe0c 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 61bb8fed99..7b62690341 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -15,7 +15,7 @@ */ import { UrlReader } from '@backstage/backend-common'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; @@ -30,7 +30,6 @@ import { LocationSpec, processingResult, } from '../../api'; -import { locationSpecToLocationEntity } from '../../util'; const CACHE_KEY = 'v1'; diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 56988569cf..033dfd4ae3 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -119,7 +119,7 @@ export class ProcessorOutputCollector { } else if (i.type === 'error') { this.errors.push(i.error); } else if (i.type === 'refresh') { - this.refreshKeys.push({ key: i.key, entityRef: i.entityRef }); + this.refreshKeys.push({ key: i.key }); } } } From b79a82028a064e6e331edc0855cfac7ffb9c5b53 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 14:27:49 +0200 Subject: [PATCH 19/64] update api-report.md Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index fbbf2186ca..c410f166cc 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -271,7 +271,6 @@ export type CatalogProcessorParser = (options: { // @public (undocumented) export type CatalogProcessorRefreshKeysResult = { type: 'refresh'; - entityRef: string; key: string; }; @@ -613,7 +612,7 @@ export const processingResult: Readonly<{ newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; - readonly refresh: (entityRef: string, key: string) => CatalogProcessorResult; + readonly refresh: (key: string) => CatalogProcessorResult; }>; // @public (undocumented) From 197ed5b83c91bcfb72cce5f6099d4737621324e4 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 1 Jul 2022 17:21:47 +0200 Subject: [PATCH 20/64] add and fix tests Signed-off-by: Kiss Miklos --- .../modules/core/FileReaderProcessor.test.ts | 5 +-- .../modules/core/PlaceholderProcessor.test.ts | 31 +++++++++++++++++++ .../modules/core/UrlReaderProcessor.test.ts | 4 +++ .../src/service/AuthorizedRefreshService.ts | 4 +-- .../src/service/DefaultRefreshService.ts | 2 +- .../src/service/createRouter.test.ts | 4 +-- plugins/catalog-backend/src/service/types.ts | 2 +- 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index ad47350346..0c114f49a1 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,8 +89,9 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].entityRef).toEqual( - 'component:default/component-test', + expect(emit.mock.calls[1][0].key).toContain('file:'); + expect(emit.mock.calls[1][0].key).toContain( + 'fileReaderProcessor/component.yaml', ); expect(emit.mock.calls[2][0].entity).toEqual({ kind: 'API', diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index 3ec699bc04..aa448dd955 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -17,6 +17,7 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; +import { CatalogProcessorResult } from '../../api'; import { jsonPlaceholderResolver, PlaceholderProcessor, @@ -357,6 +358,36 @@ describe('PlaceholderProcessor', () => { expect(read).not.toBeCalled(); }); + it('should emit the resolverValue as a refreshKey', async () => { + read.mockResolvedValue( + Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + ); + + const processor = new PlaceholderProcessor({ + resolvers: { + json: jsonPlaceholderResolver, + }, + reader, + integrations, + }); + + const emitted = new Array(); + await processor.preProcessEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { a: [{ b: { $json: './path-to-file.json' } }] }, + }, + { type: 'fake', target: 'http://example.com' }, + result => emitted.push(result), + ); + console.log(emitted); + expect(emitted[0]).toEqual({ + type: 'refresh', + key: 'url:./path-to-file.json', + }); + }); }); describe('yamlPlaceholderResolver', () => { diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index 631f463fd5..fb3f48c939 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -86,6 +86,10 @@ describe('UrlReaderProcessor', () => { location: spec, entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, }); + expect(emitted[1]).toEqual({ + type: 'refresh', + key: 'url:http://localhost/component.yaml', + }); expect(mockCache.set).toBeCalledWith('v1', { etag: 'my-etag', value: [ diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 2e48b3d8b6..19f6270da6 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -49,7 +49,7 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } - async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { - await this.service.refreshByRefreshKey(options); + async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { + await this.service.refreshByRefreshKeys(options); } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index e5deab1790..deb69bd041 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -49,7 +49,7 @@ export class DefaultRefreshService implements RefreshService { }); }); } - async refreshByRefreshKey(options: RefreshByRefreshKeysOptions) { + async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { await this.database.transaction(async tx => { await this.database.refreshByRefreshKeys(tx, options); }); diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index d652893b55..b7eeb6f1bc 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -712,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKey: jest.fn() }; + refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index 1e2142fc3a..a073e5bb6c 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -74,7 +74,7 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; - refreshByRefreshKey(options: RefreshByRefreshKeysOptions): Promise; + refreshByRefreshKeys(options: RefreshByRefreshKeysOptions): Promise; } /** From 859c547303080451ce34c576a5c0d3a227e7298b Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 17:09:53 +0200 Subject: [PATCH 21/64] add test for storing refreshKeys Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 3 +- .../DefaultProcessingDatabase.test.ts | 58 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 16 ++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index a4e02545c7..ded53e4f67 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -25,7 +25,7 @@ exports.up = async function up(knex) { table .text('entity_ref') .notNullable() - .references('entity_id') + .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); @@ -35,6 +35,7 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); + table.unique(['entity_ref', 'key']); table.index('entity_ref', 'refresh_keys_entity_ref_idx'); table.index('key', 'refresh_keys_key_idx'); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 1e96c13dc5..f9f51b7a0d 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,6 +24,7 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -473,6 +474,63 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'stores the refresh keys for the entity', + async databaseId => { + const mockLogger = { + debug: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + }; + const { knex, db } = await createDatabase( + databaseId, + mockLogger as unknown as Logger, + ); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const deferredEntities = [ + { + entity: { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'next', + }, + }, + locationKey: 'mock', + }, + ]; + + await db.transaction(tx => + db.updateProcessedEntity(tx, { + id, + processedEntity, + resultHash: '', + relations: [], + deferredEntities, + refreshKeys: [{ key: 'protocol:foo-bar.com' }], + }), + ); + + const refreshKeys = await knex('refresh_keys') + .where({ entity_ref: stringifyEntityRef(processedEntity) }) + .select(); + + expect(refreshKeys[0]).toEqual({ + entity_ref: 'location:default/fakelocation', + key: 'protocol:foo-bar.com', + }); + }, + ); }); describe('updateEntityCache', () => { diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 11b723a117..1f984c1bab 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -141,12 +141,24 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); - // Insert the refresh keys for the procssed entity + // Find the top-level location entity that manages the processedEntity + let entityRefToRefresh = sourceEntityRef; + const { entityRefs } = await this.listAncestors(tx, { + entityRef: sourceEntityRef, + }); + const locationAncestor = entityRefs.find(ref => + ref.startsWith('location:'), + ); + if (locationAncestor) { + entityRefToRefresh = locationAncestor; + } + + // Insert the refresh keys for the processed entity await Promise.all( options.refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: sourceEntityRef, + entity_ref: entityRefToRefresh, key: k.key, }) .onConflict(['entity_ref', 'key']) From aa1e84e78a9b2599dede960a405e282174148e82 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 18:23:28 +0200 Subject: [PATCH 22/64] fix types Signed-off-by: Kiss Miklos --- .../src/service/AuthorizedRefreshService.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index 17a3ef3b45..f22b5b86f4 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,7 +22,7 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), - refreshByRefreshKey: jest.fn(), + refreshByRefreshKeys: jest.fn(), }; const permissionApi = { authorize: jest.fn(), From 61aca33a258b4d37f427666bb9df73f51a5b084a Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Mon, 4 Jul 2022 18:35:02 +0200 Subject: [PATCH 23/64] use onConflict Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 1f984c1bab..5b191dce32 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -141,24 +141,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); - // Find the top-level location entity that manages the processedEntity - let entityRefToRefresh = sourceEntityRef; - const { entityRefs } = await this.listAncestors(tx, { - entityRef: sourceEntityRef, - }); - const locationAncestor = entityRefs.find(ref => - ref.startsWith('location:'), - ); - if (locationAncestor) { - entityRefToRefresh = locationAncestor; - } - // Insert the refresh keys for the processed entity await Promise.all( options.refreshKeys.map(k => { return tx('refresh_keys') .insert({ - entity_ref: entityRefToRefresh, + entity_ref: sourceEntityRef, key: k.key, }) .onConflict(['entity_ref', 'key']) From b170099b58b503130cdf70311aa996e1172df64a Mon Sep 17 00:00:00 2001 From: Herman Jensen Date: Tue, 5 Jul 2022 11:20:57 +0200 Subject: [PATCH 24/64] docs: add Intility to adopter list Signed-off-by: Herman Jensen --- ADOPTERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index 1cb0577c44..2275c8da3d 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -189,4 +189,4 @@ _You can do this by using the [Adopter form](https://form.typeform.com/to/zcOaKi | [William Hill](https://www.williamhillgroup.com/) | [Pat Mills](mailto:pat.mills@williamhill.com), [Nathan Flynn](mailto:nflynn@williamhill.co.uk), and [Nishkarsh Raj](mailto:nishkarsh.raj@williamhill.co.uk) | William Hill are leveraging Backstage to build our Engineering Portal. Our mission is to centralize the software catalog inventory to enable service discoverability, reduce the onboarding time for new Engineers, provide a single pane of glass to accelerate Developer Productivity and Save Engineers time. Our aspiration is to create an InnerSource community focussed on organization-wide patterns that are re-usable and can be self-served with the Scaffolder. | | [Vodafone NewZealand Limited](https://vodafone.co.nz) | [Ankit Gupta](mailto:ankit.gupta@vodafone.nz), [DevOps COE](mailto:devopstooling@vodafone.nz) | Vodafone NZ are leveraging Backstage to build centralised and self service Engineering Portal. Our mission is to standardised Pipeline templates across the Engineering teams, One shop stop to create the pipelines and repository with a template approach which reduces creation part from days to minutes and no wait time for developers. A unified view for Azure DevOps pipeline, Azure Repo pull requests, Deployment status from Azure RedHat Openshift-ArgoCD and SonarQube Security and code quality scans report on a single pan to provide a streamlined view for all microservices across the app stack. | | [Coamo](http://www.coamo.com.br) | [@holiiveira](https://github.com/holiiveira), [@gpxlnx](https://github.com/gpxlnx) | We're starting to use it as the main tool of a DevOps platform. Our goal is to provide software templates, centralize our software catalog enabling efficient service discovery, and make it easy to manage the entire software ecosystem in one place. - | +| [Intility](https://intility.no/en/) | [@daniwk](https://github.com/daniwk) | We are creating a developer portal powered by Backstage, with software catalog, documentation, templates and integrations to our infrastructure and internal tools. | From e75bde1342122eb7885ea91e188d00b8121ffe30 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 14:39:10 +0200 Subject: [PATCH 25/64] use entity_id instead of entity_ref Signed-off-by: Kiss Miklos --- .../migrations/20220616202842_refresh_keys.js | 9 +++--- .../DefaultProcessingDatabase.test.ts | 2 +- .../src/database/DefaultProcessingDatabase.ts | 29 ++++++++++--------- .../catalog-backend/src/database/tables.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 4 --- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index ded53e4f67..b1b67f68f2 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -23,9 +23,9 @@ exports.up = async function up(knex) { 'This table contains relations between entities and keys to trigger refreshes with', ); table - .text('entity_ref') + .text('entity_id') .notNullable() - .references('entity_ref') + .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); @@ -35,8 +35,7 @@ exports.up = async function up(knex) { .comment( 'A reference to a key which should be used to trigger a refresh on this entity', ); - table.unique(['entity_ref', 'key']); - table.index('entity_ref', 'refresh_keys_entity_ref_idx'); + table.index('entity_id', 'refresh_keys_entity_id_idx'); table.index('key', 'refresh_keys_key_idx'); }); }; @@ -46,7 +45,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { await knex.schema.alterTable('refresh_keys', table => { - table.dropIndex([], 'refresh_keys_entity_ref_idx'); + table.dropIndex([], 'refresh_keys_entity_id_idx'); table.dropIndex([], 'refresh_keys_key_idx'); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index f9f51b7a0d..d280f3a493 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -522,7 +522,7 @@ describe('Default Processing Database', () => { ); const refreshKeys = await knex('refresh_keys') - .where({ entity_ref: stringifyEntityRef(processedEntity) }) + .where({ entity_id: id }) .select(); expect(refreshKeys[0]).toEqual({ diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 5b191dce32..f0779212be 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -79,6 +79,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, relations, deferredEntities, + refreshKeys, locationKey, } = options; const refreshResult = await tx('refresh_state') @@ -141,17 +142,19 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); + // Delete old refresh keys + await tx('refresh_keys') + .where({ entity_id: id }) + .delete(); + // Insert the refresh keys for the processed entity - await Promise.all( - options.refreshKeys.map(k => { - return tx('refresh_keys') - .insert({ - entity_ref: sourceEntityRef, - key: k.key, - }) - .onConflict(['entity_ref', 'key']) - .ignore(); - }), + await tx.batchInsert( + 'refresh_keys', + refreshKeys.map(k => ({ + entity_id: id, + key: k.key, + })), + BATCH_SIZE, ); return { @@ -540,18 +543,18 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { keys } = options; const updateResult = await tx('refresh_state') - .whereIn('entity_ref', function selectEntityRefs(tx2) { + .whereIn('entity_id', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) .select({ - entity_ref: 'refresh_keys.entity_ref', + entity_id: 'refresh_keys.entity_id', }) .from('refresh_keys'); }) .update({ next_update_at: tx.fn.now() }); if (updateResult === 0) { - throw new NotFoundError( + this.options.logger.info( `Failed to schedule ${JSON.stringify(keys)} for keys`, ); } diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 33c3ab0ef3..b9fe12be11 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -44,7 +44,7 @@ export type DbRefreshStateRow = { }; export type DbRefreshKeysRow = { - entity_ref: string; + entity_id: string; key: string; }; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index da7aa65714..8a839c0650 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -82,10 +82,6 @@ export type ReplaceUnprocessedEntitiesOptions = type: 'delta'; }; -export type RefreshKeyOptions = { - refreshKeys: RefreshKeyData[]; -}; - export type RefreshByKeyOptions = { keys: string[]; }; From 7c85fa3aa35e49a82ae9f7db826dacdbc255c556 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 15:23:27 +0200 Subject: [PATCH 26/64] use absolute urls Signed-off-by: Kiss Miklos --- .../src/modules/core/FileReaderProcessor.test.ts | 2 +- .../src/modules/core/FileReaderProcessor.ts | 5 +++-- .../src/modules/core/PlaceholderProcessor.test.ts | 2 +- .../src/modules/core/PlaceholderProcessor.ts | 12 +++++++++++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index 0c114f49a1..d726edc4db 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,7 +89,7 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].key).toContain('file:'); + expect(emit.mock.calls[1][0].key).toContain('file://'); expect(emit.mock.calls[1][0].key).toContain( 'fileReaderProcessor/component.yaml', ); diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 59795589b2..188bfd0031 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -52,6 +52,7 @@ export class FileReaderProcessor implements CatalogProcessor { if (fileMatches.length > 0) { for (const fileMatch of fileMatches) { const data = await fs.readFile(fileMatch); + const normalizedFilePath = path.normalize(fileMatch); // The normalize converts to native slashes; the glob library returns // forward slashes even on windows @@ -59,13 +60,13 @@ export class FileReaderProcessor implements CatalogProcessor { data: data, location: { type: LOCATION_TYPE, - target: path.normalize(fileMatch), + target: normalizedFilePath, }, })) { emit(parseResult); emit( processingResult.refresh( - `${LOCATION_TYPE}:${path.normalize(fileMatch)}`, + `${LOCATION_TYPE}://${normalizedFilePath}`, ), ); } diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index aa448dd955..02f30c9e69 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -385,7 +385,7 @@ describe('PlaceholderProcessor', () => { console.log(emitted); expect(emitted[0]).toEqual({ type: 'refresh', - key: 'url:./path-to-file.json', + key: 'url:http://example.com/path-to-file.json', }); }); }); diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 8a1891fe0c..bcd474016b 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -134,7 +134,17 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit(processingResult.refresh(`url:${resolverValue}`)); + emit( + processingResult.refresh( + `url:${relativeUrl({ + key: resolverKey, + value: resolverValue, + baseUrl: location.target, + read, + resolveUrl, + })}`, + ), + ); return [ await resolver({ From 990d448055396a84e5f2f67e428c001f791020c0 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:21:21 +0200 Subject: [PATCH 27/64] fix tests Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.test.ts | 2 +- .../src/modules/core/PlaceholderProcessor.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index d280f3a493..6a0986b345 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -526,7 +526,7 @@ describe('Default Processing Database', () => { .select(); expect(refreshKeys[0]).toEqual({ - entity_ref: 'location:default/fakelocation', + entity_id: id, key: 'protocol:foo-bar.com', }); }, diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index 02f30c9e69..c8a5a5355a 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -382,7 +382,6 @@ describe('PlaceholderProcessor', () => { { type: 'fake', target: 'http://example.com' }, result => emitted.push(result), ); - console.log(emitted); expect(emitted[0]).toEqual({ type: 'refresh', key: 'url:http://example.com/path-to-file.json', From 04f67153877155023d4fc6528f6f683754bf8808 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:22:47 +0200 Subject: [PATCH 28/64] remove log based on condition Signed-off-by: Kiss Miklos --- .../src/database/DefaultProcessingDatabase.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index f0779212be..903dd59e9e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -542,7 +542,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const { keys } = options; - const updateResult = await tx('refresh_state') + await tx('refresh_state') .whereIn('entity_id', function selectEntityRefs(tx2) { tx2 .whereIn('key', keys) @@ -552,12 +552,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .from('refresh_keys'); }) .update({ next_update_at: tx.fn.now() }); - - if (updateResult === 0) { - this.options.logger.info( - `Failed to schedule ${JSON.stringify(keys)} for keys`, - ); - } } async transaction(fn: (tx: Transaction) => Promise): Promise { From d2bd28696d749f12c7998470a7b9e6ae865a5a43 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Tue, 5 Jul 2022 18:33:53 +0200 Subject: [PATCH 29/64] add refreshKeys to the hashBuilder Signed-off-by: Kiss Miklos --- .../src/processing/DefaultCatalogProcessingEngine.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 10d6fc6975..82062a8e2c 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -134,6 +134,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { .update(stableStringify({ ...result.completedEntity })) .update(stableStringify([...result.deferredEntities])) .update(stableStringify([...result.relations])) + .update(stableStringify([...result.refreshKeys])) .update(stableStringify([...parents])); } From 735853353b0968d62251fea9679a46e5fdb729f1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Jul 2022 09:39:31 +0000 Subject: [PATCH 30/64] fix(deps): update dependency @octokit/webhooks to v10 Signed-off-by: Renovate Bot --- .changeset/renovate-e091137.md | 5 +++++ plugins/scaffolder-backend/package.json | 2 +- yarn.lock | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 .changeset/renovate-e091137.md diff --git a/.changeset/renovate-e091137.md b/.changeset/renovate-e091137.md new file mode 100644 index 0000000000..a5f53a1ec5 --- /dev/null +++ b/.changeset/renovate-e091137.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Updated dependency `@octokit/webhooks` to `^10.0.0`. diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index a70e60f714..a1bb552435 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -45,7 +45,7 @@ "@backstage/types": "^1.0.0", "@gitbeaker/core": "^35.6.0", "@gitbeaker/node": "^35.1.0", - "@octokit/webhooks": "^9.14.1", + "@octokit/webhooks": "^10.0.0", "@types/express": "^4.17.6", "azure-devops-node-api": "^11.0.1", "command-exists": "^1.2.9", diff --git a/yarn.lock b/yarn.lock index b03b5fdf07..8b7bc631e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5212,7 +5212,22 @@ resolved "https://registry.npmjs.org/@octokit/webhooks-types/-/webhooks-types-5.8.0.tgz#b76d1a3e3ad82cec5680d3c6c3443a620047a6ef" integrity sha512-8adktjIb76A7viIdayQSFuBEwOzwhDC+9yxZpKNHjfzrlostHCw0/N7JWpWMObfElwvJMk2fY2l1noENCk9wmw== -"@octokit/webhooks@^9.0.1", "@octokit/webhooks@^9.14.1": +"@octokit/webhooks-types@6.2.0": + version "6.2.0" + resolved "https://registry.npmjs.org/@octokit/webhooks-types/-/webhooks-types-6.2.0.tgz#ae84341dadd4a44d032f58b2dce7e3b766ef9da0" + integrity sha512-1YOSIrdb7WwLXHfCwa+7+VKcsZZ0sRsCiqvYvBwzstF4xg0qZ0w2K5Y8OkNvd7NtzKPP7KKgP9hSQZUccbPg0Q== + +"@octokit/webhooks@^10.0.0": + version "10.0.3" + resolved "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-10.0.3.tgz#13c20477b397731bff15e2578f6c86984bb41432" + integrity sha512-ormPrmocaqcZZdGOrLLlVH+FKJJY+I8fdUXzXRhiVpYCBctXHkfJ92MwNDq5vdQ4XeC2IacDhNEu1vfGfwGyFg== + dependencies: + "@octokit/request-error" "^2.0.2" + "@octokit/webhooks-methods" "^2.0.0" + "@octokit/webhooks-types" "6.2.0" + aggregate-error "^3.1.0" + +"@octokit/webhooks@^9.0.1": version "9.26.0" resolved "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-9.26.0.tgz#cf453bb313da3b66f1a90c84464d978e1c625cce" integrity sha512-foZlsgrTDwAmD5j2Czn6ji10lbWjGDVsUxTIydjG9KTkAWKJrFapXJgO5SbGxRwfPd3OJdhK3nA2YPqVhxLXqA== From a1e12ab502455e070576bd0217ad77e8e3aca838 Mon Sep 17 00:00:00 2001 From: Julius Berger Date: Wed, 6 Jul 2022 11:55:47 +0200 Subject: [PATCH 31/64] set default owner/repo when more than 1 allowed values defined Signed-off-by: Julius Berger --- .../src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index b36b332d9e..f0ef5ba4b5 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -89,12 +89,12 @@ export const RepoUrlPicker = ( /* we deal with calling the repo setting here instead of in each components for ease */ useEffect(() => { - if (allowedOwners.length === 1) { + if (allowedOwners.length > 0) { setState(prevState => ({ ...prevState, owner: allowedOwners[0] })); } }, [setState, allowedOwners]); useEffect(() => { - if (allowedRepos.length === 1) { + if (allowedRepos.length > 0) { setState(prevState => ({ ...prevState, repoName: allowedRepos[0] })); } }, [setState, allowedRepos]); From a35aae43b01127a19c08134e9048e777ccf55f37 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 11:58:43 +0200 Subject: [PATCH 32/64] remove // from refreshKeys Signed-off-by: Kiss Miklos --- .../src/modules/core/FileReaderProcessor.test.ts | 2 +- plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index d726edc4db..0c114f49a1 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -89,7 +89,7 @@ describe('FileReaderProcessor', () => { type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].key).toContain('file://'); + expect(emit.mock.calls[1][0].key).toContain('file:'); expect(emit.mock.calls[1][0].key).toContain( 'fileReaderProcessor/component.yaml', ); diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 188bfd0031..8ececed59f 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -66,7 +66,7 @@ export class FileReaderProcessor implements CatalogProcessor { emit(parseResult); emit( processingResult.refresh( - `${LOCATION_TYPE}://${normalizedFilePath}`, + `${LOCATION_TYPE}:${normalizedFilePath}`, ), ); } From d6ab0613a3797935022e73d883678e77a8569cbb Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 6 Jul 2022 13:46:10 +0200 Subject: [PATCH 33/64] Extend model so we can represent fact retrievers in the UI Signed-off-by: sblausten --- .../service/fact/factRetrievers/entityMetadataFactRetriever.ts | 3 +++ .../fact/factRetrievers/entityOwnershipFactRetriever.ts | 3 +++ .../src/service/fact/factRetrievers/techdocsFactRetriever.ts | 3 +++ 3 files changed, 9 insertions(+) diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts index caa19dbeaf..3103f9bdd5 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts @@ -30,6 +30,9 @@ import isEmpty from 'lodash/isEmpty'; export const entityMetadataFactRetriever: FactRetriever = { id: 'entityMetadataFactRetriever', version: '0.0.1', + title: 'Entity Metadata', + description: + 'Generates facts which indicate the completeness of entity metadata', schema: { hasTitle: { type: 'boolean', diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts index c1e9b2710e..974f1d30cd 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts @@ -29,6 +29,9 @@ import { Entity } from '@backstage/catalog-model'; export const entityOwnershipFactRetriever: FactRetriever = { id: 'entityOwnershipFactRetriever', version: '0.0.1', + title: 'Entity Ownership', + description: + 'Generates facts which indicate the quality of data in the spec.owner field', entityFilter: [ { kind: ['component', 'domain', 'system', 'api', 'resource', 'template'] }, ], diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts index 6bc84f4aa9..dc14298a55 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts @@ -34,6 +34,9 @@ const techdocsAnnotationFactName = export const techdocsFactRetriever: FactRetriever = { id: 'techdocsFactRetriever', version: '0.0.1', + title: 'Tech Docs', + description: + 'Generates facts related to the completeness of techdocs configuration for entities', schema: { [techdocsAnnotationFactName]: { type: 'boolean', From bcc122c46d34f4340c2f4e628900451472d2c00a Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 6 Jul 2022 13:59:42 +0200 Subject: [PATCH 34/64] Api report and changelog Signed-off-by: sblausten --- .changeset/rich-steaks-juggle.md | 9 +++++++++ plugins/tech-insights-node/api-report.md | 2 ++ plugins/tech-insights-node/src/facts.ts | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 .changeset/rich-steaks-juggle.md diff --git a/.changeset/rich-steaks-juggle.md b/.changeset/rich-steaks-juggle.md new file mode 100644 index 0000000000..291e9dac2b --- /dev/null +++ b/.changeset/rich-steaks-juggle.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-tech-insights-node': minor +'@backstage/plugin-tech-insights-backend': patch +--- + +**Breaking**: The FactRetriever model is extended by adding required title and description fields. This allows us to +display Fact Retrievers in the UI in future. + +If you have existing custom `FactRetriever` implementations hardcoded, you'll need to add a `title` and `description` to them. diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md index 09d409fecd..060158a619 100644 --- a/plugins/tech-insights-node/api-report.md +++ b/plugins/tech-insights-node/api-report.md @@ -46,12 +46,14 @@ export type FactLifecycle = TTL | MaxItems; // @public export interface FactRetriever { + description: string; entityFilter?: | Record[] | Record; handler: (ctx: FactRetrieverContext) => Promise; id: string; schema: FactSchema; + title: string; version: string; } diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts index bcc2aa42e1..6905a12f9c 100644 --- a/plugins/tech-insights-node/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -174,6 +174,16 @@ export interface FactRetriever { */ version: string; + /** + * A short display title for the fact retriever to be used in the interface + */ + title: string; + + /** + * A short display description for the fact retriever to be used in the interface. + */ + description: string; + /** * Handler function that needs to be implemented to retrieve fact values for entities. * From 54f4eba5611386c2300d225e625587a79e7f3aa1 Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 6 Jul 2022 16:09:27 +0200 Subject: [PATCH 35/64] Refactor Signed-off-by: sblausten --- .changeset/rich-steaks-juggle.md | 7 +- plugins/tech-insights-backend/api-report.md | 2 +- .../service/fact/FactRetrieverEngine.test.ts | 4 +- .../src/service/fact/FactRetrieverEngine.ts | 2 +- .../src/service/fact/FactRetrieverRegistry.ts | 2 +- .../src/service/fact/createFactRetriever.ts | 4 +- .../entityMetadataFactRetriever.ts | 6 +- .../entityOwnershipFactRetriever.ts | 6 +- .../factRetrievers/techdocsFactRetriever.ts | 6 +- plugins/tech-insights-common/api-report.md | 16 +++ plugins/tech-insights-common/package.json | 3 +- plugins/tech-insights-common/src/index.ts | 62 +++++++++ plugins/tech-insights-node/api-report.md | 14 +- plugins/tech-insights-node/src/facts.ts | 58 +------- yarn.lock | 126 ++++++++++++++++++ 15 files changed, 226 insertions(+), 92 deletions(-) diff --git a/.changeset/rich-steaks-juggle.md b/.changeset/rich-steaks-juggle.md index 291e9dac2b..b873efd59e 100644 --- a/.changeset/rich-steaks-juggle.md +++ b/.changeset/rich-steaks-juggle.md @@ -1,9 +1,10 @@ --- -'@backstage/plugin-tech-insights-node': minor -'@backstage/plugin-tech-insights-backend': patch +'@backstage/plugin-tech-insights-node': patch +'@backstage/plugin-tech-insights-backend': minor +'@backstage/plugin-tech-insights-common': minor --- -**Breaking**: The FactRetriever model is extended by adding required title and description fields. This allows us to +**Breaking**: The FactRetriever model is extended by adding a title and description fields and moved to the common package. This allows us to display Fact Retrievers in the UI in future. If you have existing custom `FactRetriever` implementations hardcoded, you'll need to add a `title` and `description` to them. diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md index 32d45f8afd..616d0f2c79 100644 --- a/plugins/tech-insights-backend/api-report.md +++ b/plugins/tech-insights-backend/api-report.md @@ -9,7 +9,7 @@ import express from 'express'; import { FactChecker } from '@backstage/plugin-tech-insights-node'; import { FactCheckerFactory } from '@backstage/plugin-tech-insights-node'; import { FactLifecycle } from '@backstage/plugin-tech-insights-node'; -import { FactRetriever } from '@backstage/plugin-tech-insights-node'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node'; import { FactSchema } from '@backstage/plugin-tech-insights-node'; import { Logger } from 'winston'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index 454711171f..34b816479d 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -14,7 +14,6 @@ * limitations under the License. */ import { - FactRetriever, FactRetrieverRegistration, FactSchemaDefinition, TechInsightFact, @@ -30,12 +29,15 @@ import { import { ConfigReader } from '@backstage/config'; import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { TaskScheduler } from '@backstage/backend-tasks'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; jest.useFakeTimers(); const testFactRetriever: FactRetriever = { id: 'test_factretriever', version: '0.0.1', + title: 'Test 1', + description: 'testing', entityFilter: [{ kind: 'component' }], schema: { testnumberfact: { diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts index 25e7610809..1d37e4d857 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -15,7 +15,6 @@ */ import { FactLifecycle, - FactRetriever, FactRetrieverContext, FactRetrieverRegistration, TechInsightFact, @@ -25,6 +24,7 @@ import { FactRetrieverRegistry } from './FactRetrieverRegistry'; import { Logger } from 'winston'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Duration } from 'luxon'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; function randomDailyCron() { const rand = (min: number, max: number) => diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts index 504d561f17..17139f0992 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -15,11 +15,11 @@ */ import { - FactRetriever, FactRetrieverRegistration, FactSchema, } from '@backstage/plugin-tech-insights-node'; import { ConflictError, NotFoundError } from '@backstage/errors'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * @public diff --git a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts index de49888e1b..bd4960d10f 100644 --- a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts @@ -15,15 +15,15 @@ */ import { FactLifecycle, - FactRetriever, FactRetrieverRegistration, } from '@backstage/plugin-tech-insights-node'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * @public * * @param cadence - cron expression to indicate when the fact retriever should be triggered - * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler + * @param factRetriever - Implementation of fact retriever definition * @param lifecycle - Optional lifecycle definition indicating the cleanup logic of facts when this retriever is run * */ diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts index 3103f9bdd5..7bddb4c0c6 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts @@ -14,13 +14,11 @@ * limitations under the License. */ -import { - FactRetriever, - FactRetrieverContext, -} from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import isEmpty from 'lodash/isEmpty'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * Generates facts which indicate the completeness of entity metadata. diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts index 974f1d30cd..39fbd0dea7 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts @@ -14,12 +14,10 @@ * limitations under the License. */ -import { - FactRetriever, - FactRetrieverContext, -} from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * Generates facts which indicate the quality of data in the spec.owner field. diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts index dc14298a55..95b043cfc8 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts @@ -14,13 +14,11 @@ * limitations under the License. */ -import { - FactRetriever, - FactRetrieverContext, -} from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { entityHasAnnotation, generateAnnotationFactName } from './utils'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; const techdocsAnnotation = 'backstage.io/techdocs-ref'; const techdocsAnnotationFactName = diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index 0236c0619a..746d99f663 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -4,7 +4,10 @@ ```ts import { DateTime } from 'luxon'; +import { FactRetrieverContext } from '../../tech-insights-node'; +import { FactSchema } from '../../tech-insights-node'; import { JsonValue } from '@backstage/types'; +import { TechInsightFact } from '../../tech-insights-node'; // @public export interface BooleanCheckResult extends CheckResult { @@ -47,5 +50,18 @@ export type FactResponse = { }; }; +// @public +export interface FactRetriever { + description: string; + entityFilter?: + | Record[] + | Record; + handler: (ctx: FactRetrieverContext) => Promise; + id: string; + schema: FactSchema; + title: string; + version: string; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index 14ec147bb9..bc98cdb888 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -35,7 +35,8 @@ "dependencies": { "@types/luxon": "^2.0.5", "luxon": "^2.0.2", - "@backstage/types": "^1.0.0" + "@backstage/types": "^1.0.0", + "@backstage/plugin-tech-insights-node": "^0.3.1" }, "devDependencies": { "@backstage/cli": "^0.18.0-next.1" diff --git a/plugins/tech-insights-common/src/index.ts b/plugins/tech-insights-common/src/index.ts index 66af5c2f93..42abb3dd44 100644 --- a/plugins/tech-insights-common/src/index.ts +++ b/plugins/tech-insights-common/src/index.ts @@ -16,6 +16,11 @@ import { DateTime } from 'luxon'; import { JsonValue } from '@backstage/types'; +import { + FactRetrieverContext, + FactSchema, + TechInsightFact, +} from '@backstage/plugin-tech-insights-node'; /** * @public @@ -101,6 +106,63 @@ export type FactResponse = { }; }; +/** + * FactRetriever interface + * + * @public + */ +export interface FactRetriever { + /** + * A unique identifier of the retriever. + * Used to identify and store individual facts returned from this retriever + * and schemas defined by this retriever. + */ + id: string; + + /** + * Semver string indicating the version of this fact retriever + * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. + * + * Should be incremented on changes to returned data from the handler or if the schema changes. + */ + version: string; + + /** + * A short display title for the fact retriever to be used in the interface + */ + title: string; + + /** + * A short display description for the fact retriever to be used in the interface. + */ + description: string; + + /** + * Handler function that needs to be implemented to retrieve fact values for entities. + * + * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations + * @returns - A collection of TechInsightFacts grouped by entities. + */ + handler: (ctx: FactRetrieverContext) => Promise; + + /** + * A fact schema defining the shape of data returned from the handler method for each entity + */ + schema: FactSchema; + + /** + * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. + * If omitted, the retriever should apply to all entities. + * + * Should be defined for example: + * \{ field: 'kind', values: \['component'\] \} + * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} + */ + entityFilter?: + | Record[] + | Record; +} + /** * Generic CheckResult * diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md index 060158a619..0ca1e22787 100644 --- a/plugins/tech-insights-node/api-report.md +++ b/plugins/tech-insights-node/api-report.md @@ -8,6 +8,7 @@ import { Config } from '@backstage/config'; import { DateTime } from 'luxon'; import { Duration } from 'luxon'; import { DurationLike } from 'luxon'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; import { JsonValue } from '@backstage/types'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -44,19 +45,6 @@ export interface FactCheckerFactory< // @public export type FactLifecycle = TTL | MaxItems; -// @public -export interface FactRetriever { - description: string; - entityFilter?: - | Record[] - | Record; - handler: (ctx: FactRetrieverContext) => Promise; - id: string; - schema: FactSchema; - title: string; - version: string; -} - // @public export type FactRetrieverContext = { config: Config; diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts index 6905a12f9c..6c37616b0c 100644 --- a/plugins/tech-insights-node/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -21,6 +21,7 @@ import { TokenManager, } from '@backstage/backend-common'; import { Logger } from 'winston'; +import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * A container for facts. The shape of the fact records needs to correspond to the FactSchema with same `ref` value. @@ -153,63 +154,6 @@ export type FactRetrieverContext = { | Record; }; -/** - * FactRetriever interface - * - * @public - */ -export interface FactRetriever { - /** - * A unique identifier of the retriever. - * Used to identify and store individual facts returned from this retriever - * and schemas defined by this retriever. - */ - id: string; - - /** - * Semver string indicating the version of this fact retriever - * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. - * - * Should be incremented on changes to returned data from the handler or if the schema changes. - */ - version: string; - - /** - * A short display title for the fact retriever to be used in the interface - */ - title: string; - - /** - * A short display description for the fact retriever to be used in the interface. - */ - description: string; - - /** - * Handler function that needs to be implemented to retrieve fact values for entities. - * - * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations - * @returns - A collection of TechInsightFacts grouped by entities. - */ - handler: (ctx: FactRetrieverContext) => Promise; - - /** - * A fact schema defining the shape of data returned from the handler method for each entity - */ - schema: FactSchema; - - /** - * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. - * If omitted, the retriever should apply to all entities. - * - * Should be defined for example: - * \{ field: 'kind', values: \['component'\] \} - * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} - */ - entityFilter?: - | Record[] - | Record; -} - /** * A Luxon duration like object for time to live value * diff --git a/yarn.lock b/yarn.lock index f06a187ebf..8f557e9551 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1535,6 +1535,59 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@backstage/backend-common@^0.14.0": + version "0.14.0" + resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.14.0.tgz#da7537c7a58ce596db36be3bb584b9c199f58296" + integrity sha512-xiEOknaWtyfiWcL8+fkQKMfSnpQr1FsKFX6xVc74Lrgmd46VMW7pbhDQULQDaeCN+jhwTfxCcsemhtcJjimq+w== + dependencies: + "@backstage/cli-common" "^0.1.9" + "@backstage/config" "^1.0.1" + "@backstage/config-loader" "^1.1.2" + "@backstage/errors" "^1.0.0" + "@backstage/integration" "^1.2.1" + "@backstage/types" "^1.0.0" + "@google-cloud/storage" "^6.0.0" + "@keyv/redis" "^2.2.3" + "@manypkg/get-packages" "^1.1.3" + "@octokit/rest" "^18.5.3" + "@types/cors" "^2.8.6" + "@types/dockerode" "^3.3.0" + "@types/express" "^4.17.6" + "@types/luxon" "^2.0.4" + "@types/webpack-env" "^1.15.2" + archiver "^5.0.2" + aws-sdk "^2.840.0" + base64-stream "^1.0.0" + compression "^1.7.4" + concat-stream "^2.0.0" + cors "^2.8.5" + dockerode "^3.3.1" + express "^4.17.1" + express-promise-router "^4.1.0" + fs-extra "10.1.0" + git-url-parse "^11.6.0" + helmet "^5.0.2" + isomorphic-git "^1.8.0" + jose "^4.6.0" + keyv "^4.0.3" + keyv-memcache "^1.2.5" + knex "^1.0.2" + lodash "^4.17.21" + logform "^2.3.2" + luxon "^2.3.1" + minimatch "^5.0.0" + minimist "^1.2.5" + morgan "^1.10.0" + node-abort-controller "^3.0.1" + node-fetch "^2.6.7" + raw-body "^2.4.1" + selfsigned "^2.0.0" + stoppable "^1.1.0" + tar "^6.1.2" + unzipper "^0.10.11" + winston "^3.2.1" + yn "^4.0.0" + "@backstage/catalog-client@^1.0.3": version "1.0.3" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-1.0.3.tgz#cb91472ccc31df322f69e7e4e80939b3535660cd" @@ -1557,6 +1610,27 @@ lodash "^4.17.21" uuid "^8.0.0" +"@backstage/config-loader@^1.1.2": + version "1.1.2" + resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-1.1.2.tgz#72cb0d7b2647f5a646bb279360bc34732e06521f" + integrity sha512-c5ZO7xDJn609DBIsYAWGE5kgh+7SPYUmG2ADtVX9SbXaql3VCafGlhc2hAZQa/O12W04qi3GgwGg0bqSFmx5uw== + dependencies: + "@backstage/cli-common" "^0.1.9" + "@backstage/config" "^1.0.1" + "@backstage/errors" "^1.0.0" + "@backstage/types" "^1.0.0" + "@types/json-schema" "^7.0.6" + ajv "^8.10.0" + chokidar "^3.5.2" + fs-extra "10.1.0" + json-schema "^0.4.0" + json-schema-merge-allof "^0.8.1" + json-schema-traverse "^1.0.0" + node-fetch "^2.6.7" + typescript-json-schema "^0.53.0" + yaml "^1.9.2" + yup "^0.32.9" + "@backstage/core-components@^0.9.0", "@backstage/core-components@^0.9.5": version "0.9.5" resolved "https://registry.npmjs.org/@backstage/core-components/-/core-components-0.9.5.tgz#5a0b34867aaee0549bfa67b39a69c09588fa3c7a" @@ -1747,6 +1821,19 @@ qs "^6.9.4" react-use "^17.2.4" +"@backstage/plugin-tech-insights-node@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/plugin-tech-insights-node/-/plugin-tech-insights-node-0.3.1.tgz#d9bf40b07c57bac67b251ac8d53c0ad26ad2e651" + integrity sha512-okfIZIVUVdhejqmd9UHaVg42VukHPT0RIVeUblALXDxip9PND+W4hKgZTRs7/2Z/IWOxGyGLEwAkRG0QSLnysg== + dependencies: + "@backstage/backend-common" "^0.14.0" + "@backstage/config" "^1.0.1" + "@backstage/plugin-tech-insights-common" "^0.2.4" + "@backstage/types" "^1.0.0" + "@types/luxon" "^2.0.5" + luxon "^2.0.2" + winston "^3.2.1" + "@backstage/theme@^0.2.15", "@backstage/theme@^0.2.6", "@backstage/theme@^0.2.7", "@backstage/theme@^0.2.9": version "0.2.15" resolved "https://registry.npmjs.org/@backstage/theme/-/theme-0.2.15.tgz#478491c9bca9dca85d5af08767ba512eabcd3669" @@ -17052,6 +17139,26 @@ kleur@^4.0.3, kleur@^4.1.4: resolved "https://registry.npmjs.org/kleur/-/kleur-4.1.4.tgz#8c202987d7e577766d039a8cd461934c01cda04d" integrity sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA== +knex@^1.0.2: + version "1.0.7" + resolved "https://registry.npmjs.org/knex/-/knex-1.0.7.tgz#965f4490efc451b140aac4c5c6efa39fd877597b" + integrity sha512-89jxuRATt4qJMb9ZyyaKBy0pQ4d5h7eOFRqiNFnUvsgU+9WZ2eIaZKrAPG1+F3mgu5UloPUnkVE5Yo2sKZUs6Q== + dependencies: + colorette "2.0.16" + commander "^9.1.0" + debug "4.3.4" + escalade "^3.1.1" + esm "^3.2.25" + get-package-type "^0.1.0" + getopts "2.3.0" + interpret "^2.2.0" + lodash "^4.17.21" + pg-connection-string "2.5.0" + rechoir "^0.8.0" + resolve-from "^5.0.0" + tarn "^3.0.2" + tildify "2.0.0" + knex@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/knex/-/knex-2.1.0.tgz#9348aace3a08ff5be26eb1c8e838416ddf1aa216" @@ -20420,6 +20527,11 @@ path-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" +path-equal@1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/path-equal/-/path-equal-1.1.2.tgz#260e7c449c4c2022f68cc5fa6e617e892858250d" + integrity sha512-p5kxPPwCdbf5AdXzT1bUBJomhgBlEjRBavYNr1XUpMFIE4Hnf2roueCMXudZK5tnaAu1tTmp3GPzqwJK45IHEA== + path-equal@^1.1.2: version "1.2.2" resolved "https://registry.npmjs.org/path-equal/-/path-equal-1.2.2.tgz#fa2997f0a829de22ec8f5f86461ca5590d49b832" @@ -25218,6 +25330,20 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= +typescript-json-schema@^0.53.0: + version "0.53.1" + resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.53.1.tgz#9204547f3e145169b40928998366ff6d28b81d32" + integrity sha512-Hg+RnOKUd38MOzC0rDft03a8xvwO+gCcj1F77smw2tCoZYQpFoLtrXWBGdvCX+REliko5WYel2kux17HPFqjLQ== + dependencies: + "@types/json-schema" "^7.0.9" + "@types/node" "^16.9.2" + glob "^7.1.7" + path-equal "1.1.2" + safe-stable-stringify "^2.2.0" + ts-node "^10.2.1" + typescript "~4.6.0" + yargs "^17.1.1" + typescript-json-schema@^0.54.0: version "0.54.0" resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.54.0.tgz#b3fc42ad90df6a0f6ab57571ebc8b4d41125df4f" From 3e3d8282efcca45022abde7b3c4db585fe361e9d Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:14:14 +0200 Subject: [PATCH 36/64] pass emit to resolvers Signed-off-by: Kiss Miklos --- .../modules/core/PlaceholderProcessor.test.ts | 2 ++ .../src/modules/core/PlaceholderProcessor.ts | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index c8a5a5355a..9bfa9027d9 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -397,6 +397,7 @@ describe('yamlPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { @@ -442,6 +443,7 @@ describe('jsonPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index bcd474016b..be2e9f9f5b 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -42,6 +42,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; /** @public */ @@ -134,18 +135,6 @@ export class PlaceholderProcessor implements CatalogProcessor { base, }); - emit( - processingResult.refresh( - `url:${relativeUrl({ - key: resolverKey, - value: resolverValue, - baseUrl: location.target, - read, - resolveUrl, - })}`, - ), - ); - return [ await resolver({ key: resolverKey, @@ -153,6 +142,7 @@ export class PlaceholderProcessor implements CatalogProcessor { baseUrl: location.target, read, resolveUrl, + emit, }), true, ]; @@ -170,11 +160,13 @@ export class PlaceholderProcessor implements CatalogProcessor { export async function yamlPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); let documents: yaml.Document.Parsed[]; try { - documents = yaml.parseAllDocuments(text).filter(d => d); + documents = yaml.parseAllDocuments(content).filter(d => d); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`, @@ -201,10 +193,12 @@ export async function yamlPlaceholderResolver( export async function jsonPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); try { - return JSON.parse(text); + return JSON.parse(content); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`, @@ -215,7 +209,11 @@ export async function jsonPlaceholderResolver( export async function textPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - return await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + + return content; } /* @@ -224,12 +222,12 @@ export async function textPlaceholderResolver( async function readTextLocation( params: PlaceholderResolverParams, -): Promise { +): Promise<{ content: string; url: string }> { const newUrl = relativeUrl(params); try { const data = await params.read(newUrl); - return data.toString('utf-8'); + return { content: data.toString('utf-8'), url: newUrl }; } catch (e) { throw new Error( `Placeholder \$${params.key} could not read location ${params.value}, ${e}`, From b309638d7af5917d877db646642fecc792e72f42 Mon Sep 17 00:00:00 2001 From: Cory Fisher Date: Wed, 6 Jul 2022 10:16:20 -0400 Subject: [PATCH 37/64] Fix patch version release typo. Signed-off-by: Cory Fisher --- docs/overview/versioning-policy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/overview/versioning-policy.md b/docs/overview/versioning-policy.md index 282906c580..885a599ed8 100644 --- a/docs/overview/versioning-policy.md +++ b/docs/overview/versioning-policy.md @@ -49,7 +49,7 @@ functionality, breaking changes, and bug fixes, according the [versioning policy](#release-versioning-policy). Patch versions will only be released to address critical bug fixes. They are not -bound to the regular cadence and are instead releases whenever needed. +bound to the regular cadence and are instead released whenever needed. ## Next Release Line From 1dd6c22cc88a49aa8ff9544fe9ec8b9435979b96 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:29:24 +0200 Subject: [PATCH 38/64] add changeset Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/cold-coins-tickle.md diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md new file mode 100644 index 0000000000..7fab336352 --- /dev/null +++ b/.changeset/cold-coins-tickle.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key. + +The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute url of the catalog file as a refresh key. In the format of `:` +The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. From 9a54bb6d2c075c2e714b71c29de9bbf2bb926798 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:34:41 +0200 Subject: [PATCH 39/64] add emit to resolvers Signed-off-by: Kiss Miklos --- plugins/catalog-backend/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index c410f166cc..09ca4d8dfe 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -578,6 +578,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; // @public (undocumented) From f3caf2e4b2e6af82690e0a224b531f270669a354 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 6 Jul 2022 16:46:54 +0200 Subject: [PATCH 40/64] fix url spelling Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md index 7fab336352..80f0958daa 100644 --- a/.changeset/cold-coins-tickle.md +++ b/.changeset/cold-coins-tickle.md @@ -4,5 +4,5 @@ Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key. -The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute url of the catalog file as a refresh key. In the format of `:` +The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `:` The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. From 4f17e4eb5517cce765e59f61301dc92da33632bb Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 6 Jul 2022 17:13:42 +0200 Subject: [PATCH 41/64] Revert "Refactor" This reverts commit 54f4eba5611386c2300d225e625587a79e7f3aa1. Signed-off-by: sblausten --- .changeset/rich-steaks-juggle.md | 7 +- plugins/tech-insights-backend/api-report.md | 2 +- .../service/fact/FactRetrieverEngine.test.ts | 4 +- .../src/service/fact/FactRetrieverEngine.ts | 2 +- .../src/service/fact/FactRetrieverRegistry.ts | 2 +- .../src/service/fact/createFactRetriever.ts | 4 +- .../entityMetadataFactRetriever.ts | 6 +- .../entityOwnershipFactRetriever.ts | 6 +- .../factRetrievers/techdocsFactRetriever.ts | 6 +- plugins/tech-insights-common/api-report.md | 16 --- plugins/tech-insights-common/package.json | 3 +- plugins/tech-insights-common/src/index.ts | 62 --------- plugins/tech-insights-node/api-report.md | 14 +- plugins/tech-insights-node/src/facts.ts | 58 +++++++- yarn.lock | 126 ------------------ 15 files changed, 92 insertions(+), 226 deletions(-) diff --git a/.changeset/rich-steaks-juggle.md b/.changeset/rich-steaks-juggle.md index b873efd59e..291e9dac2b 100644 --- a/.changeset/rich-steaks-juggle.md +++ b/.changeset/rich-steaks-juggle.md @@ -1,10 +1,9 @@ --- -'@backstage/plugin-tech-insights-node': patch -'@backstage/plugin-tech-insights-backend': minor -'@backstage/plugin-tech-insights-common': minor +'@backstage/plugin-tech-insights-node': minor +'@backstage/plugin-tech-insights-backend': patch --- -**Breaking**: The FactRetriever model is extended by adding a title and description fields and moved to the common package. This allows us to +**Breaking**: The FactRetriever model is extended by adding required title and description fields. This allows us to display Fact Retrievers in the UI in future. If you have existing custom `FactRetriever` implementations hardcoded, you'll need to add a `title` and `description` to them. diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md index 616d0f2c79..32d45f8afd 100644 --- a/plugins/tech-insights-backend/api-report.md +++ b/plugins/tech-insights-backend/api-report.md @@ -9,7 +9,7 @@ import express from 'express'; import { FactChecker } from '@backstage/plugin-tech-insights-node'; import { FactCheckerFactory } from '@backstage/plugin-tech-insights-node'; import { FactLifecycle } from '@backstage/plugin-tech-insights-node'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; +import { FactRetriever } from '@backstage/plugin-tech-insights-node'; import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node'; import { FactSchema } from '@backstage/plugin-tech-insights-node'; import { Logger } from 'winston'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index 34b816479d..454711171f 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { + FactRetriever, FactRetrieverRegistration, FactSchemaDefinition, TechInsightFact, @@ -29,15 +30,12 @@ import { import { ConfigReader } from '@backstage/config'; import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { TaskScheduler } from '@backstage/backend-tasks'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; jest.useFakeTimers(); const testFactRetriever: FactRetriever = { id: 'test_factretriever', version: '0.0.1', - title: 'Test 1', - description: 'testing', entityFilter: [{ kind: 'component' }], schema: { testnumberfact: { diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts index 1d37e4d857..25e7610809 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -15,6 +15,7 @@ */ import { FactLifecycle, + FactRetriever, FactRetrieverContext, FactRetrieverRegistration, TechInsightFact, @@ -24,7 +25,6 @@ import { FactRetrieverRegistry } from './FactRetrieverRegistry'; import { Logger } from 'winston'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Duration } from 'luxon'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; function randomDailyCron() { const rand = (min: number, max: number) => diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts index 17139f0992..504d561f17 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -15,11 +15,11 @@ */ import { + FactRetriever, FactRetrieverRegistration, FactSchema, } from '@backstage/plugin-tech-insights-node'; import { ConflictError, NotFoundError } from '@backstage/errors'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * @public diff --git a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts index bd4960d10f..de49888e1b 100644 --- a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts @@ -15,15 +15,15 @@ */ import { FactLifecycle, + FactRetriever, FactRetrieverRegistration, } from '@backstage/plugin-tech-insights-node'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * @public * * @param cadence - cron expression to indicate when the fact retriever should be triggered - * @param factRetriever - Implementation of fact retriever definition + * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler * @param lifecycle - Optional lifecycle definition indicating the cleanup logic of facts when this retriever is run * */ diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts index 7bddb4c0c6..3103f9bdd5 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityMetadataFactRetriever.ts @@ -14,11 +14,13 @@ * limitations under the License. */ -import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; +import { + FactRetriever, + FactRetrieverContext, +} from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import isEmpty from 'lodash/isEmpty'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * Generates facts which indicate the completeness of entity metadata. diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts index 39fbd0dea7..974f1d30cd 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/entityOwnershipFactRetriever.ts @@ -14,10 +14,12 @@ * limitations under the License. */ -import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; +import { + FactRetriever, + FactRetrieverContext, +} from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * Generates facts which indicate the quality of data in the spec.owner field. diff --git a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts index 95b043cfc8..dc14298a55 100644 --- a/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/factRetrievers/techdocsFactRetriever.ts @@ -14,11 +14,13 @@ * limitations under the License. */ -import { FactRetrieverContext } from '@backstage/plugin-tech-insights-node'; +import { + FactRetriever, + FactRetrieverContext, +} from '@backstage/plugin-tech-insights-node'; import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { entityHasAnnotation, generateAnnotationFactName } from './utils'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; const techdocsAnnotation = 'backstage.io/techdocs-ref'; const techdocsAnnotationFactName = diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index 746d99f663..0236c0619a 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -4,10 +4,7 @@ ```ts import { DateTime } from 'luxon'; -import { FactRetrieverContext } from '../../tech-insights-node'; -import { FactSchema } from '../../tech-insights-node'; import { JsonValue } from '@backstage/types'; -import { TechInsightFact } from '../../tech-insights-node'; // @public export interface BooleanCheckResult extends CheckResult { @@ -50,18 +47,5 @@ export type FactResponse = { }; }; -// @public -export interface FactRetriever { - description: string; - entityFilter?: - | Record[] - | Record; - handler: (ctx: FactRetrieverContext) => Promise; - id: string; - schema: FactSchema; - title: string; - version: string; -} - // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index bc98cdb888..14ec147bb9 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -35,8 +35,7 @@ "dependencies": { "@types/luxon": "^2.0.5", "luxon": "^2.0.2", - "@backstage/types": "^1.0.0", - "@backstage/plugin-tech-insights-node": "^0.3.1" + "@backstage/types": "^1.0.0" }, "devDependencies": { "@backstage/cli": "^0.18.0-next.1" diff --git a/plugins/tech-insights-common/src/index.ts b/plugins/tech-insights-common/src/index.ts index 42abb3dd44..66af5c2f93 100644 --- a/plugins/tech-insights-common/src/index.ts +++ b/plugins/tech-insights-common/src/index.ts @@ -16,11 +16,6 @@ import { DateTime } from 'luxon'; import { JsonValue } from '@backstage/types'; -import { - FactRetrieverContext, - FactSchema, - TechInsightFact, -} from '@backstage/plugin-tech-insights-node'; /** * @public @@ -106,63 +101,6 @@ export type FactResponse = { }; }; -/** - * FactRetriever interface - * - * @public - */ -export interface FactRetriever { - /** - * A unique identifier of the retriever. - * Used to identify and store individual facts returned from this retriever - * and schemas defined by this retriever. - */ - id: string; - - /** - * Semver string indicating the version of this fact retriever - * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. - * - * Should be incremented on changes to returned data from the handler or if the schema changes. - */ - version: string; - - /** - * A short display title for the fact retriever to be used in the interface - */ - title: string; - - /** - * A short display description for the fact retriever to be used in the interface. - */ - description: string; - - /** - * Handler function that needs to be implemented to retrieve fact values for entities. - * - * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations - * @returns - A collection of TechInsightFacts grouped by entities. - */ - handler: (ctx: FactRetrieverContext) => Promise; - - /** - * A fact schema defining the shape of data returned from the handler method for each entity - */ - schema: FactSchema; - - /** - * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. - * If omitted, the retriever should apply to all entities. - * - * Should be defined for example: - * \{ field: 'kind', values: \['component'\] \} - * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} - */ - entityFilter?: - | Record[] - | Record; -} - /** * Generic CheckResult * diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md index 0ca1e22787..060158a619 100644 --- a/plugins/tech-insights-node/api-report.md +++ b/plugins/tech-insights-node/api-report.md @@ -8,7 +8,6 @@ import { Config } from '@backstage/config'; import { DateTime } from 'luxon'; import { Duration } from 'luxon'; import { DurationLike } from 'luxon'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; import { JsonValue } from '@backstage/types'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -45,6 +44,19 @@ export interface FactCheckerFactory< // @public export type FactLifecycle = TTL | MaxItems; +// @public +export interface FactRetriever { + description: string; + entityFilter?: + | Record[] + | Record; + handler: (ctx: FactRetrieverContext) => Promise; + id: string; + schema: FactSchema; + title: string; + version: string; +} + // @public export type FactRetrieverContext = { config: Config; diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts index 6c37616b0c..6905a12f9c 100644 --- a/plugins/tech-insights-node/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -21,7 +21,6 @@ import { TokenManager, } from '@backstage/backend-common'; import { Logger } from 'winston'; -import { FactRetriever } from '@backstage/plugin-tech-insights-common'; /** * A container for facts. The shape of the fact records needs to correspond to the FactSchema with same `ref` value. @@ -154,6 +153,63 @@ export type FactRetrieverContext = { | Record; }; +/** + * FactRetriever interface + * + * @public + */ +export interface FactRetriever { + /** + * A unique identifier of the retriever. + * Used to identify and store individual facts returned from this retriever + * and schemas defined by this retriever. + */ + id: string; + + /** + * Semver string indicating the version of this fact retriever + * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. + * + * Should be incremented on changes to returned data from the handler or if the schema changes. + */ + version: string; + + /** + * A short display title for the fact retriever to be used in the interface + */ + title: string; + + /** + * A short display description for the fact retriever to be used in the interface. + */ + description: string; + + /** + * Handler function that needs to be implemented to retrieve fact values for entities. + * + * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations + * @returns - A collection of TechInsightFacts grouped by entities. + */ + handler: (ctx: FactRetrieverContext) => Promise; + + /** + * A fact schema defining the shape of data returned from the handler method for each entity + */ + schema: FactSchema; + + /** + * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. + * If omitted, the retriever should apply to all entities. + * + * Should be defined for example: + * \{ field: 'kind', values: \['component'\] \} + * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} + */ + entityFilter?: + | Record[] + | Record; +} + /** * A Luxon duration like object for time to live value * diff --git a/yarn.lock b/yarn.lock index 8f557e9551..f06a187ebf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1535,59 +1535,6 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" -"@backstage/backend-common@^0.14.0": - version "0.14.0" - resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.14.0.tgz#da7537c7a58ce596db36be3bb584b9c199f58296" - integrity sha512-xiEOknaWtyfiWcL8+fkQKMfSnpQr1FsKFX6xVc74Lrgmd46VMW7pbhDQULQDaeCN+jhwTfxCcsemhtcJjimq+w== - dependencies: - "@backstage/cli-common" "^0.1.9" - "@backstage/config" "^1.0.1" - "@backstage/config-loader" "^1.1.2" - "@backstage/errors" "^1.0.0" - "@backstage/integration" "^1.2.1" - "@backstage/types" "^1.0.0" - "@google-cloud/storage" "^6.0.0" - "@keyv/redis" "^2.2.3" - "@manypkg/get-packages" "^1.1.3" - "@octokit/rest" "^18.5.3" - "@types/cors" "^2.8.6" - "@types/dockerode" "^3.3.0" - "@types/express" "^4.17.6" - "@types/luxon" "^2.0.4" - "@types/webpack-env" "^1.15.2" - archiver "^5.0.2" - aws-sdk "^2.840.0" - base64-stream "^1.0.0" - compression "^1.7.4" - concat-stream "^2.0.0" - cors "^2.8.5" - dockerode "^3.3.1" - express "^4.17.1" - express-promise-router "^4.1.0" - fs-extra "10.1.0" - git-url-parse "^11.6.0" - helmet "^5.0.2" - isomorphic-git "^1.8.0" - jose "^4.6.0" - keyv "^4.0.3" - keyv-memcache "^1.2.5" - knex "^1.0.2" - lodash "^4.17.21" - logform "^2.3.2" - luxon "^2.3.1" - minimatch "^5.0.0" - minimist "^1.2.5" - morgan "^1.10.0" - node-abort-controller "^3.0.1" - node-fetch "^2.6.7" - raw-body "^2.4.1" - selfsigned "^2.0.0" - stoppable "^1.1.0" - tar "^6.1.2" - unzipper "^0.10.11" - winston "^3.2.1" - yn "^4.0.0" - "@backstage/catalog-client@^1.0.3": version "1.0.3" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-1.0.3.tgz#cb91472ccc31df322f69e7e4e80939b3535660cd" @@ -1610,27 +1557,6 @@ lodash "^4.17.21" uuid "^8.0.0" -"@backstage/config-loader@^1.1.2": - version "1.1.2" - resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-1.1.2.tgz#72cb0d7b2647f5a646bb279360bc34732e06521f" - integrity sha512-c5ZO7xDJn609DBIsYAWGE5kgh+7SPYUmG2ADtVX9SbXaql3VCafGlhc2hAZQa/O12W04qi3GgwGg0bqSFmx5uw== - dependencies: - "@backstage/cli-common" "^0.1.9" - "@backstage/config" "^1.0.1" - "@backstage/errors" "^1.0.0" - "@backstage/types" "^1.0.0" - "@types/json-schema" "^7.0.6" - ajv "^8.10.0" - chokidar "^3.5.2" - fs-extra "10.1.0" - json-schema "^0.4.0" - json-schema-merge-allof "^0.8.1" - json-schema-traverse "^1.0.0" - node-fetch "^2.6.7" - typescript-json-schema "^0.53.0" - yaml "^1.9.2" - yup "^0.32.9" - "@backstage/core-components@^0.9.0", "@backstage/core-components@^0.9.5": version "0.9.5" resolved "https://registry.npmjs.org/@backstage/core-components/-/core-components-0.9.5.tgz#5a0b34867aaee0549bfa67b39a69c09588fa3c7a" @@ -1821,19 +1747,6 @@ qs "^6.9.4" react-use "^17.2.4" -"@backstage/plugin-tech-insights-node@^0.3.1": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/plugin-tech-insights-node/-/plugin-tech-insights-node-0.3.1.tgz#d9bf40b07c57bac67b251ac8d53c0ad26ad2e651" - integrity sha512-okfIZIVUVdhejqmd9UHaVg42VukHPT0RIVeUblALXDxip9PND+W4hKgZTRs7/2Z/IWOxGyGLEwAkRG0QSLnysg== - dependencies: - "@backstage/backend-common" "^0.14.0" - "@backstage/config" "^1.0.1" - "@backstage/plugin-tech-insights-common" "^0.2.4" - "@backstage/types" "^1.0.0" - "@types/luxon" "^2.0.5" - luxon "^2.0.2" - winston "^3.2.1" - "@backstage/theme@^0.2.15", "@backstage/theme@^0.2.6", "@backstage/theme@^0.2.7", "@backstage/theme@^0.2.9": version "0.2.15" resolved "https://registry.npmjs.org/@backstage/theme/-/theme-0.2.15.tgz#478491c9bca9dca85d5af08767ba512eabcd3669" @@ -17139,26 +17052,6 @@ kleur@^4.0.3, kleur@^4.1.4: resolved "https://registry.npmjs.org/kleur/-/kleur-4.1.4.tgz#8c202987d7e577766d039a8cd461934c01cda04d" integrity sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA== -knex@^1.0.2: - version "1.0.7" - resolved "https://registry.npmjs.org/knex/-/knex-1.0.7.tgz#965f4490efc451b140aac4c5c6efa39fd877597b" - integrity sha512-89jxuRATt4qJMb9ZyyaKBy0pQ4d5h7eOFRqiNFnUvsgU+9WZ2eIaZKrAPG1+F3mgu5UloPUnkVE5Yo2sKZUs6Q== - dependencies: - colorette "2.0.16" - commander "^9.1.0" - debug "4.3.4" - escalade "^3.1.1" - esm "^3.2.25" - get-package-type "^0.1.0" - getopts "2.3.0" - interpret "^2.2.0" - lodash "^4.17.21" - pg-connection-string "2.5.0" - rechoir "^0.8.0" - resolve-from "^5.0.0" - tarn "^3.0.2" - tildify "2.0.0" - knex@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/knex/-/knex-2.1.0.tgz#9348aace3a08ff5be26eb1c8e838416ddf1aa216" @@ -20527,11 +20420,6 @@ path-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" -path-equal@1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/path-equal/-/path-equal-1.1.2.tgz#260e7c449c4c2022f68cc5fa6e617e892858250d" - integrity sha512-p5kxPPwCdbf5AdXzT1bUBJomhgBlEjRBavYNr1XUpMFIE4Hnf2roueCMXudZK5tnaAu1tTmp3GPzqwJK45IHEA== - path-equal@^1.1.2: version "1.2.2" resolved "https://registry.npmjs.org/path-equal/-/path-equal-1.2.2.tgz#fa2997f0a829de22ec8f5f86461ca5590d49b832" @@ -25330,20 +25218,6 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript-json-schema@^0.53.0: - version "0.53.1" - resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.53.1.tgz#9204547f3e145169b40928998366ff6d28b81d32" - integrity sha512-Hg+RnOKUd38MOzC0rDft03a8xvwO+gCcj1F77smw2tCoZYQpFoLtrXWBGdvCX+REliko5WYel2kux17HPFqjLQ== - dependencies: - "@types/json-schema" "^7.0.9" - "@types/node" "^16.9.2" - glob "^7.1.7" - path-equal "1.1.2" - safe-stable-stringify "^2.2.0" - ts-node "^10.2.1" - typescript "~4.6.0" - yargs "^17.1.1" - typescript-json-schema@^0.54.0: version "0.54.0" resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.54.0.tgz#b3fc42ad90df6a0f6ab57571ebc8b4d41125df4f" From 307ade749948966d7e01ffa6de3b78a95a091fd5 Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 6 Jul 2022 17:17:52 +0200 Subject: [PATCH 42/64] Fix test and make fields optional Signed-off-by: sblausten --- .changeset/rich-steaks-juggle.md | 8 +++----- .../src/service/fact/FactRetrieverEngine.test.ts | 2 ++ plugins/tech-insights-node/api-report.md | 4 ++-- plugins/tech-insights-node/src/facts.ts | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.changeset/rich-steaks-juggle.md b/.changeset/rich-steaks-juggle.md index 291e9dac2b..f47de1af0d 100644 --- a/.changeset/rich-steaks-juggle.md +++ b/.changeset/rich-steaks-juggle.md @@ -1,9 +1,7 @@ --- -'@backstage/plugin-tech-insights-node': minor +'@backstage/plugin-tech-insights-node': patch '@backstage/plugin-tech-insights-backend': patch --- -**Breaking**: The FactRetriever model is extended by adding required title and description fields. This allows us to -display Fact Retrievers in the UI in future. - -If you have existing custom `FactRetriever` implementations hardcoded, you'll need to add a `title` and `description` to them. +The FactRetriever model is extended by adding optional title and description fields. This allows us to display Fact +Retrievers in the UI in future. diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index 454711171f..f9b145c876 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -36,6 +36,8 @@ jest.useFakeTimers(); const testFactRetriever: FactRetriever = { id: 'test_factretriever', version: '0.0.1', + title: 'Test 1', + description: 'testing', entityFilter: [{ kind: 'component' }], schema: { testnumberfact: { diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md index 060158a619..84677bdcbc 100644 --- a/plugins/tech-insights-node/api-report.md +++ b/plugins/tech-insights-node/api-report.md @@ -46,14 +46,14 @@ export type FactLifecycle = TTL | MaxItems; // @public export interface FactRetriever { - description: string; + description?: string; entityFilter?: | Record[] | Record; handler: (ctx: FactRetrieverContext) => Promise; id: string; schema: FactSchema; - title: string; + title?: string; version: string; } diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts index 6905a12f9c..cda238694f 100644 --- a/plugins/tech-insights-node/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -177,12 +177,12 @@ export interface FactRetriever { /** * A short display title for the fact retriever to be used in the interface */ - title: string; + title?: string; /** * A short display description for the fact retriever to be used in the interface. */ - description: string; + description?: string; /** * Handler function that needs to be implemented to retrieve fact values for entities. From d600cb2ab60e601bca2230be45f5cfbf1ed89aa2 Mon Sep 17 00:00:00 2001 From: Jonathan Ash Date: Wed, 6 Jul 2022 16:27:09 +0100 Subject: [PATCH 43/64] contextMenu prop passed through to ScaffolderPage from Router Signed-off-by: Jonathan Ash --- .changeset/nervous-hounds-matter.md | 5 +++++ .../src/components/ScaffolderPage/ScaffolderPage.tsx | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 .changeset/nervous-hounds-matter.md diff --git a/.changeset/nervous-hounds-matter.md b/.changeset/nervous-hounds-matter.md new file mode 100644 index 0000000000..0f60b7ae2c --- /dev/null +++ b/.changeset/nervous-hounds-matter.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +contextMenu prop passed through to from the component diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 2bc36ff0d2..60a761d171 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -133,11 +133,13 @@ export const ScaffolderPageContents = ({ export const ScaffolderPage = ({ TemplateCardComponent, groups, + contextMenu, }: ScaffolderPageProps) => ( ); From 35233735e81fb58dbdf91200933a91a3f9f2074c Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 09:32:24 -0500 Subject: [PATCH 44/64] feat: [#12248] Switch to new okta strategy This new Okta passport strategy support custom authorization server IDs Signed-off-by: David Zemon --- plugins/auth-backend/package.json | 2 +- .../src/providers/okta/provider.ts | 4 +++ yarn.lock | 27 +++++++++++-------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index d6b027e6ea..5ef20a1ccf 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -40,6 +40,7 @@ "@backstage/config": "^1.0.1", "@backstage/errors": "^1.1.0-next.0", "@backstage/types": "^1.0.0", + "@davidzemon/passport-okta-oauth": "^0.0.4-2", "@google-cloud/firestore": "^5.0.2", "@types/express": "^4.17.6", "@types/passport": "^1.0.3", @@ -68,7 +69,6 @@ "passport-google-oauth20": "^2.0.0", "passport-microsoft": "^1.0.0", "passport-oauth2": "^1.6.1", - "passport-okta-oauth": "^0.0.1", "passport-onelogin-oauth": "^0.0.1", "passport-saml": "^3.1.2", "uuid": "^8.0.0", diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index de9d128641..b11df1291e 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -55,6 +55,7 @@ type PrivateInfo = { export type OktaAuthProviderOptions = OAuthProviderOptions & { audience: string; + authServerId?: string; signInResolver?: SignInResolver; authHandler: AuthHandler; resolverContext: AuthResolverContext; @@ -94,6 +95,7 @@ export class OktaAuthProvider implements OAuthHandlers { clientSecret: options.clientSecret, callbackURL: options.callbackUrl, audience: options.audience, + authServerID: options.authServerId, passReqToCallback: false, store: this.store, response_type: 'code', @@ -220,6 +222,7 @@ export const okta = createAuthProviderIntegration({ const clientId = envConfig.getString('clientId'); const clientSecret = envConfig.getString('clientSecret'); const audience = envConfig.getString('audience'); + const authServerId = envConfig.getString('authServerId'); const customCallbackUrl = envConfig.getOptionalString('callbackUrl'); const callbackUrl = customCallbackUrl || @@ -240,6 +243,7 @@ export const okta = createAuthProviderIntegration({ const provider = new OktaAuthProvider({ audience, + authServerId, clientId, clientSecret, callbackUrl, diff --git a/yarn.lock b/yarn.lock index f06a187ebf..474e2f2370 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2126,6 +2126,15 @@ dependencies: "@date-io/core" "^1.3.13" +"@davidzemon/passport-okta-oauth@^0.0.4-2": + version "0.0.4-2" + resolved "https://registry.npmjs.org/@davidzemon/passport-okta-oauth/-/passport-okta-oauth-0.0.4-2.tgz#67155fa1f591a30e65bb8b1c8a1433280ec515e2" + integrity sha512-RYyDc1w12LbpTfnTzxmVfyyvkVYYPHoH4TIUAozv27TY1VgY/QEpla2G7fidXDFlu/6itsK0s678s2idd2jhhg== + dependencies: + passport-oauth "^1.0.0" + pkginfo "^0.4.1" + uid2 "^1.0.0" + "@elastic/elasticsearch-mock@^1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@elastic/elasticsearch-mock/-/elasticsearch-mock-1.0.0.tgz#1fbaf6bf220cd74a34494ff3a8673551d9dfc66f" @@ -20349,7 +20358,7 @@ passport-oauth2@1.6.1, passport-oauth2@1.x.x, passport-oauth2@^1.1.2, passport-o uid2 "0.0.x" utils-merge "1.x.x" -passport-oauth@1.0.0: +passport-oauth@1.0.0, passport-oauth@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/passport-oauth/-/passport-oauth-1.0.0.tgz#90aff63387540f02089af28cdad39ea7f80d77df" integrity sha1-kK/2M4dUDwIImvKM2tOep/gNd98= @@ -20357,15 +20366,6 @@ passport-oauth@1.0.0: passport-oauth1 "1.x.x" passport-oauth2 "1.x.x" -passport-okta-oauth@^0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/passport-okta-oauth/-/passport-okta-oauth-0.0.1.tgz#c8bcee02af3d56ca79d3cca776f2df7cf15a5748" - integrity sha1-yLzuAq89Vsp508yndvLffPFaV0g= - dependencies: - passport-oauth "1.0.0" - pkginfo "0.2.x" - uid2 "0.0.3" - passport-onelogin-oauth@^0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/passport-onelogin-oauth/-/passport-onelogin-oauth-0.0.1.tgz#6e991ac6720783fdd80d4caa08c36cc71ef19962" @@ -20760,7 +20760,7 @@ pkginfo@0.2.x: resolved "https://registry.npmjs.org/pkginfo/-/pkginfo-0.2.3.tgz#7239c42a5ef6c30b8f328439d9b9ff71042490f8" integrity sha1-cjnEKl72wwuPMoQ52bn/cQQkkPg= -pkginfo@0.4.x: +pkginfo@0.4.x, pkginfo@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff" integrity sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8= @@ -25269,6 +25269,11 @@ uid2@0.0.3, uid2@0.0.x: resolved "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz#483126e11774df2f71b8b639dcd799c376162b82" integrity sha1-SDEm4Rd03y9xuLY53NeZw3YWK4I= +uid2@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/uid2/-/uid2-1.0.0.tgz#ef8d95a128d7c5c44defa1a3d052eecc17a06bfb" + integrity sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ== + unbox-primitive@^1.0.1, unbox-primitive@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" From 3e154238970fd946fe830255e4b8fc0fd02d5165 Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 09:48:49 -0500 Subject: [PATCH 45/64] docs: [#12248] Document new configuration options Signed-off-by: David Zemon --- docs/auth/okta/provider.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/auth/okta/provider.md b/docs/auth/okta/provider.md index 35394094f8..e4fc8bfb45 100644 --- a/docs/auth/okta/provider.md +++ b/docs/auth/okta/provider.md @@ -41,6 +41,8 @@ auth: clientId: ${AUTH_OKTA_CLIENT_ID} clientSecret: ${AUTH_OKTA_CLIENT_SECRET} audience: ${AUTH_OKTA_DOMAIN} + authServerId: ${AUTH_OKTA_AUTH_SERVER_ID} # Optional + idp: ${AUTH_OKTA_IDP} # Optional ``` The values referenced are found on the Application page on your Okta site. @@ -48,8 +50,10 @@ The values referenced are found on the Application page on your Okta site. - `clientId`: The client ID that you generated on Okta, e.g. `3abe134ejxzF21HU74c1` - `clientSecret`: The client secret shown for the Application. -- `audience`: The Okta domain shown for your Application, e.g. +- `audience`: The Okta domain shown for the Application, e.g. `https://company.okta.com` +- `authServerId`: The authorization server ID for the Application +- `idp`: The identity provider for the application, e.g. `0oaulob4BFVa4zQvt0g3` ## Adding the provider to the Backstage frontend From 3a014730dc6d20431fc8bd992d5668d0b558ebd2 Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 09:54:17 -0500 Subject: [PATCH 46/64] docs: [#12248] Add changeset Signed-off-by: David Zemon --- .changeset/spicy-walls-repair.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/spicy-walls-repair.md diff --git a/.changeset/spicy-walls-repair.md b/.changeset/spicy-walls-repair.md new file mode 100644 index 0000000000..26a84be044 --- /dev/null +++ b/.changeset/spicy-walls-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': minor +--- + +Add new config option for okta auth server and IDP From a5a78be5b8ff41dfb9a167d31e15982c23726826 Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 10:12:15 -0500 Subject: [PATCH 47/64] fix: [#12248] Fix bad import Signed-off-by: David Zemon --- plugins/auth-backend/src/providers/okta/provider.ts | 2 +- plugins/auth-backend/src/providers/okta/types.d.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index b11df1291e..5ee7a1b5c0 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -26,7 +26,7 @@ import { OAuthRefreshRequest, OAuthResult, } from '../../lib/oauth'; -import { Strategy as OktaStrategy } from 'passport-okta-oauth'; +import { Strategy as OktaStrategy } from '@davidzemon/passport-okta-oauth'; import passport from 'passport'; import { executeFrameHandlerStrategy, diff --git a/plugins/auth-backend/src/providers/okta/types.d.ts b/plugins/auth-backend/src/providers/okta/types.d.ts index ab83bf89ec..09ba3e5a7b 100644 --- a/plugins/auth-backend/src/providers/okta/types.d.ts +++ b/plugins/auth-backend/src/providers/okta/types.d.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -declare module 'passport-okta-oauth' { +declare module '@davidzemon/passport-okta-oauth' { export class Strategy { constructor(options: any, verify: any); } From 4443a5e9bebfaa8c597ab7c424b451ceadfd1a0b Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 10:34:51 -0500 Subject: [PATCH 48/64] fix: [#12248] Okta's auth server ID is optional, not required Signed-off-by: David Zemon --- plugins/auth-backend/src/providers/okta/provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index 5ee7a1b5c0..c050f7a428 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -222,7 +222,7 @@ export const okta = createAuthProviderIntegration({ const clientId = envConfig.getString('clientId'); const clientSecret = envConfig.getString('clientSecret'); const audience = envConfig.getString('audience'); - const authServerId = envConfig.getString('authServerId'); + const authServerId = envConfig.getOptionalString('authServerId'); const customCallbackUrl = envConfig.getOptionalString('callbackUrl'); const callbackUrl = customCallbackUrl || From 777bddd123e9bc2135ba1006fb230825a665531a Mon Sep 17 00:00:00 2001 From: David Zemon Date: Thu, 30 Jun 2022 10:35:21 -0500 Subject: [PATCH 49/64] feat: [#12248] Add Okta IDP option Signed-off-by: David Zemon --- plugins/auth-backend/src/providers/okta/provider.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/auth-backend/src/providers/okta/provider.ts b/plugins/auth-backend/src/providers/okta/provider.ts index c050f7a428..cb686f17e9 100644 --- a/plugins/auth-backend/src/providers/okta/provider.ts +++ b/plugins/auth-backend/src/providers/okta/provider.ts @@ -56,6 +56,7 @@ type PrivateInfo = { export type OktaAuthProviderOptions = OAuthProviderOptions & { audience: string; authServerId?: string; + idp?: string; signInResolver?: SignInResolver; authHandler: AuthHandler; resolverContext: AuthResolverContext; @@ -96,6 +97,7 @@ export class OktaAuthProvider implements OAuthHandlers { callbackURL: options.callbackUrl, audience: options.audience, authServerID: options.authServerId, + idp: options.idp, passReqToCallback: false, store: this.store, response_type: 'code', @@ -223,6 +225,7 @@ export const okta = createAuthProviderIntegration({ const clientSecret = envConfig.getString('clientSecret'); const audience = envConfig.getString('audience'); const authServerId = envConfig.getOptionalString('authServerId'); + const idp = envConfig.getOptionalString('idp'); const customCallbackUrl = envConfig.getOptionalString('callbackUrl'); const callbackUrl = customCallbackUrl || @@ -244,6 +247,7 @@ export const okta = createAuthProviderIntegration({ const provider = new OktaAuthProvider({ audience, authServerId, + idp, clientId, clientSecret, callbackUrl, From 58005b8bcdb407b5c92043ee5e0dcefa09d11033 Mon Sep 17 00:00:00 2001 From: David Zemon Date: Wed, 6 Jul 2022 12:08:16 -0500 Subject: [PATCH 50/64] feat: [#12248] Update to latest @davidzemon/passport-oauth-okta Signed-off-by: David Zemon --- .changeset/spicy-walls-repair.md | 2 +- plugins/auth-backend/package.json | 2 +- .../src/providers/okta/types.d.ts | 20 ----------------- yarn.lock | 22 ++++++++++++++----- 4 files changed, 18 insertions(+), 28 deletions(-) delete mode 100644 plugins/auth-backend/src/providers/okta/types.d.ts diff --git a/.changeset/spicy-walls-repair.md b/.changeset/spicy-walls-repair.md index 26a84be044..36187600c7 100644 --- a/.changeset/spicy-walls-repair.md +++ b/.changeset/spicy-walls-repair.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-auth-backend': minor +'@backstage/plugin-auth-backend': patch --- Add new config option for okta auth server and IDP diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 5ef20a1ccf..949a6a0f42 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -40,7 +40,7 @@ "@backstage/config": "^1.0.1", "@backstage/errors": "^1.1.0-next.0", "@backstage/types": "^1.0.0", - "@davidzemon/passport-okta-oauth": "^0.0.4-2", + "@davidzemon/passport-okta-oauth": "^0.0.4", "@google-cloud/firestore": "^5.0.2", "@types/express": "^4.17.6", "@types/passport": "^1.0.3", diff --git a/plugins/auth-backend/src/providers/okta/types.d.ts b/plugins/auth-backend/src/providers/okta/types.d.ts deleted file mode 100644 index 09ba3e5a7b..0000000000 --- a/plugins/auth-backend/src/providers/okta/types.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -declare module '@davidzemon/passport-okta-oauth' { - export class Strategy { - constructor(options: any, verify: any); - } -} diff --git a/yarn.lock b/yarn.lock index 474e2f2370..398c9b978c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2126,12 +2126,13 @@ dependencies: "@date-io/core" "^1.3.13" -"@davidzemon/passport-okta-oauth@^0.0.4-2": - version "0.0.4-2" - resolved "https://registry.npmjs.org/@davidzemon/passport-okta-oauth/-/passport-okta-oauth-0.0.4-2.tgz#67155fa1f591a30e65bb8b1c8a1433280ec515e2" - integrity sha512-RYyDc1w12LbpTfnTzxmVfyyvkVYYPHoH4TIUAozv27TY1VgY/QEpla2G7fidXDFlu/6itsK0s678s2idd2jhhg== +"@davidzemon/passport-okta-oauth@^0.0.4": + version "0.0.4" + resolved "https://registry.npmjs.org/@davidzemon/passport-okta-oauth/-/passport-okta-oauth-0.0.4.tgz#25cfdb8f4abee1038449d4c62ada9566b286b55d" + integrity sha512-LYGDtnYnmAvBDpSfdjzq0Ip6VJRlt3vRPECGrJDT6CQHNlt34UpeqQDvDSPPplSUmZkxJcS+mQk++l4Kqxkzaw== dependencies: - passport-oauth "^1.0.0" + "@types/passport-oauth2" "^1.4.11" + passport-oauth2 "^1.6.1" pkginfo "^0.4.1" uid2 "^1.0.0" @@ -6815,6 +6816,15 @@ "@types/oauth" "*" "@types/passport" "*" +"@types/passport-oauth2@^1.4.11": + version "1.4.11" + resolved "https://registry.npmjs.org/@types/passport-oauth2/-/passport-oauth2-1.4.11.tgz#fbca527ecb44258774d17bcb251630c321515fa9" + integrity sha512-KUNwmGhe/3xPbjkzkPwwcPmyFwfyiSgtV1qOrPBLaU4i4q9GSCdAOyCbkFG0gUxAyEmYwqo9OAF/rjPjJ6ImdA== + dependencies: + "@types/express" "*" + "@types/oauth" "*" + "@types/passport" "*" + "@types/passport-saml@^1.1.3": version "1.1.3" resolved "https://registry.npmjs.org/@types/passport-saml/-/passport-saml-1.1.3.tgz#efc57902a07ebe1ec114d00acd8d990e873813a1" @@ -20358,7 +20368,7 @@ passport-oauth2@1.6.1, passport-oauth2@1.x.x, passport-oauth2@^1.1.2, passport-o uid2 "0.0.x" utils-merge "1.x.x" -passport-oauth@1.0.0, passport-oauth@^1.0.0: +passport-oauth@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/passport-oauth/-/passport-oauth-1.0.0.tgz#90aff63387540f02089af28cdad39ea7f80d77df" integrity sha1-kK/2M4dUDwIImvKM2tOep/gNd98= From 8449c328e31ac7b0f6b8ae43d292e6b6a7d798e7 Mon Sep 17 00:00:00 2001 From: David Zemon Date: Wed, 6 Jul 2022 15:36:12 -0500 Subject: [PATCH 51/64] fix: [#12248] Use fixed version of @davidzemon/passport-oauth-okta Signed-off-by: David Zemon --- plugins/auth-backend/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 949a6a0f42..b1c2f39e27 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -40,7 +40,7 @@ "@backstage/config": "^1.0.1", "@backstage/errors": "^1.1.0-next.0", "@backstage/types": "^1.0.0", - "@davidzemon/passport-okta-oauth": "^0.0.4", + "@davidzemon/passport-okta-oauth": "^0.0.5", "@google-cloud/firestore": "^5.0.2", "@types/express": "^4.17.6", "@types/passport": "^1.0.3", diff --git a/yarn.lock b/yarn.lock index 398c9b978c..5b8110ec78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2126,10 +2126,10 @@ dependencies: "@date-io/core" "^1.3.13" -"@davidzemon/passport-okta-oauth@^0.0.4": - version "0.0.4" - resolved "https://registry.npmjs.org/@davidzemon/passport-okta-oauth/-/passport-okta-oauth-0.0.4.tgz#25cfdb8f4abee1038449d4c62ada9566b286b55d" - integrity sha512-LYGDtnYnmAvBDpSfdjzq0Ip6VJRlt3vRPECGrJDT6CQHNlt34UpeqQDvDSPPplSUmZkxJcS+mQk++l4Kqxkzaw== +"@davidzemon/passport-okta-oauth@^0.0.5": + version "0.0.5" + resolved "https://registry.npmjs.org/@davidzemon/passport-okta-oauth/-/passport-okta-oauth-0.0.5.tgz#905d14521f4e8b422f4074c1f7fe1cbe0c88e44b" + integrity sha512-eaC2Ve2MIoqR7dLKgpHxhVKRcfgJCes0Fozxm5SefZh/zqLNb8tGIou+dj0EbylksLmB+nVlhr8p8qwjA9n2sA== dependencies: "@types/passport-oauth2" "^1.4.11" passport-oauth2 "^1.6.1" From 2dbbb583c5d4a87a263b4f11805e796d1f8ed1f8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 6 Jul 2022 17:40:40 +0200 Subject: [PATCH 52/64] Update .changeset/rich-steaks-juggle.md Signed-off-by: sblausten --- .changeset/rich-steaks-juggle.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.changeset/rich-steaks-juggle.md b/.changeset/rich-steaks-juggle.md index f47de1af0d..5879f2dace 100644 --- a/.changeset/rich-steaks-juggle.md +++ b/.changeset/rich-steaks-juggle.md @@ -3,5 +3,4 @@ '@backstage/plugin-tech-insights-backend': patch --- -The FactRetriever model is extended by adding optional title and description fields. This allows us to display Fact -Retrievers in the UI in future. +The `FactRetriever` model has been extended by adding optional title and description fields, allowing you to display them in the UI. From 7d770a10d63af8face78c4f0aa51dec2daa76907 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 7 Jul 2022 09:36:13 +0200 Subject: [PATCH 53/64] workflows/pr-review-comment: debug time Signed-off-by: Patrik Oldsberg --- .github/workflows/pr-review-comment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-review-comment.yaml b/.github/workflows/pr-review-comment.yaml index 37cdea5aa9..b7a97abbe4 100644 --- a/.github/workflows/pr-review-comment.yaml +++ b/.github/workflows/pr-review-comment.yaml @@ -15,6 +15,7 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.workflow_run.conclusion == 'success' steps: + - uses: hmarr/debug-action@v2 - uses: backstage/actions/re-review@v0.5.3 with: app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }} From f5c9730639b152e57562330d1499cdeb6fefe1d8 Mon Sep 17 00:00:00 2001 From: Matthew Clarke Date: Thu, 7 Jul 2022 03:45:25 -0400 Subject: [PATCH 54/64] (feat): easier Kubernetes local development steps (#12373) * (feat): easier kubernetes local deevlopment steps Signed-off-by: Matthew Clarke * add changeset Signed-off-by: Matthew Clarke * update api docs Signed-off-by: Matthew Clarke * fix typo Signed-off-by: Matthew Clarke --- .changeset/fifty-cars-compare.md | 13 ++++++ docs/features/kubernetes/configuration.md | 7 +++ .../examples/dice-roller/README.md | 46 +++++++------------ .../LocalKubectlProxyLocator.ts} | 27 +++++++---- .../src/cluster-locator/index.ts | 3 ++ .../KubernetesAuthTranslatorGenerator.ts | 3 ++ plugins/kubernetes/api-report.md | 30 ++---------- .../AwsKubernetesAuthProvider.ts | 27 ----------- .../AzureKubernetesAuthProvider.ts | 27 ----------- .../KubernetesAuthProviders.ts | 20 ++++---- ...hProvider.ts => ServerSideAuthProvider.ts} | 11 +++-- .../src/kubernetes-auth-provider/index.ts | 4 +- 12 files changed, 84 insertions(+), 134 deletions(-) create mode 100644 .changeset/fifty-cars-compare.md rename plugins/{kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts => kubernetes-backend/src/cluster-locator/LocalKubectlProxyLocator.ts} (54%) delete mode 100644 plugins/kubernetes/src/kubernetes-auth-provider/AwsKubernetesAuthProvider.ts delete mode 100644 plugins/kubernetes/src/kubernetes-auth-provider/AzureKubernetesAuthProvider.ts rename plugins/kubernetes/src/kubernetes-auth-provider/{GoogleServiceAccountAuthProvider.ts => ServerSideAuthProvider.ts} (76%) diff --git a/.changeset/fifty-cars-compare.md b/.changeset/fifty-cars-compare.md new file mode 100644 index 0000000000..86115a4b02 --- /dev/null +++ b/.changeset/fifty-cars-compare.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-kubernetes': minor +'@backstage/plugin-kubernetes-backend': minor +--- + +Add `localKubectlProxy` cluster locator method to make local development simpler to setup. + +Consolidated no-op server side auth decorators. +The following Kubernetes auth decorators are now one class (`ServerSideKubernetesAuthProvider`): + +- `AwsKubernetesAuthProvider` +- `AzureKubernetesAuthProvider` +- `ServiceAccountKubernetesAuthProvider` diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index b78021c8f1..633584bdb5 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -57,10 +57,17 @@ This is an array used to determine where to retrieve cluster configuration from. Valid cluster locator methods are: +- [`localKubectlProxy`](#localKubectlProxy) - [`config`](#config) - [`gke`](#gke) - [custom `KubernetesClustersSupplier`](#custom-kubernetesclusterssupplier) +#### `localKubectlProxy` + +This cluster locator method will assume a locally running [`kubectl proxy`](https://kubernetes.io/docs/tasks/extend-kubernetes/http-proxy-access-api/#using-kubectl-to-start-a-proxy-server) process using the default port (8001). + +NOTE: This cluster locator method is for local development only and should not be used in production. + #### `config` This cluster locator method will read cluster information from your app-config diff --git a/plugins/kubernetes-backend/examples/dice-roller/README.md b/plugins/kubernetes-backend/examples/dice-roller/README.md index f7bcdd3973..93dd181c10 100644 --- a/plugins/kubernetes-backend/examples/dice-roller/README.md +++ b/plugins/kubernetes-backend/examples/dice-roller/README.md @@ -2,48 +2,34 @@ This can be used to run the kubernetes plugin locally against a mock service. -# Viewing in local Minikube running Backstage locally +# Viewing in local Kind running Backstage locally ## Prerequisites -- kubectl installed -- Minikube installed, with the following addons - - metrics-server - - ingress -- jq installed +- [kubectl installed](https://kubernetes.io/docs/tasks/tools/#kubectl) +- [Kind installed](https://kind.sigs.k8s.io/docs/user/quick-start/) - Backstage locally built and ready to run ## Steps -1. Start minikube -2. Get the Kubernetes master base URL `kubectl cluster-info` -3. Apply manifests `kubectl apply -f dice-roller-manifests.yaml` -4. Get service account token (see below) -5. Start Backstage UI and backend -6. Register existing component in Backstage - - https://github.com/mclarke47/dice-roller/blob/master/catalog-info.yaml +1. Start kind +2. Apply manifests `kubectl apply -f plugins/kubernetes-backend/examples/dice-roller/dice-roller-manifests.yaml` +3. Run `kubectl proxy` +4. In separate terminal windows start Backstage UI and backend +5. Register a test component ([example](https://github.com/mclarke47/dice-roller/blob/master/catalog-info.yaml)) +6. Visit [kubernetes plugin page](http://localhost:3000/catalog/default/component/dice-roller/kubernetes) -Add or update `app-config.local.yaml` with the following: +### Example `app-config.local.yaml` ```yaml kubernetes: serviceLocatorMethod: type: 'multiTenant' clusterLocatorMethods: - - type: 'config' - clusters: - - url: - name: minikube - serviceAccountToken: - authProvider: 'serviceAccount' + - type: 'localKubectlProxy' + +catalog: + locations: + - type: url + target: https://github.com/mclarke47/dice-roller/blob/master/catalog-info.yaml ``` - -### Getting the service account token - -Mac copy to clipboard: - -``` -kubectl get secret $(kubectl get sa dice-roller -o=json | jq -r '.secrets[0].name') -o=json | jq -r '.data["token"]' | base64 --decode | pbcopy -``` - -Paste into `app-config.local.yaml` `kubernetes.clusters[0].serviceAccountToken` diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts b/plugins/kubernetes-backend/src/cluster-locator/LocalKubectlProxyLocator.ts similarity index 54% rename from plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts rename to plugins/kubernetes-backend/src/cluster-locator/LocalKubectlProxyLocator.ts index 5671112ac8..b2c11fc744 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/LocalKubectlProxyLocator.ts @@ -14,16 +14,25 @@ * limitations under the License. */ -import { KubernetesAuthProvider } from './types'; -import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; +import { ClusterDetails, KubernetesClustersSupplier } from '../types/types'; -export class ServiceAccountKubernetesAuthProvider - implements KubernetesAuthProvider +export class LocalKubectlProxyClusterLocator + implements KubernetesClustersSupplier { - async decorateRequestBodyForAuth( - requestBody: KubernetesRequestBody, - ): Promise { - // No-op, with service account for auth, cluster config/details should already have serviceAccountToken - return requestBody; + private readonly clusterDetails: ClusterDetails[]; + + public constructor() { + this.clusterDetails = [ + { + name: 'local', + url: 'http:/localhost:8001', + authProvider: 'localKubectlProxy', + skipMetricsLookup: true, + }, + ]; + } + + async getClusters(): Promise { + return this.clusterDetails; } } diff --git a/plugins/kubernetes-backend/src/cluster-locator/index.ts b/plugins/kubernetes-backend/src/cluster-locator/index.ts index 53aeb44f8b..d32f7ffee8 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/index.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/index.ts @@ -19,6 +19,7 @@ import { Duration } from 'luxon'; import { ClusterDetails, KubernetesClustersSupplier } from '../types/types'; import { ConfigClusterLocator } from './ConfigClusterLocator'; import { GkeClusterLocator } from './GkeClusterLocator'; +import { LocalKubectlProxyClusterLocator } from './LocalKubectlProxyLocator'; class CombinedClustersSupplier implements KubernetesClustersSupplier { constructor(readonly clusterSuppliers: KubernetesClustersSupplier[]) {} @@ -45,6 +46,8 @@ export const getCombinedClusterSupplier = ( .map(clusterLocatorMethod => { const type = clusterLocatorMethod.getString('type'); switch (type) { + case 'localKubectlProxy': + return new LocalKubectlProxyClusterLocator(); case 'config': return ConfigClusterLocator.fromConfig(clusterLocatorMethod); case 'gke': diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts index c2df3e6c23..24de85d7b7 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts @@ -49,6 +49,9 @@ export class KubernetesAuthTranslatorGenerator { case 'oidc': { return new OidcKubernetesAuthTranslator(); } + case 'localKubectlProxy': { + return new NoopKubernetesAuthTranslator(); + } default: { throw new Error( `authProvider "${authProvider}" has no KubernetesAuthTranslator associated with it`, diff --git a/plugins/kubernetes/api-report.md b/plugins/kubernetes/api-report.md index dc2c222712..1a95446e5f 100644 --- a/plugins/kubernetes/api-report.md +++ b/plugins/kubernetes/api-report.md @@ -32,17 +32,6 @@ import { V1ReplicaSet } from '@kubernetes/client-node'; import { V1Service } from '@kubernetes/client-node'; import { V1StatefulSet } from '@kubernetes/client-node'; -// Warning: (ae-forgotten-export) The symbol "KubernetesAuthProvider" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "AwsKubernetesAuthProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class AwsKubernetesAuthProvider implements KubernetesAuthProvider { - // (undocumented) - decorateRequestBodyForAuth( - requestBody: KubernetesRequestBody, - ): Promise; -} - // Warning: (ae-forgotten-export) The symbol "ClusterProps" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Cluster" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -147,6 +136,7 @@ export function formatClusterLink( options: FormatClusterLinkOptions, ): string | undefined; +// Warning: (ae-forgotten-export) The symbol "KubernetesAuthProvider" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "GoogleKubernetesAuthProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -160,18 +150,6 @@ export class GoogleKubernetesAuthProvider implements KubernetesAuthProvider { ): Promise; } -// Warning: (ae-missing-release-tag) "GoogleServiceAccountAuthProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class GoogleServiceAccountAuthProvider - implements KubernetesAuthProvider -{ - // (undocumented) - decorateRequestBodyForAuth( - requestBody: KubernetesRequestBody, - ): Promise; -} - // Warning: (ae-missing-release-tag) "GroupedResponses" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -386,10 +364,8 @@ export const PodsTable: ({ // @public (undocumented) export const Router: (props: { refreshIntervalMs?: number }) => JSX.Element; -// Warning: (ae-missing-release-tag) "ServiceAccountKubernetesAuthProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class ServiceAccountKubernetesAuthProvider +// @public +export class ServerSideKubernetesAuthProvider implements KubernetesAuthProvider { // (undocumented) diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/AwsKubernetesAuthProvider.ts b/plugins/kubernetes/src/kubernetes-auth-provider/AwsKubernetesAuthProvider.ts deleted file mode 100644 index 789da9af50..0000000000 --- a/plugins/kubernetes/src/kubernetes-auth-provider/AwsKubernetesAuthProvider.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { KubernetesAuthProvider } from './types'; -import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; - -export class AwsKubernetesAuthProvider implements KubernetesAuthProvider { - async decorateRequestBodyForAuth( - requestBody: KubernetesRequestBody, - ): Promise { - // No-op, with aws auth, server's AWS credentials are used for access - return requestBody; - } -} diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/AzureKubernetesAuthProvider.ts b/plugins/kubernetes/src/kubernetes-auth-provider/AzureKubernetesAuthProvider.ts deleted file mode 100644 index 60401bbe4d..0000000000 --- a/plugins/kubernetes/src/kubernetes-auth-provider/AzureKubernetesAuthProvider.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { KubernetesAuthProvider } from './types'; -import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; - -export class AzureKubernetesAuthProvider implements KubernetesAuthProvider { - async decorateRequestBodyForAuth( - requestBody: KubernetesRequestBody, - ): Promise { - // No-op, with azure auth, server's Azure credentials are used for access - return requestBody; - } -} diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts b/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts index bcd1b2911a..eb3ea456ea 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts @@ -17,11 +17,8 @@ import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; import { KubernetesAuthProvider, KubernetesAuthProvidersApi } from './types'; import { GoogleKubernetesAuthProvider } from './GoogleKubernetesAuthProvider'; -import { ServiceAccountKubernetesAuthProvider } from './ServiceAccountKubernetesAuthProvider'; -import { AwsKubernetesAuthProvider } from './AwsKubernetesAuthProvider'; +import { ServerSideKubernetesAuthProvider } from './ServerSideAuthProvider'; import { OAuthApi, OpenIdConnectApi } from '@backstage/core-plugin-api'; -import { GoogleServiceAccountAuthProvider } from './GoogleServiceAccountAuthProvider'; -import { AzureKubernetesAuthProvider } from './AzureKubernetesAuthProvider'; import { OidcKubernetesAuthProvider } from './OidcKubernetesAuthProvider'; export class KubernetesAuthProviders implements KubernetesAuthProvidersApi { @@ -41,16 +38,23 @@ export class KubernetesAuthProviders implements KubernetesAuthProvidersApi { ); this.kubernetesAuthProviderMap.set( 'serviceAccount', - new ServiceAccountKubernetesAuthProvider(), + new ServerSideKubernetesAuthProvider(), ); this.kubernetesAuthProviderMap.set( 'googleServiceAccount', - new GoogleServiceAccountAuthProvider(), + new ServerSideKubernetesAuthProvider(), + ); + this.kubernetesAuthProviderMap.set( + 'aws', + new ServerSideKubernetesAuthProvider(), ); - this.kubernetesAuthProviderMap.set('aws', new AwsKubernetesAuthProvider()); this.kubernetesAuthProviderMap.set( 'azure', - new AzureKubernetesAuthProvider(), + new ServerSideKubernetesAuthProvider(), + ); + this.kubernetesAuthProviderMap.set( + 'localKubectlProxy', + new ServerSideKubernetesAuthProvider(), ); if (options.oidcProviders) { diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/GoogleServiceAccountAuthProvider.ts b/plugins/kubernetes/src/kubernetes-auth-provider/ServerSideAuthProvider.ts similarity index 76% rename from plugins/kubernetes/src/kubernetes-auth-provider/GoogleServiceAccountAuthProvider.ts rename to plugins/kubernetes/src/kubernetes-auth-provider/ServerSideAuthProvider.ts index 9f522bea64..1c475b9568 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/GoogleServiceAccountAuthProvider.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/ServerSideAuthProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,18 @@ import { KubernetesAuthProvider } from './types'; import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; -export class GoogleServiceAccountAuthProvider +/** + * No-op KubernetesAuthProvider, authorization will be handled in the kubernetes-backend plugin + * + * @public + */ +export class ServerSideKubernetesAuthProvider implements KubernetesAuthProvider { async decorateRequestBodyForAuth( requestBody: KubernetesRequestBody, ): Promise { - // No-op, with google service account auth, server's AWS credentials are used for access + // No-op, auth will be taken care of on the server-side return requestBody; } } diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/index.ts b/plugins/kubernetes/src/kubernetes-auth-provider/index.ts index 6f8e3f415f..23197321f6 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/index.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/index.ts @@ -17,7 +17,5 @@ export { kubernetesAuthProvidersApiRef } from './types'; export type { KubernetesAuthProvidersApi } from './types'; export { KubernetesAuthProviders } from './KubernetesAuthProviders'; -export { AwsKubernetesAuthProvider } from './AwsKubernetesAuthProvider'; export { GoogleKubernetesAuthProvider } from './GoogleKubernetesAuthProvider'; -export { GoogleServiceAccountAuthProvider } from './GoogleServiceAccountAuthProvider'; -export { ServiceAccountKubernetesAuthProvider } from './ServiceAccountKubernetesAuthProvider'; +export { ServerSideKubernetesAuthProvider } from './ServerSideAuthProvider'; From 80d7ca1a8e105522f21849f9e3d5a7374689587e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 07:59:29 +0000 Subject: [PATCH 55/64] chore(deps): update dependency @types/express-session to v1.17.5 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 5b8110ec78..eedd5ec039 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6343,9 +6343,9 @@ "@types/range-parser" "*" "@types/express-session@^1.17.2": - version "1.17.4" - resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.4.tgz#97a30a35e853a61bdd26e727453b8ed314d6166b" - integrity sha512-7cNlSI8+oOBUHTfPXMwDxF/Lchx5aJ3ho7+p9jJZYVg9dVDJFh3qdMXmJtRsysnvS+C6x46k9DRYmrmCkE+MVg== + version "1.17.5" + resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.5.tgz#13f48852b4aa60ff595835faeb4b4dda0ba0866e" + integrity sha512-l0DhkvNVfyUPEEis8fcwbd46VptfA/jmMwHfob2TfDMf3HyPLiB9mKD71LXhz5TMUobODXPD27zXSwtFQLHm+w== dependencies: "@types/express" "*" From 83f0df1bcf2ed7dfbda7a874fc58b78cdbb382ee Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 08:13:20 +0000 Subject: [PATCH 56/64] fix(deps): update dependency moment to v2.29.4 [security] Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6b78dbbab6..f3f39b6875 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18912,9 +18912,9 @@ modify-values@^1.0.0: integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== moment@^2.27.0, moment@^2.29.1: - version "2.29.3" - resolved "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" - integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== + version "2.29.4" + resolved "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== morgan@^1.10.0: version "1.10.0" From 1c981abef78ffe62bcb58ff63e92c46068a565fe Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 08:16:47 +0000 Subject: [PATCH 57/64] chore(deps): update dependency msw to v0.43.1 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6b78dbbab6..7b0f4e0cf0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18978,9 +18978,9 @@ msw@^0.39.2: yargs "^17.3.1" msw@^0.43.0: - version "0.43.0" - resolved "https://registry.npmjs.org/msw/-/msw-0.43.0.tgz#10c6fc3fb1752c0a144179e5ab04c6a512cc9959" - integrity sha512-XJylZP0qW3D5WUGWh9FFefJEl3MGG4y1I+/8a833d0eedm6B+GaPm6wPVZNcnlS2YVTagvEgShVJ7ZtY66tTRQ== + version "0.43.1" + resolved "https://registry.npmjs.org/msw/-/msw-0.43.1.tgz#57cb4af56f07442e8a6d14d76032a0ab41434256" + integrity sha512-wzhPpL6RsiYkyIUlTCg0aZY0aRZa4Eiubd6MOA5oJVgfuapDmvZrI8OMi4h4e+fpHD+Qsy+4unAjv3wpWia5yw== dependencies: "@mswjs/cookies" "^0.2.0" "@mswjs/interceptors" "^0.16.3" From b557e6c58d811f7c5f8ee3668d4e9945db59017c Mon Sep 17 00:00:00 2001 From: Julius Berger Date: Thu, 7 Jul 2022 10:44:23 +0200 Subject: [PATCH 58/64] add changeset Signed-off-by: Julius Berger --- .changeset/cuddly-comics-pump.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cuddly-comics-pump.md diff --git a/.changeset/cuddly-comics-pump.md b/.changeset/cuddly-comics-pump.md new file mode 100644 index 0000000000..af2c08cd6c --- /dev/null +++ b/.changeset/cuddly-comics-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Fixed that adding more than one `allowedOwner` or `allowedRepo` in the template config will now still set the first value as default in the initial form state of RepoUrlPicker. From 212aaab976b52a3e6918e419a4e035e093f217b4 Mon Sep 17 00:00:00 2001 From: Julius Berger Date: Thu, 7 Jul 2022 10:46:35 +0200 Subject: [PATCH 59/64] add changeset Signed-off-by: Julius Berger --- .changeset/cuddly-comics-pump.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cuddly-comics-pump.md b/.changeset/cuddly-comics-pump.md index af2c08cd6c..6d224bd34c 100644 --- a/.changeset/cuddly-comics-pump.md +++ b/.changeset/cuddly-comics-pump.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder': patch --- -Fixed that adding more than one `allowedOwner` or `allowedRepo` in the template config will now still set the first value as default in the initial form state of RepoUrlPicker. +Fixed that adding more than one `allowedOwner` or `allowedRepo` in the template config will now still set the first value as default in the initial form state of `RepoUrlPicker`. From d14e60a6a6c6da7fa7b923317de505331f52b208 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:08:20 +0000 Subject: [PATCH 60/64] fix(deps): update dependency graphiql to v1.9.11 Signed-off-by: Renovate Bot --- yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/yarn.lock b/yarn.lock index b14ea3e401..3e7bfe9b12 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2358,10 +2358,10 @@ teeny-request "^8.0.0" uuid "^8.0.0" -"@graphiql/react@^0.4.3": - version "0.4.3" - resolved "https://registry.npmjs.org/@graphiql/react/-/react-0.4.3.tgz#29ca3125c7a349de5e9beb2aef660137fbcbde2e" - integrity sha512-MnH+7LqRqFnwo8YBDtgDfJOdeS5QqNTzH2gTvTbEYYWaNQW1+Vhbsyu2RKHnPSgKVQlfLzdjw6SAMtSb3q1Qog== +"@graphiql/react@^0.5.0": + version "0.5.0" + resolved "https://registry.npmjs.org/@graphiql/react/-/react-0.5.0.tgz#3bc00de581b09d962a9f391ed725d06f34a757d3" + integrity sha512-Tn0V4yTcRPBgBVq9SVkqGkmlUi7aqmOrn4+vmhwnlxSFJU7mnKak7DZRc9wPVFBTdcHc0pWNKBzY/demFHSxSw== dependencies: "@graphiql/toolkit" "^0.6.0" codemirror "^5.65.3" @@ -14208,11 +14208,11 @@ grapheme-splitter@^1.0.4: integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== graphiql@^1.5.12, graphiql@^1.8.8: - version "1.9.10" - resolved "https://registry.npmjs.org/graphiql/-/graphiql-1.9.10.tgz#ba7113ed8b1bc988d00d8d7cc51901b51b684bde" - integrity sha512-uYYK/zBTV9Y/IHw4uopGVRusbF2xmPnptx7OMFTnTHOTQOmzfIGRTn58WuPgevyEi+9zSLojM86lot9xvNA1/w== + version "1.9.11" + resolved "https://registry.npmjs.org/graphiql/-/graphiql-1.9.11.tgz#3ade0802a0459792d1533f71e95b39bedce9c3ac" + integrity sha512-/nIFi+y8PNpkC/fb/mmkWUDsX+nIS7DdemxrTPugKpGku/hqVLe0ppXFsJbPuT0UrleL1XASv0sZm+Axh4XGWA== dependencies: - "@graphiql/react" "^0.4.3" + "@graphiql/react" "^0.5.0" "@graphiql/toolkit" "^0.6.0" entities "^2.0.0" graphql-language-service "^5.0.6" From a43a7863a6f4076fed739d4b2c6c528db5c5f7e9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:09:45 +0000 Subject: [PATCH 61/64] fix(deps): update dependency aws-sdk to v2.1169.0 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index b14ea3e401..c2db0bada0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8445,9 +8445,9 @@ aws-sdk-mock@^5.2.1: traverse "^0.6.6" aws-sdk@^2.1122.0, aws-sdk@^2.840.0, aws-sdk@^2.948.0: - version "2.1168.0" - resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1168.0.tgz#7936459edee0c999065f1ecf2ef0bea89ea056c1" - integrity sha512-9+WYoYTHHjLqeWdSSLbNpmc/NgnZpX4LGiyYjXenh4WNRBXshXI0XioTK8BQgDscVzB978EJV8kG1nZGE3USzw== + version "2.1169.0" + resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1169.0.tgz#06c910092e0dd97ac884ce0939a69c062308184a" + integrity sha512-zBJXLih/iBueYt7hDednt7Jncb+6wzUjDh6JMdl2e0A61xb+Z5o0NNkjbJTUJELdx96u3R4+yil0DoRTOipksA== dependencies: buffer "4.9.2" events "1.1.1" From 6714971845c9f331abc6e32266a8fb7240bcba47 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 7 Jul 2022 12:03:37 +0200 Subject: [PATCH 62/64] remove refreshing by keys from refresh service Signed-off-by: Kiss Miklos --- .../src/service/AuthorizedRefreshService.test.ts | 1 - .../src/service/AuthorizedRefreshService.ts | 9 +-------- .../src/service/DefaultRefreshService.ts | 11 +---------- .../catalog-backend/src/service/createRouter.test.ts | 4 ++-- plugins/catalog-backend/src/service/types.ts | 4 ---- 5 files changed, 4 insertions(+), 25 deletions(-) diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts index f22b5b86f4..5d0a192470 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.test.ts @@ -22,7 +22,6 @@ import { AuthorizedRefreshService } from './AuthorizedRefreshService'; describe('AuthorizedRefreshService', () => { const refreshService = { refresh: jest.fn(), - refreshByRefreshKeys: jest.fn(), }; const permissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 19f6270da6..8634fbf86d 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -20,11 +20,7 @@ import { AuthorizeResult, PermissionEvaluator, } from '@backstage/plugin-permission-common'; -import { - RefreshByRefreshKeysOptions, - RefreshOptions, - RefreshService, -} from './types'; +import { RefreshOptions, RefreshService } from './types'; export class AuthorizedRefreshService implements RefreshService { constructor( @@ -49,7 +45,4 @@ export class AuthorizedRefreshService implements RefreshService { } await this.service.refresh(options); } - async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { - await this.service.refreshByRefreshKeys(options); - } } diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index deb69bd041..3b982a0e46 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -15,11 +15,7 @@ */ import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; -import { - RefreshByRefreshKeysOptions, - RefreshOptions, - RefreshService, -} from './types'; +import { RefreshOptions, RefreshService } from './types'; export class DefaultRefreshService implements RefreshService { private database: DefaultProcessingDatabase; @@ -49,9 +45,4 @@ export class DefaultRefreshService implements RefreshService { }); }); } - async refreshByRefreshKeys(options: RefreshByRefreshKeysOptions) { - await this.database.transaction(async tx => { - await this.database.refreshByRefreshKeys(tx, options); - }); - } } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index b7eeb6f1bc..16a36eb8c4 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -57,7 +57,7 @@ describe('createRouter readonly disabled', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; + refreshService = { refresh: jest.fn() }; orchestrator = { process: jest.fn() }; const router = await createRouter({ entitiesCatalog, @@ -712,7 +712,7 @@ describe('NextRouter permissioning', () => { listLocations: jest.fn(), deleteLocation: jest.fn(), }; - refreshService = { refresh: jest.fn(), refreshByRefreshKeys: jest.fn() }; + refreshService = { refresh: jest.fn() }; const router = await createRouter({ entitiesCatalog, locationService, diff --git a/plugins/catalog-backend/src/service/types.ts b/plugins/catalog-backend/src/service/types.ts index a073e5bb6c..f6cfa2e783 100644 --- a/plugins/catalog-backend/src/service/types.ts +++ b/plugins/catalog-backend/src/service/types.ts @@ -61,9 +61,6 @@ export type RefreshOptions = { authorizationToken?: string; }; -export type RefreshByRefreshKeysOptions = { - keys: string[]; -}; /** * A service that manages refreshes of entities in the catalog. * @@ -74,7 +71,6 @@ export interface RefreshService { * Request a refresh of entities in the catalog. */ refresh(options: RefreshOptions): Promise; - refreshByRefreshKeys(options: RefreshByRefreshKeysOptions): Promise; } /** From 1c23763ce8de8ced945963818b8c012a90d31147 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Thu, 7 Jul 2022 12:09:22 +0200 Subject: [PATCH 63/64] update changeset Signed-off-by: Kiss Miklos --- .changeset/cold-coins-tickle.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md index 80f0958daa..301a9d0a84 100644 --- a/.changeset/cold-coins-tickle.md +++ b/.changeset/cold-coins-tickle.md @@ -6,3 +6,14 @@ Added an option to be able to trigger refreshes on entities based on a prestored The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `:` The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. + +The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts` + +```ts + // yamlPlaceholderResolver + ... + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + ... +``` From 0bb958ff0b8cc491356183a6e29ddd4f1d1df722 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 7 Jul 2022 13:57:08 +0200 Subject: [PATCH 64/64] workflows: new method for signaling PR number of review comments Signed-off-by: Patrik Oldsberg --- .../workflows/pr-review-comment-trigger.yaml | 11 ++++++++- .github/workflows/pr-review-comment.yaml | 23 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-review-comment-trigger.yaml b/.github/workflows/pr-review-comment-trigger.yaml index 25475a03dc..67e4c5abba 100644 --- a/.github/workflows/pr-review-comment-trigger.yaml +++ b/.github/workflows/pr-review-comment-trigger.yaml @@ -16,4 +16,13 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.comment.user.id == github.event.pull_request.user.id steps: - - run: echo "This PR needs another review" + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + mkdir -p ./pr + echo $PR_NUMBER > ./pr/pr_number + - uses: actions/upload-artifact@v3 + with: + name: pr_number-${{ github.event.pull_request.number }} + path: pr/ diff --git a/.github/workflows/pr-review-comment.yaml b/.github/workflows/pr-review-comment.yaml index b7a97abbe4..fac7fd2fc4 100644 --- a/.github/workflows/pr-review-comment.yaml +++ b/.github/workflows/pr-review-comment.yaml @@ -15,11 +15,30 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.workflow_run.conclusion == 'success' steps: - - uses: hmarr/debug-action@v2 + # Inspired by https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow + - name: Read PR Number + id: pr-number + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + const [artifact] = allArtifacts.data.artifacts.filter(artifact => artifact.name.startsWith('pr_number-')) + if (!artifact) { + throw new Error('No PR Number artifact available') + } + + const prNumber = artifact.name.slice('pr_number-'.length) + console.log(`::set-output name=pr-number::${prNumber}`); + - uses: backstage/actions/re-review@v0.5.3 with: app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }} private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }} installation-id: ${{ secrets.BACKSTAGE_GOALIE_INSTALLATION_ID }} project-id: PVT_kwDOBFKqdc02LQ - issue-number: ${{ github.event.workflow_run.pull_requests[0].number }} + issue-number: ${{ steps.pr-number.outputs.pr-number }}