diff --git a/.changeset/cold-coins-tickle.md b/.changeset/cold-coins-tickle.md new file mode 100644 index 0000000000..301a9d0a84 --- /dev/null +++ b/.changeset/cold-coins-tickle.md @@ -0,0 +1,19 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key. + +The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `:` +The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities. + +The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts` + +```ts + // yamlPlaceholderResolver + ... + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + ... +``` diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index cb8a5abb8a..09ca4d8dfe 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -268,6 +268,12 @@ export type CatalogProcessorParser = (options: { location: LocationSpec; }) => AsyncIterable; +// @public (undocumented) +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + key: string; +}; + // @public (undocumented) export type CatalogProcessorRelationResult = { type: 'relation'; @@ -279,7 +285,8 @@ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; // @public (undocumented) export class CodeOwnersProcessor implements CatalogProcessor { @@ -545,7 +552,11 @@ export class PlaceholderProcessor implements CatalogProcessor { // (undocumented) getProcessorName(): string; // (undocumented) - preProcessEntity(entity: Entity, location: LocationSpec): Promise; + preProcessEntity( + entity: Entity, + location: LocationSpec, + emit: CatalogProcessorEmit, + ): Promise; } // @public (undocumented) @@ -567,6 +578,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; // @public (undocumented) @@ -601,6 +613,7 @@ export const processingResult: Readonly<{ newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; + readonly refresh: (key: string) => CatalogProcessorResult; }>; // @public (undocumented) diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js new file mode 100644 index 0000000000..b1b67f68f2 --- /dev/null +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('refresh_keys', table => { + table.comment( + 'This table contains relations between entities and keys to trigger refreshes with', + ); + table + .text('entity_id') + .notNullable() + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE') + .comment('A reference to the entity that the refresh key is tied to'); + table + .text('key') + .notNullable() + .comment( + 'A reference to a key which should be used to trigger a refresh on this entity', + ); + table.index('entity_id', 'refresh_keys_entity_id_idx'); + table.index('key', 'refresh_keys_key_idx'); + }); +}; + +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_keys', table => { + table.dropIndex([], 'refresh_keys_entity_id_idx'); + table.dropIndex([], 'refresh_keys_key_idx'); + }); + + await knex.schema.dropTable('refresh_keys'); +}; diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts index 1cb4b71aa3..136c23d20d 100644 --- a/plugins/catalog-backend/src/api/index.ts +++ b/plugins/catalog-backend/src/api/index.ts @@ -26,6 +26,7 @@ export type { CatalogProcessorRelationResult, CatalogProcessorErrorResult, CatalogProcessorResult, + CatalogProcessorRefreshKeysResult, } from './processor'; export type { EntityProvider, diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts index 2fc2c9eac7..84f1f4b70d 100644 --- a/plugins/catalog-backend/src/api/processingResult.ts +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -65,4 +65,8 @@ export const processingResult = Object.freeze({ relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; }, + + refresh(key: string): CatalogProcessorResult { + return { type: 'refresh', key }; + }, } as const); diff --git a/plugins/catalog-backend/src/api/processor.ts b/plugins/catalog-backend/src/api/processor.ts index 3f274d13ea..44e8b298b0 100644 --- a/plugins/catalog-backend/src/api/processor.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -169,9 +169,16 @@ export type CatalogProcessorErrorResult = { location: LocationSpec; }; +/** @public */ +export type CatalogProcessorRefreshKeysResult = { + type: 'refresh'; + key: string; +}; + /** @public */ export type CatalogProcessorResult = | CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult - | CatalogProcessorErrorResult; + | CatalogProcessorErrorResult + | CatalogProcessorRefreshKeysResult; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 251388f0d2..6a0986b345 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -24,6 +24,7 @@ import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from './migrations'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -98,6 +99,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities: [], + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'undefined'`, @@ -117,6 +119,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }; const { knex, db } = await createDatabase(databaseId); @@ -143,6 +146,7 @@ describe('Default Processing Database', () => { ...options, resultHash: '', locationKey: 'fail', + refreshKeys: [], }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'fail'`, @@ -174,6 +178,7 @@ describe('Default Processing Database', () => { relations: [], deferredEntities: [], locationKey: 'key', + refreshKeys: [], errors: "['something broke']", }), ); @@ -228,6 +233,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -250,6 +256,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: relations, deferredEntities: [], + refreshKeys: [], }), ); @@ -309,6 +316,7 @@ describe('Default Processing Database', () => { resultHash: '', relations: [], deferredEntities, + refreshKeys: [], }), ); @@ -400,6 +408,7 @@ describe('Default Processing Database', () => { processedEntity, resultHash: '', relations: [], + refreshKeys: [], deferredEntities: [ { entity: { @@ -465,6 +474,63 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'stores the refresh keys for the entity', + async databaseId => { + const mockLogger = { + debug: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + }; + const { knex, db } = await createDatabase( + databaseId, + mockLogger as unknown as Logger, + ); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const deferredEntities = [ + { + entity: { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'next', + }, + }, + locationKey: 'mock', + }, + ]; + + await db.transaction(tx => + db.updateProcessedEntity(tx, { + id, + processedEntity, + resultHash: '', + relations: [], + deferredEntities, + refreshKeys: [{ key: 'protocol:foo-bar.com' }], + }), + ); + + const refreshKeys = await knex('refresh_keys') + .where({ entity_id: id }) + .select(); + + expect(refreshKeys[0]).toEqual({ + entity_id: id, + key: 'protocol:foo-bar.com', + }); + }, + ); }); describe('updateEntityCache', () => { diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index dd28cf1290..903dd59e9e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -33,12 +33,14 @@ import { UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, + RefreshByKeyOptions, } from './types'; import { DeferredEntity } from '../processing/types'; import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; import { + DbRefreshKeysRow, DbRefreshStateReferencesRow, DbRefreshStateRow, DbRelationsRow, @@ -77,6 +79,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, relations, deferredEntities, + refreshKeys, locationKey, } = options; const refreshResult = await tx('refresh_state') @@ -100,11 +103,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { `Conflicting write of processing result for ${id} with location key '${locationKey}'`, ); } + const sourceEntityRef = stringifyEntityRef(processedEntity); // Schedule all deferred entities for future processing. await this.addUnprocessedEntities(tx, { entities: deferredEntities, - sourceEntityRef: stringifyEntityRef(processedEntity), + sourceEntityRef, }); // Delete old relations @@ -138,6 +142,21 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { BATCH_SIZE, ); + // Delete old refresh keys + await tx('refresh_keys') + .where({ entity_id: id }) + .delete(); + + // Insert the refresh keys for the processed entity + await tx.batchInsert( + 'refresh_keys', + refreshKeys.map(k => ({ + entity_id: id, + key: k.key, + })), + BATCH_SIZE, + ); + return { previous: { relations: previousRelationRows, @@ -516,6 +535,25 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } + async refreshByRefreshKeys( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ) { + const tx = txOpaque as Knex.Transaction; + const { keys } = options; + + await tx('refresh_state') + .whereIn('entity_id', function selectEntityRefs(tx2) { + tx2 + .whereIn('key', keys) + .select({ + entity_id: 'refresh_keys.entity_id', + }) + .from('refresh_keys'); + }) + .update({ next_update_at: tx.fn.now() }); + } + async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 40b4cb9a12..b9fe12be11 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -43,6 +43,11 @@ export type DbRefreshStateRow = { location_key?: string; }; +export type DbRefreshKeysRow = { + entity_id: string; + key: string; +}; + export type DbRefreshStateReferencesRow = { source_key?: string; source_entity_ref?: string; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ce45a88938..8a839c0650 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; import { EntityRelationSpec } from '../api'; -import { DeferredEntity } from '../processing/types'; +import { DeferredEntity, RefreshKeyData } from '../processing/types'; import { DbRelationsRow } from './tables'; /** @@ -38,6 +38,7 @@ export type UpdateProcessedEntityOptions = { relations: EntityRelationSpec[]; deferredEntities: DeferredEntity[]; locationKey?: string; + refreshKeys: RefreshKeyData[]; }; export type UpdateEntityCacheOptions = { @@ -81,6 +82,10 @@ export type ReplaceUnprocessedEntitiesOptions = type: 'delta'; }; +export type RefreshByKeyOptions = { + keys: string[]; +}; + export type RefreshOptions = { entityRef: string; }; diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index a3eb5e4542..0c114f49a1 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -43,7 +43,10 @@ describe('FileReaderProcessor', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual(spec); - expect(generated.entity).toEqual({ kind: 'Component' }); + expect(generated.entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); }); it('should fail load from file with error', async () => { @@ -77,14 +80,24 @@ describe('FileReaderProcessor', () => { defaultEntityDataParser, ); - expect(emit).toBeCalledTimes(2); - expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); + expect(emit).toBeCalledTimes(4); + expect(emit.mock.calls[0][0].entity).toEqual({ + kind: 'Component', + metadata: { name: 'component-test' }, + }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); - expect(emit.mock.calls[1][0].entity).toEqual({ kind: 'API' }); - expect(emit.mock.calls[1][0].location).toEqual({ + expect(emit.mock.calls[1][0].key).toContain('file:'); + expect(emit.mock.calls[1][0].key).toContain( + 'fileReaderProcessor/component.yaml', + ); + expect(emit.mock.calls[2][0].entity).toEqual({ + kind: 'API', + metadata: { name: 'api-test' }, + }); + expect(emit.mock.calls[2][0].location).toEqual({ type: 'file', target: expect.stringMatching(/^[^*]*$/), }); diff --git a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 66c6479fa7..8ececed59f 100644 --- a/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -28,6 +28,8 @@ import { const glob = promisify(g); +const LOCATION_TYPE = 'file'; + /** @public */ export class FileReaderProcessor implements CatalogProcessor { getProcessorName(): string { @@ -40,7 +42,7 @@ export class FileReaderProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'file') { + if (location.type !== LOCATION_TYPE) { return false; } @@ -50,17 +52,23 @@ export class FileReaderProcessor implements CatalogProcessor { if (fileMatches.length > 0) { for (const fileMatch of fileMatches) { const data = await fs.readFile(fileMatch); + const normalizedFilePath = path.normalize(fileMatch); // The normalize converts to native slashes; the glob library returns // forward slashes even on windows for await (const parseResult of parser({ data: data, location: { - type: 'file', - target: path.normalize(fileMatch), + type: LOCATION_TYPE, + target: normalizedFilePath, }, })) { emit(parseResult); + emit( + processingResult.refresh( + `${LOCATION_TYPE}:${normalizedFilePath}`, + ), + ); } } } else if (!optional) { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts index adff97336e..9bfa9027d9 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts @@ -17,6 +17,7 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; +import { CatalogProcessorResult } from '../../api'; import { jsonPlaceholderResolver, PlaceholderProcessor, @@ -51,7 +52,7 @@ describe('PlaceholderProcessor', () => { integrations, }); await expect( - processor.preProcessEntity(input, { type: 't', target: 'l' }), + processor.preProcessEntity(input, { type: 't', target: 'l' }, () => {}), ).resolves.toBe(input); }); @@ -76,6 +77,7 @@ describe('PlaceholderProcessor', () => { spec: { a: [{ b: { $upper: 'text' } }] }, }, { type: 'fake', target: 'http://example.com' }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -110,7 +112,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -131,7 +133,7 @@ describe('PlaceholderProcessor', () => { }; await expect( - processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}), ).resolves.toEqual(entity); expect(read).not.toBeCalled(); @@ -158,6 +160,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -194,6 +197,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -228,6 +232,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -266,6 +271,7 @@ describe('PlaceholderProcessor', () => { target: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -303,6 +309,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).resolves.toEqual({ apiVersion: 'a', @@ -343,6 +350,7 @@ describe('PlaceholderProcessor', () => { type: 'url', target: './a/b/catalog-info.yaml', }, + () => {}, ), ).rejects.toThrow( /^Placeholder \$text could not form a URL out of \.\/a\/b\/catalog-info\.yaml and \.\.\/c\/catalog-info\.yaml, TypeError \[ERR_INVALID_URL\]/, @@ -350,6 +358,35 @@ describe('PlaceholderProcessor', () => { expect(read).not.toBeCalled(); }); + it('should emit the resolverValue as a refreshKey', async () => { + read.mockResolvedValue( + Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), + ); + + const processor = new PlaceholderProcessor({ + resolvers: { + json: jsonPlaceholderResolver, + }, + reader, + integrations, + }); + + const emitted = new Array(); + await processor.preProcessEntity( + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n' }, + spec: { a: [{ b: { $json: './path-to-file.json' } }] }, + }, + { type: 'fake', target: 'http://example.com' }, + result => emitted.push(result), + ); + expect(emitted[0]).toEqual({ + type: 'refresh', + key: 'url:http://example.com/path-to-file.json', + }); + }); }); describe('yamlPlaceholderResolver', () => { @@ -360,6 +397,7 @@ describe('yamlPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { @@ -405,6 +443,7 @@ describe('jsonPlaceholderResolver', () => { baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml', read, resolveUrl: (url, base) => integrations.resolveUrl({ url, base }), + emit: () => {}, }; beforeEach(() => { diff --git a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index ddd5f951db..be2e9f9f5b 100644 --- a/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,12 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from '../../api'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; @@ -37,6 +42,7 @@ export type PlaceholderResolverParams = { baseUrl: string; read: PlaceholderResolverRead; resolveUrl: PlaceholderResolverResolveUrl; + emit: CatalogProcessorEmit; }; /** @public */ @@ -66,6 +72,7 @@ export class PlaceholderProcessor implements CatalogProcessor { async preProcessEntity( entity: Entity, location: LocationSpec, + emit: CatalogProcessorEmit, ): Promise { const process = async (data: any): Promise<[any, boolean]> => { if (!data || !(data instanceof Object)) { @@ -102,6 +109,7 @@ export class PlaceholderProcessor implements CatalogProcessor { const resolverKey = keys[0].substr(1); const resolverValue = data[keys[0]]; + const resolver = this.options.resolvers[resolverKey]; if (!resolver || typeof resolverValue !== 'string') { // If there was no such placeholder resolver or if the value was not a @@ -134,6 +142,7 @@ export class PlaceholderProcessor implements CatalogProcessor { baseUrl: location.target, read, resolveUrl, + emit, }), true, ]; @@ -151,11 +160,13 @@ export class PlaceholderProcessor implements CatalogProcessor { export async function yamlPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); let documents: yaml.Document.Parsed[]; try { - documents = yaml.parseAllDocuments(text).filter(d => d); + documents = yaml.parseAllDocuments(content).filter(d => d); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`, @@ -182,10 +193,12 @@ export async function yamlPlaceholderResolver( export async function jsonPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - const text = await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); try { - return JSON.parse(text); + return JSON.parse(content); } catch (e) { throw new Error( `Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`, @@ -196,7 +209,11 @@ export async function jsonPlaceholderResolver( export async function textPlaceholderResolver( params: PlaceholderResolverParams, ): Promise { - return await readTextLocation(params); + const { content, url } = await readTextLocation(params); + + params.emit(processingResult.refresh(`url:${url}`)); + + return content; } /* @@ -205,12 +222,12 @@ export async function textPlaceholderResolver( async function readTextLocation( params: PlaceholderResolverParams, -): Promise { +): Promise<{ content: string; url: string }> { const newUrl = relativeUrl(params); try { const data = await params.read(newUrl); - return data.toString('utf-8'); + return { content: data.toString('utf-8'), url: newUrl }; } catch (e) { throw new Error( `Placeholder \$${params.key} could not read location ${params.value}, ${e}`, diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index dbcb9e34df..fb3f48c939 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -61,7 +61,13 @@ describe('UrlReaderProcessor', () => { server.use( rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => - res(ctx.set({ ETag: 'my-etag' }), ctx.json({ mock: 'entity' })), + res( + ctx.set({ ETag: 'my-etag' }), + ctx.json({ + kind: 'component', + metadata: { name: 'mock-url-entity' }, + }), + ), ), ); @@ -74,15 +80,25 @@ describe('UrlReaderProcessor', () => { mockCache, ); - expect(emitted.length).toBe(1); + expect(emitted.length).toBe(2); expect(emitted[0]).toEqual({ type: 'entity', location: spec, - entity: { mock: 'entity' }, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, + }); + expect(emitted[1]).toEqual({ + type: 'refresh', + key: 'url:http://localhost/component.yaml', }); expect(mockCache.set).toBeCalledWith('v1', { etag: 'my-etag', - value: [{ type: 'entity', location: spec, entity: { mock: 'entity' } }], + value: [ + { + type: 'entity', + location: spec, + entity: { kind: 'component', metadata: { name: 'mock-url-entity' } }, + }, + ], }); expect(mockCache.set).toBeCalledTimes(1); }); diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 7f27bcd19d..7b62690341 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -93,6 +93,8 @@ export class UrlReaderProcessor implements CatalogProcessor { value: parseResults as CatalogProcessorEntityResult[], }); } + + emit(processingResult.refresh(`${location.type}:${location.target}`)); } catch (error) { assertError(error); const message = `Unable to read ${location.type}, ${error}`; diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml index 8524aaf14a..b5844bd6ed 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml @@ -1 +1,3 @@ kind: Component +metadata: + name: component-test diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml index a894e34f70..ad33548f13 100644 --- a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml @@ -1 +1,3 @@ kind: API +metadata: + name: api-test diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts index 10de36a1be..3b6f126391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts @@ -30,6 +30,7 @@ describe('DefaultCatalogProcessingEngine', () => { updateProcessedEntity: jest.fn(), updateEntityCache: jest.fn(), listParents: jest.fn(), + setRefreshKeys: jest.fn(), } as unknown as jest.Mocked; const orchestrator: jest.Mocked = { process: jest.fn(), @@ -58,6 +59,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -123,6 +125,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( getVoidLogger(), @@ -203,6 +206,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); const engine = new DefaultCatalogProcessingEngine( @@ -413,6 +417,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }) .mockResolvedValueOnce({ ok: true, @@ -432,6 +437,7 @@ describe('DefaultCatalogProcessingEngine', () => { errors: [], deferredEntities: [], state: {}, + refreshKeys: [], }); await engine.start(); diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index b7ef5c9344..82062a8e2c 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -134,6 +134,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { .update(stableStringify({ ...result.completedEntity })) .update(stableStringify([...result.deferredEntities])) .update(stableStringify([...result.relations])) + .update(stableStringify([...result.refreshKeys])) .update(stableStringify([...parents])); } @@ -180,6 +181,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { relations: result.relations, deferredEntities: result.deferredEntities, locationKey, + refreshKeys: result.refreshKeys, }); oldRelationSources = new Set( previous.relations.map(r => r.source_entity_ref), diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 6e494d784e..b2d9846391 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -102,6 +102,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ok: true, completedEntity: entity, deferredEntities: [], + refreshKeys: [], errors: [], relations: [], state: { @@ -119,6 +120,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { ).resolves.toEqual({ ok: true, completedEntity: entity, + refreshKeys: [], deferredEntities: [ { locationKey: 'url:./new-place', diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index db7a050dce..033dfd4ae3 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -24,7 +24,7 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity } from './types'; +import { DeferredEntity, RefreshKeyData } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, @@ -38,6 +38,7 @@ export class ProcessorOutputCollector { private readonly errors = new Array(); private readonly relations = new Array(); private readonly deferredEntities = new Array(); + private readonly refreshKeys = new Array(); private done = false; constructor( @@ -54,6 +55,7 @@ export class ProcessorOutputCollector { return { errors: this.errors, relations: this.relations, + refreshKeys: this.refreshKeys, deferredEntities: this.deferredEntities, }; } @@ -116,6 +118,8 @@ export class ProcessorOutputCollector { this.relations.push(i.relation); } else if (i.type === 'error') { this.errors.push(i.error); + } else if (i.type === 'refresh') { + this.refreshKeys.push({ key: i.key }); } } } diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b178522c14..e94125b6f5 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -26,7 +26,6 @@ export type EntityProcessingRequest = { entity: Entity; state?: JsonObject; // Versions for multiple deployments etc }; - /** * The result of processing an entity. * @public @@ -38,6 +37,7 @@ export type EntityProcessingResult = completedEntity: Entity; deferredEntities: DeferredEntity[]; relations: EntityRelationSpec[]; + refreshKeys: RefreshKeyData[]; errors: Error[]; } | { @@ -45,6 +45,14 @@ export type EntityProcessingResult = errors: Error[]; }; +/** + * A string to associate to the entity itself. + * @public + */ +export type RefreshKeyData = { + key: string; +}; + /** * Responsible for executing the individual processing steps in order to fully process an entity. * @public diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts index bd166cbd50..8151bbf18d 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.test.ts @@ -48,6 +48,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -75,6 +76,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -134,6 +136,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -161,6 +164,7 @@ describe('DefaultLocationServiceTest', () => { name: 'foo', }, }, + refreshKeys: [], deferredEntities: [ { entity: { @@ -188,6 +192,7 @@ describe('DefaultLocationServiceTest', () => { }, }, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], }); @@ -211,6 +216,7 @@ describe('DefaultLocationServiceTest', () => { name: 'bar', }, }, + refreshKeys: [], deferredEntities: [], relations: [], errors: [], diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 84bfde1259..419aedc15d 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -138,6 +138,7 @@ describe('Refresh integration', () => { errors: [], deferredEntities, state: {}, + refreshKeys: [], }; }, }, diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 8eafd8bfa3..16a36eb8c4 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -409,6 +409,7 @@ describe('createRouter readonly disabled', () => { state: {}, completedEntity: entity, deferredEntities: [], + refreshKeys: [], relations: [], errors: [], });