diff --git a/.changeset/fifty-humans-repair.md b/.changeset/fifty-humans-repair.md new file mode 100644 index 0000000000..28db460d63 --- /dev/null +++ b/.changeset/fifty-humans-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added support for emitting entities with an explicit location key during processing. diff --git a/.changeset/hip-hotels-type.md b/.changeset/hip-hotels-type.md new file mode 100644 index 0000000000..6c2d1dddb9 --- /dev/null +++ b/.changeset/hip-hotels-type.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed an bug in the entity processing caching that would prevent entities that were emitted during processing to be restored after being overridden. diff --git a/.changeset/nine-ears-destroy.md b/.changeset/nine-ears-destroy.md new file mode 100644 index 0000000000..c987fd80ff --- /dev/null +++ b/.changeset/nine-ears-destroy.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-node': minor +--- + +Added the `locationKey` option to `processingResult.entity(...)`. diff --git a/plugins/catalog-backend/report.api.md b/plugins/catalog-backend/report.api.md index a1391e9b44..a6389185a7 100644 --- a/plugins/catalog-backend/report.api.md +++ b/plugins/catalog-backend/report.api.md @@ -479,6 +479,11 @@ export const processingResult: Readonly<{ readonly entity: ( atLocation: LocationSpec_2, newEntity: Entity, + options?: + | { + locationKey?: string | null | undefined; + } + | undefined, ) => CatalogProcessorResult_2; readonly relation: (spec: EntityRelationSpec_2) => CatalogProcessorResult_2; readonly refresh: (key: string) => CatalogProcessorResult_2; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index e530c30d8b..750eebbadb 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -784,7 +784,7 @@ describe('DefaultProcessingDatabase', () => { }); const result1 = await db.transaction(async tx => - db.listParents(tx, { entityRef: 'component:default/foobar' }), + db.listParents(tx, { entityRefs: ['component:default/foobar'] }), ); expect(result1.entityRefs).toEqual([ 'location:default/root-1', @@ -792,12 +792,12 @@ describe('DefaultProcessingDatabase', () => { ]); const result2 = await db.transaction(async tx => - db.listParents(tx, { entityRef: 'location:default/root-1' }), + db.listParents(tx, { entityRefs: ['location:default/root-1'] }), ); expect(result2.entityRefs).toEqual(['location:default/root-2']); const result3 = await db.transaction(async tx => - db.listParents(tx, { entityRef: 'location:default/root-2' }), + db.listParents(tx, { entityRefs: ['location:default/root-2'] }), ); expect(result3.entityRefs).toEqual([]); }, diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index bb5c38de18..0aa31dd36d 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -272,7 +272,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const rows = await tx( 'refresh_state_references', ) - .where({ target_entity_ref: options.entityRef }) + .whereIn('target_entity_ref', options.entityRefs) .select(); const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean); diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ad2be8e06c..82a1ac3f6b 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -101,7 +101,7 @@ export type ListAncestorsResult = { }; export type ListParentsOptions = { - entityRef: string; + entityRefs: string[]; }; export type ListParentsResult = { diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 59843ac430..a3ef926b47 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -215,7 +215,12 @@ export class DefaultCatalogProcessingEngine { const { entityRefs: parents } = await this.processingDatabase.transaction(tx => this.processingDatabase.listParents(tx, { - entityRef, + entityRefs: [ + entityRef, + ...result.deferredEntities.map(e => + stringifyEntityRef(e.entity), + ), + ], }), ); diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 0de6b12b90..ee488fc1d5 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -137,7 +137,9 @@ export class ProcessorOutputCollector { }, }; - this.deferredEntities.push({ entity, locationKey: location }); + const locationKey = + i.locationKey === undefined ? location : i.locationKey ?? undefined; + this.deferredEntities.push({ entity, locationKey }); } else if (i.type === 'location') { const entity = locationSpecToLocationEntity({ location: i.location, diff --git a/plugins/catalog-backend/src/tests/integration.test.ts b/plugins/catalog-backend/src/tests/integration.test.ts index 1c134d4c60..aa6d241d23 100644 --- a/plugins/catalog-backend/src/tests/integration.test.ts +++ b/plugins/catalog-backend/src/tests/integration.test.ts @@ -1166,4 +1166,151 @@ describe('Catalog Backend Integration', () => { 'component:default/b': withOutputFields(entityB), }); }); + + it('should be able to emit entities during processing with custom location keys', async () => { + const baseEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:.', + 'backstage.io/managed-by-origin-location': 'url:.', + }, + }, + }; + const entityA = merge({ metadata: { name: 'a' } }, baseEntity); + const entityB = merge({ metadata: { name: 'b' } }, baseEntity); + const entityBOverride = merge({ metadata: { override: true } }, entityB); + + const processEntity = jest.fn( + async ( + entity: Entity, + _location: LocationSpec, + _emit: CatalogProcessorEmit, + ) => entity, + ); + const harness = await TestHarness.create({ processEntity }); + + processEntity.mockImplementation(async (entity, location, emit) => { + if (entity.metadata.name === entityA.metadata.name) { + emit( + processingResult.entity(location, entityB, { + locationKey: null, + }), + ); + } + return entity; + }); + + // Start with just A, which outputs B via processing + await harness.setInputEntities([entityA]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields(entityB), + }); + + // Now apply A and B', which should override B since it has a null location key + await harness.setInputEntities([ + entityA, + { ...entityBOverride, locationKey: 'test' }, + ]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields(entityBOverride), + }); + }); + + it('should resolve handle location key conflicts across processed and provided entities', async () => { + const baseEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'url:.', + 'backstage.io/managed-by-origin-location': 'url:.', + }, + }, + }; + const entityA = merge({ metadata: { name: 'a' } }, baseEntity); + const entityB = merge({ metadata: { name: 'b' } }, baseEntity); + const entityBOverride = merge({ metadata: { override: true } }, entityB); + + const processEntity = jest.fn( + async ( + entity: Entity, + _location: LocationSpec, + _emit: CatalogProcessorEmit, + ) => entity, + ); + const harness = await TestHarness.create({ processEntity }); + + processEntity.mockImplementation(async (entity, location, emit) => { + if (entity.metadata.name === entityA.metadata.name) { + emit( + processingResult.entity(location, entityB, { + locationKey: 'my-key', + }), + ); + } + return entity; + }); + + // Start with just A, which outputs B via processing + await harness.setInputEntities([entityA]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields(entityB), + }); + + // Now apply A and B', but since B already has a different location key, B' should not be used + await harness.setInputEntities([entityA, entityBOverride]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields(entityB), + }); + + // Stop emitting B from A + processEntity.mockImplementation(async e => e); + + // Apply A and B', but since B still has a location key set it should remain, but now orphaned + await harness.setInputEntities([entityA, entityBOverride]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields( + merge(entityB, { + metadata: { annotations: { 'backstage.io/orphan': 'true' } }, + }), + ), + }); + + // Apply A and B' but this time with the matching location key, causing B' to override B + await harness.setInputEntities([ + entityA, + { ...entityBOverride, locationKey: 'my-key' }, + ]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + 'component:default/b': withOutputFields(entityBOverride), + }); + + // Finally, remove B', leaving just A + await harness.setInputEntities([entityA]); + await expect(harness.process()).resolves.toEqual({}); + + await expect(harness.getOutputEntities()).resolves.toEqual({ + 'component:default/a': withOutputFields(entityA), + }); + }); }); diff --git a/plugins/catalog-node/report.api.md b/plugins/catalog-node/report.api.md index e196fff230..ddfac756b4 100644 --- a/plugins/catalog-node/report.api.md +++ b/plugins/catalog-node/report.api.md @@ -76,6 +76,7 @@ export type CatalogProcessorEntityResult = { type: 'entity'; entity: Entity; location: LocationSpec_2; + locationKey?: string | null; }; // @public (undocumented) @@ -334,6 +335,9 @@ export const processingResult: Readonly<{ readonly entity: ( atLocation: LocationSpec_2, newEntity: Entity, + options?: { + locationKey?: string | null; + }, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; readonly refresh: (key: string) => CatalogProcessorResult; diff --git a/plugins/catalog-node/src/api/processingResult.ts b/plugins/catalog-node/src/api/processingResult.ts index 0d4341fcb7..021160d347 100644 --- a/plugins/catalog-node/src/api/processingResult.ts +++ b/plugins/catalog-node/src/api/processingResult.ts @@ -76,8 +76,24 @@ export const processingResult = Object.freeze({ /** * Emits a child of the current entity, associated with a certain location. */ - entity(atLocation: LocationSpec, newEntity: Entity): CatalogProcessorResult { - return { type: 'entity', location: atLocation, entity: newEntity }; + entity( + atLocation: LocationSpec, + newEntity: Entity, + options?: { + /** + * Sets the location key of the emitted entity, overriding the default one derived from the location. + * + * To set a `null` location key the value `null` must be used. + */ + locationKey?: string | null; + }, + ): CatalogProcessorResult { + return { + type: 'entity', + location: atLocation, + entity: newEntity, + locationKey: options?.locationKey, + }; }, /** diff --git a/plugins/catalog-node/src/api/processor.ts b/plugins/catalog-node/src/api/processor.ts index 3ad080f014..e43fdef071 100644 --- a/plugins/catalog-node/src/api/processor.ts +++ b/plugins/catalog-node/src/api/processor.ts @@ -155,6 +155,7 @@ export type CatalogProcessorEntityResult = { type: 'entity'; entity: Entity; location: LocationSpec; + locationKey?: string | null; }; /** @public */