Merge pull request #28642 from backstage/rugvip/processing

catalog-backend: allow processors to include location key when emitting entities

.changeset/fifty-humans-repair.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added support for emitting entities with an explicit location key during processing.

.changeset/hip-hotels-type.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Fixed an bug in the entity processing caching that would prevent entities that were emitted during processing to be restored after being overridden.

.changeset/nine-ears-destroy.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-node': minor
+---
+
+Added the `locationKey` option to `processingResult.entity(...)`.

plugins/catalog-backend/report.api.md

@@ -479,6 +479,11 @@ export const processingResult: Readonly<{
   readonly entity: (
     atLocation: LocationSpec_2,
     newEntity: Entity,
+    options?:
+      | {
+          locationKey?: string | null | undefined;
+        }
+      | undefined,
   ) => CatalogProcessorResult_2;
   readonly relation: (spec: EntityRelationSpec_2) => CatalogProcessorResult_2;
   readonly refresh: (key: string) => CatalogProcessorResult_2;

plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts

@@ -784,7 +784,7 @@ describe('DefaultProcessingDatabase', () => {
         });
 
         const result1 = await db.transaction(async tx =>
-          db.listParents(tx, { entityRef: 'component:default/foobar' }),
+          db.listParents(tx, { entityRefs: ['component:default/foobar'] }),
         );
         expect(result1.entityRefs).toEqual([
           'location:default/root-1',
@@ -792,12 +792,12 @@ describe('DefaultProcessingDatabase', () => {
         ]);
 
         const result2 = await db.transaction(async tx =>
-          db.listParents(tx, { entityRef: 'location:default/root-1' }),
+          db.listParents(tx, { entityRefs: ['location:default/root-1'] }),
         );
         expect(result2.entityRefs).toEqual(['location:default/root-2']);
 
         const result3 = await db.transaction(async tx =>
-          db.listParents(tx, { entityRef: 'location:default/root-2' }),
+          db.listParents(tx, { entityRefs: ['location:default/root-2'] }),
         );
         expect(result3.entityRefs).toEqual([]);
       },

plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts

@@ -272,7 +272,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     const rows = await tx<DbRefreshStateReferencesRow>(
       'refresh_state_references',
     )
-      .where({ target_entity_ref: options.entityRef })
+      .whereIn('target_entity_ref', options.entityRefs)
       .select();
 
     const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);

plugins/catalog-backend/src/database/types.ts

@@ -101,7 +101,7 @@ export type ListAncestorsResult = {
 };
 
 export type ListParentsOptions = {
-  entityRef: string;
+  entityRefs: string[];
 };
 
 export type ListParentsResult = {

plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts

@@ -215,7 +215,12 @@ export class DefaultCatalogProcessingEngine {
               const { entityRefs: parents } =
                 await this.processingDatabase.transaction(tx =>
                   this.processingDatabase.listParents(tx, {
-                    entityRef,
+                    entityRefs: [
+                      entityRef,
+                      ...result.deferredEntities.map(e =>
+                        stringifyEntityRef(e.entity),
+                      ),
+                    ],
                   }),
                 );
 

plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts

@@ -137,7 +137,9 @@ export class ProcessorOutputCollector {
         },
       };
 
-      this.deferredEntities.push({ entity, locationKey: location });
+      const locationKey =
+        i.locationKey === undefined ? location : i.locationKey ?? undefined;
+      this.deferredEntities.push({ entity, locationKey });
     } else if (i.type === 'location') {
       const entity = locationSpecToLocationEntity({
         location: i.location,

plugins/catalog-backend/src/tests/integration.test.ts

@@ -1166,4 +1166,151 @@ describe('Catalog Backend Integration', () => {
       'component:default/b': withOutputFields(entityB),
     });
   });
+
+  it('should be able to emit entities during processing with custom location keys', async () => {
+    const baseEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        annotations: {
+          'backstage.io/managed-by-location': 'url:.',
+          'backstage.io/managed-by-origin-location': 'url:.',
+        },
+      },
+    };
+    const entityA = merge({ metadata: { name: 'a' } }, baseEntity);
+    const entityB = merge({ metadata: { name: 'b' } }, baseEntity);
+    const entityBOverride = merge({ metadata: { override: true } }, entityB);
+
+    const processEntity = jest.fn(
+      async (
+        entity: Entity,
+        _location: LocationSpec,
+        _emit: CatalogProcessorEmit,
+      ) => entity,
+    );
+    const harness = await TestHarness.create({ processEntity });
+
+    processEntity.mockImplementation(async (entity, location, emit) => {
+      if (entity.metadata.name === entityA.metadata.name) {
+        emit(
+          processingResult.entity(location, entityB, {
+            locationKey: null,
+          }),
+        );
+      }
+      return entity;
+    });
+
+    // Start with just A, which outputs B via processing
+    await harness.setInputEntities([entityA]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(entityB),
+    });
+
+    // Now apply A and B', which should override B since it has a null location key
+    await harness.setInputEntities([
+      entityA,
+      { ...entityBOverride, locationKey: 'test' },
+    ]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(entityBOverride),
+    });
+  });
+
+  it('should resolve handle location key conflicts across processed and provided entities', async () => {
+    const baseEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        annotations: {
+          'backstage.io/managed-by-location': 'url:.',
+          'backstage.io/managed-by-origin-location': 'url:.',
+        },
+      },
+    };
+    const entityA = merge({ metadata: { name: 'a' } }, baseEntity);
+    const entityB = merge({ metadata: { name: 'b' } }, baseEntity);
+    const entityBOverride = merge({ metadata: { override: true } }, entityB);
+
+    const processEntity = jest.fn(
+      async (
+        entity: Entity,
+        _location: LocationSpec,
+        _emit: CatalogProcessorEmit,
+      ) => entity,
+    );
+    const harness = await TestHarness.create({ processEntity });
+
+    processEntity.mockImplementation(async (entity, location, emit) => {
+      if (entity.metadata.name === entityA.metadata.name) {
+        emit(
+          processingResult.entity(location, entityB, {
+            locationKey: 'my-key',
+          }),
+        );
+      }
+      return entity;
+    });
+
+    // Start with just A, which outputs B via processing
+    await harness.setInputEntities([entityA]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(entityB),
+    });
+
+    // Now apply A and B', but since B already has a different location key, B' should not be used
+    await harness.setInputEntities([entityA, entityBOverride]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(entityB),
+    });
+
+    // Stop emitting B from A
+    processEntity.mockImplementation(async e => e);
+
+    // Apply A and B', but since B still has a location key set it should remain, but now orphaned
+    await harness.setInputEntities([entityA, entityBOverride]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(
+        merge(entityB, {
+          metadata: { annotations: { 'backstage.io/orphan': 'true' } },
+        }),
+      ),
+    });
+
+    // Apply A and B' but this time with the matching location key, causing B' to override B
+    await harness.setInputEntities([
+      entityA,
+      { ...entityBOverride, locationKey: 'my-key' },
+    ]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+      'component:default/b': withOutputFields(entityBOverride),
+    });
+
+    // Finally, remove B', leaving just A
+    await harness.setInputEntities([entityA]);
+    await expect(harness.process()).resolves.toEqual({});
+
+    await expect(harness.getOutputEntities()).resolves.toEqual({
+      'component:default/a': withOutputFields(entityA),
+    });
+  });
 });

plugins/catalog-node/report.api.md

@@ -76,6 +76,7 @@ export type CatalogProcessorEntityResult = {
   type: 'entity';
   entity: Entity;
   location: LocationSpec_2;
+  locationKey?: string | null;
 };
 
 // @public (undocumented)
@@ -334,6 +335,9 @@ export const processingResult: Readonly<{
   readonly entity: (
     atLocation: LocationSpec_2,
     newEntity: Entity,
+    options?: {
+      locationKey?: string | null;
+    },
   ) => CatalogProcessorResult;
   readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult;
   readonly refresh: (key: string) => CatalogProcessorResult;

plugins/catalog-node/src/api/processingResult.ts

@@ -76,8 +76,24 @@ export const processingResult = Object.freeze({
   /**
    * Emits a child of the current entity, associated with a certain location.
    */
-  entity(atLocation: LocationSpec, newEntity: Entity): CatalogProcessorResult {
-    return { type: 'entity', location: atLocation, entity: newEntity };
+  entity(
+    atLocation: LocationSpec,
+    newEntity: Entity,
+    options?: {
+      /**
+       * Sets the location key of the emitted entity, overriding the default one derived from the location.
+       *
+       * To set a `null` location key the value `null` must be used.
+       */
+      locationKey?: string | null;
+    },
+  ): CatalogProcessorResult {
+    return {
+      type: 'entity',
+      location: atLocation,
+      entity: newEntity,
+      locationKey: options?.locationKey,
+    };
   },
 
   /**

plugins/catalog-node/src/api/processor.ts

@@ -155,6 +155,7 @@ export type CatalogProcessorEntityResult = {
   type: 'entity';
   entity: Entity;
   location: LocationSpec;
+  locationKey?: string | null;
 };
 
 /** @public */