diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index e3828bdfd8..0ae20c94b1 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -125,7 +125,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { }, processTask: async item => { try { - const { id, state, unprocessedEntity, entityRef, emitKey } = item; + const { id, state, unprocessedEntity, entityRef, locationKey } = item; const result = await this.orchestrator.process({ entity: unprocessedEntity, state, @@ -171,7 +171,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { errors: errorsString, relations: result.relations, deferredEntities: result.deferredEntities, - emitKey, + locationKey, }); }); diff --git a/plugins/catalog-backend/src/next/DefaultLocationService.ts b/plugins/catalog-backend/src/next/DefaultLocationService.ts index ceb51a952e..70679a3bd6 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationService.ts @@ -20,7 +20,10 @@ import { LOCATION_ANNOTATION, ORIGIN_LOCATION_ANNOTATION, } from '@backstage/catalog-model'; -import { CatalogProcessingOrchestrator } from './processing/types'; +import { + CatalogProcessingOrchestrator, + DeferredEntity, +} from './processing/types'; import { LocationService, LocationStore } from './types'; import { locationSpecToMetadataName } from './util'; @@ -73,7 +76,9 @@ export class DefaultLocationService implements LocationService { target: spec.target, }, }; - const unprocessedEntities: Entity[] = [entity]; + const unprocessedEntities: DeferredEntity[] = [ + { entity, locationKey: `${spec.type}:${spec.target}` }, + ]; const entities: Entity[] = []; const state = new Map(); // ignored while (unprocessedEntities.length) { @@ -82,11 +87,12 @@ export class DefaultLocationService implements LocationService { continue; } const processed = await this.orchestrator.process({ - entity: currentEntity, + entity: currentEntity.entity, state, }); if (processed.ok) { + // todo unprocessedEntities.push(...processed.deferredEntities); entities.push(processed.completedEntity); } else { diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.ts index 5c3cefbaf7..c9ffe9a682 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.ts @@ -64,7 +64,7 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { const entity = locationSpecToLocationEntity(location); await this.connection.applyMutation({ type: 'delta', - added: [{ entity, emitKey: getEntityLocationRef(entity) }], + added: [{ entity, locationKey: getEntityLocationRef(entity) }], removed: [], }); @@ -107,7 +107,7 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { await this.connection.applyMutation({ type: 'delta', added: [], - removed: [{ entity, emitKey: getEntityLocationRef(entity) }], + removed: [{ entity, locationKey: getEntityLocationRef(entity) }], }); } @@ -126,7 +126,7 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { const entities = locations.map(location => { const entity = locationSpecToLocationEntity(location); - return { entity, emitKey: getEntityLocationRef(entity) }; + return { entity, locationKey: getEntityLocationRef(entity) }; }); await this.connection.applyMutation({ diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 5bdde1fe78..c2bfb4f633 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -64,18 +64,18 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { errors, relations, deferredEntities, - emitKey, + locationKey, } = options; const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), cache: JSON.stringify(state), errors, - emit_key: emitKey, + location_key: locationKey, }) .where('entity_id', id) - .andWhere('emit_key', emitKey) - .orWhere('emit_key', ''); + .andWhere('location_key', locationKey) + .orWhere('location_key', ''); if (refreshResult === 0) { throw new NotFoundError(`Processing state not found for ${id}`); } @@ -286,11 +286,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { if (toAdd.length) { const state: Knex.DbRecord[] = toAdd.map( - ({ entity, emitKey }) => ({ + ({ entity, locationKey }) => ({ entity_id: uuid(), entity_ref: stringifyEntityRef(entity), unprocessed_entity: JSON.stringify(entity), - emit_key: emitKey, + location_key: locationKey, errors: '', next_update_at: tx.fn.now(), last_discovery_at: tx.fn.now(), @@ -320,13 +320,13 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const tx = txOpaque as Knex.Transaction; const stateRows = options.entities.map( - ({ entity, emitKey }) => + ({ entity, locationKey }) => ({ entity_id: uuid(), entity_ref: stringifyEntityRef(entity), unprocessed_entity: JSON.stringify(entity), errors: '', - emit_key: emitKey, + location_key: locationKey, next_update_at: tx.fn.now(), last_discovery_at: tx.fn.now(), } as Knex.DbRecord), @@ -414,7 +414,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ? JSON.parse(i.cache) : new Map(), errors: i.errors, - emitKey: i.emit_key, + locationKey: i.location_key, } as RefreshStateItem), ), }; diff --git a/plugins/catalog-backend/src/next/database/tables.ts b/plugins/catalog-backend/src/next/database/tables.ts index a298c37022..395694131d 100644 --- a/plugins/catalog-backend/src/next/database/tables.ts +++ b/plugins/catalog-backend/src/next/database/tables.ts @@ -29,7 +29,7 @@ export type DbRefreshStateRow = { next_update_at: string; last_discovery_at: string; // remove? errors?: string; - emit_key: string; + location_key?: string; }; export type DbRefreshStateReferencesRow = { diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 5e4d0becd3..cd5d2a362d 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -33,7 +33,7 @@ export type UpdateProcessedEntityOptions = { errors?: string; relations: EntityRelationSpec[]; deferredEntities: DeferredEntity[]; - emitKey: string; + locationKey?: string; }; export type UpdateProcessedEntityErrorsOptions = { @@ -50,7 +50,7 @@ export type RefreshStateItem = { lastDiscoveryAt: string; // remove? state: Map; errors?: string; - emitKey: string; + locationKey?: string; }; export type GetProcessableEntitiesResult = { diff --git a/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts index fbb2744668..507def964a 100644 --- a/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/next/processing/ProcessorOutputCollector.ts @@ -101,14 +101,14 @@ export class ProcessorOutputCollector { }; } - this.deferredEntities.push({ entity, emitKey: location }); + this.deferredEntities.push({ entity, locationKey: location }); } else if (i.type === 'location') { const entity = locationSpecToLocationEntity( i.location, this.parentEntity, ); - const emitKey = getEntityLocationRef(entity); - this.deferredEntities.push({ entity, emitKey }); + const locationKey = getEntityLocationRef(entity); + this.deferredEntities.push({ entity, locationKey }); } else if (i.type === 'relation') { this.relations.push(i.relation); } else if (i.type === 'error') { diff --git a/plugins/catalog-backend/src/next/processing/types.ts b/plugins/catalog-backend/src/next/processing/types.ts index b2a627943b..9e30d07404 100644 --- a/plugins/catalog-backend/src/next/processing/types.ts +++ b/plugins/catalog-backend/src/next/processing/types.ts @@ -42,5 +42,5 @@ export interface CatalogProcessingOrchestrator { export type DeferredEntity = { entity: Entity; - emitKey: string; + locationKey?: string; };