diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 708c3e11aa..e806ecfbdb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -16,6 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { + Entity, EntityPolicies, EntityPolicy, LocationSpec, @@ -29,11 +30,11 @@ import * as result from './processors/results'; import { LocationProcessor, LocationProcessorDataResult, + LocationProcessorEmit, LocationProcessorEntityResult, LocationProcessorErrorResult, LocationProcessorLocationResult, LocationProcessorResult, - LocationProcessorSink, } from './processors/types'; import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; @@ -74,17 +75,25 @@ export class LocationReaders implements LocationReader { for (let depth = 0; depth < MAX_DEPTH; ++depth) { const newItems: LocationProcessorResult[] = []; - const sink: LocationProcessorSink = i => newItems.push(i); + const emit: LocationProcessorEmit = i => newItems.push(i); for (const item of items) { if (item.type === 'location') { - await this.handleLocation(item, sink); + await this.handleLocation(item, emit); } else if (item.type === 'data') { - await this.handleData(item, sink); + await this.handleData(item, emit); } else if (item.type === 'entity') { - await this.handleEntity(item, sink, output); + const entity = await this.handleEntity(item, emit); + output.entities.push({ + entity, + location: item.location, + }); } else if (item.type === 'error') { - await this.handleError(item, sink, output); + await this.handleError(item, emit); + output.errors.push({ + location: item.location, + error: item.error, + }); } } @@ -103,7 +112,7 @@ export class LocationReaders implements LocationReader { private async handleLocation( item: LocationProcessorLocationResult, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ) { this.logger.debug( `Reading location ${item.location.type} ${item.location.target} optional=${item.optional}`, @@ -130,7 +139,7 @@ export class LocationReaders implements LocationReader { private async handleData( item: LocationProcessorDataResult, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ) { this.logger.debug( `Parsing data from location ${item.location.type} ${item.location.target} (${item.data.byteLength} bytes)`, @@ -155,9 +164,8 @@ export class LocationReaders implements LocationReader { private async handleEntity( item: LocationProcessorEntityResult, - emit: LocationProcessorSink, - output: ReadLocationResult, - ) { + emit: LocationProcessorEmit, + ): Promise { this.logger.debug( `Got entity at location ${item.location.type} ${item.location.target}, ${item.entity.apiVersion} ${item.entity.kind}`, ); @@ -175,13 +183,12 @@ export class LocationReaders implements LocationReader { } } - output.entities.push({ entity: current, location: item.location }); + return current; } private async handleError( item: LocationProcessorErrorResult, - emit: LocationProcessorSink, - output: ReadLocationResult, + emit: LocationProcessorEmit, ) { this.logger.debug( `Encountered error at location ${item.location.type} ${item.location.target}, ${item.error}`, @@ -197,10 +204,5 @@ export class LocationReaders implements LocationReader { } } } - - output.errors.push({ - location: item.location, - error: item.error, - }); } } diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index dc9bce56f2..07e917b97b 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -17,10 +17,10 @@ export { HigherOrderOperations } from './HigherOrderOperations'; export { LocationReaders } from './LocationReaders'; export type { - HigherOrderOperation, AddLocationResult, + HigherOrderOperation, LocationReader, - ReadLocationResult, ReadLocationEntity, ReadLocationError, + ReadLocationResult, } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts index a3d24575bc..b95e2ddb58 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts @@ -17,13 +17,13 @@ import { LocationSpec } from '@backstage/catalog-model'; import fs from 'fs-extra'; import * as result from './results'; -import { LocationProcessor, LocationProcessorSink } from './types'; +import { LocationProcessor, LocationProcessorEmit } from './types'; export class FileReaderProcessor implements LocationProcessor { async readLocation( location: LocationSpec, optional: boolean, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise { if (location.type !== 'file') { return false; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index a39380027b..b83c9a16f3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -17,13 +17,13 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch from 'node-fetch'; import * as result from './results'; -import { LocationProcessor, LocationProcessorSink } from './types'; +import { LocationProcessor, LocationProcessorEmit } from './types'; export class GithubReaderProcessor implements LocationProcessor { async readLocation( location: LocationSpec, optional: boolean, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise { if (location.type !== 'github') { return false; diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 1faf77646a..6a2b5cf419 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -18,13 +18,13 @@ import { Entity, LocationSpec } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; import * as result from './results'; -import { LocationProcessor, LocationProcessorSink } from './types'; +import { LocationProcessor, LocationProcessorEmit } from './types'; export class YamlProcessor implements LocationProcessor { async parseData( data: Buffer, location: LocationSpec, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise { if (!location.target.match(/\.ya?ml$/)) { return false; diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 43e5480029..c8f3f6f482 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { LocationSpec, Entity } from '@backstage/catalog-model'; +import { Entity, LocationSpec } from '@backstage/catalog-model'; export type LocationProcessor = { /** @@ -28,7 +28,7 @@ export type LocationProcessor = { readLocation?( location: LocationSpec, optional: boolean, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise; /** @@ -42,7 +42,7 @@ export type LocationProcessor = { parseData?( data: Buffer, location: LocationSpec, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise; /** @@ -56,7 +56,7 @@ export type LocationProcessor = { processEntity?( entity: Entity, location: LocationSpec, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise; /** @@ -70,11 +70,11 @@ export type LocationProcessor = { handleError?( error: Error, location: LocationSpec, - emit: LocationProcessorSink, + emit: LocationProcessorEmit, ): Promise; }; -export type LocationProcessorSink = ( +export type LocationProcessorEmit = ( generated: LocationProcessorResult, ) => void;