From 07de96af3be4ecd6406193014faa885bb4fa3ee0 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Apr 2021 11:25:48 +0200 Subject: [PATCH] Backwards compatible processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- .../next/CatalogProcessingOrchestratorImpl.ts | 152 ++++++++++++++++-- .../src/next/DatabaseLocationProvider.ts | 20 ++- .../src/next/LocationServiceImpl.ts | 12 +- .../src/next/LocationToEntity.ts | 39 ----- plugins/catalog-backend/src/next/util.ts | 90 +++++++++++ 5 files changed, 254 insertions(+), 59 deletions(-) delete mode 100644 plugins/catalog-backend/src/next/LocationToEntity.ts create mode 100644 plugins/catalog-backend/src/next/util.ts diff --git a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts index 1288561b07..9aaa2ffa74 100644 --- a/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts +++ b/plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts @@ -19,7 +19,12 @@ import { EntityRelationSpec, stringifyEntityRef, LOCATION_ANNOTATION, + LocationSpec, + LocationEntity, EntityPolicy, + ORIGIN_LOCATION_ANNOTATION, + stringifyLocationReference, + parseLocationReference, } from '@backstage/catalog-model'; import { CatalogProcessor, @@ -33,12 +38,56 @@ import { } from './types'; import { Logger } from 'winston'; import { InputError } from '@backstage/errors'; +import { locationSpecToLocationEntity } from './util'; +import path from 'path'; +import * as results from '../ingestion/processors/results'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +function isLocationEntity(entity: Entity): entity is LocationEntity { + return entity.kind === 'Location'; +} + +function getEntityOriginLocationRef(entity: Entity): string { + const ref = entity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION]; + if (!ref) { + const entityRef = stringifyEntityRef(entity); + throw new InputError( + `Entity '${entityRef}' does not have an origin location`, + ); + } + return ref; +} + +function toAbsoluteUrl( + integrations: ScmIntegrationRegistry, + base: LocationSpec, + type: string, + target: string, +): string { + if (base.type !== type) { + return target; + } + try { + if (type === 'file') { + if (target.startsWith('.')) { + return path.join(path.dirname(base.target), target); + } + return target; + } else if (type === 'url') { + return integrations.resolveUrl({ url: target, base: base.target }); + } + return target; + } catch (e) { + return target; + } +} export class CatalogProcessingOrchestratorImpl implements CatalogProcessingOrchestrator { constructor( private readonly options: { processors: CatalogProcessor[]; + integrations: ScmIntegrationRegistry; logger: Logger; parser: CatalogProcessorParser; policy: EntityPolicy; @@ -52,7 +101,7 @@ export class CatalogProcessingOrchestratorImpl const result = await this.processSingleEntity(entity); - console.log('result', JSON.stringify(result)); + console.log('result', JSON.stringify(result, undefined, 2)); return result; } @@ -64,19 +113,27 @@ export class CatalogProcessingOrchestratorImpl // TODO: which one do we actually use here? source-location? - maybe probably doesn't exist yet? const locationRef = unprocessedEntity.metadata?.annotations?.[LOCATION_ANNOTATION]; + if (!locationRef) { + throw new InputError(`Entity '${entityRef}' does not have a location`); + } + const location = parseLocationReference(locationRef); + const originLocation = parseLocationReference( + getEntityOriginLocationRef(unprocessedEntity), + ); - const emitter = createEmitter(this.options.logger); - + const emitter = createEmitter(this.options.logger, unprocessedEntity); try { // Pre-process phase, used to populate entities with data that is required during main processing step let entity = unprocessedEntity; for (const processor of this.options.processors) { if (processor.preProcessEntity) { try { - entity = await processor.preProcessEntity({ + entity = await processor.preProcessEntity( entity, - emit: emitter.emit, - }); + location, + emitter.emit, + originLocation, + ); } catch (e) { throw new Error( `Processor ${processor.constructor.name} threw an error while preprocessing entity ${entityRef} at ${locationRef}, ${e}`, @@ -123,14 +180,69 @@ export class CatalogProcessingOrchestratorImpl ); } + // Backwards compatible processing of location entites + if (isLocationEntity(entity)) { + const { type = location.type, optional = false } = entity.spec; + const targets = new Array(); + if (entity.spec.target) { + targets.push(entity.spec.target); + } + if (entity.spec.targets) { + targets.push(...entity.spec.targets); + } + + for (const maybeRelativeTarget of targets) { + if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { + emitter.emit( + results.inputError( + location, + `LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, + ), + ); + continue; + } + const target = toAbsoluteUrl( + this.options.integrations, + location, + type, + maybeRelativeTarget, + ); + + for (const processor of this.options.processors) { + if (processor.readLocation) { + try { + handled = await processor.readLocation( + { + type, + target, + presence: optional ? 'optional' : 'required', + }, + Boolean(entity.spec?.optional), + emitter.emit, + this.options.parser, + ); + if (handled) { + break; + } + } catch (e) { + throw new Error( + `Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`, + ); + } + } + } + } + } + // Main processing step of the entity for (const processor of this.options.processors) { - if (processor.processEntity) { + if (processor.postProcessEntity) { try { - entity = await processor.processEntity({ + entity = await processor.postProcessEntity( entity, - emit: emitter.emit, - }); + location, + emitter.emit, + ); } catch (e) { throw new Error( `Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`, @@ -152,7 +264,7 @@ export class CatalogProcessingOrchestratorImpl } } -function createEmitter(logger: Logger) { +function createEmitter(logger: Logger, parentEntity: Entity) { let done = false; const errors = new Array(); @@ -170,7 +282,23 @@ function createEmitter(logger: Logger) { return; } if (i.type === 'entity') { - deferredEntites.push(i.entity); + const originLocation = getEntityOriginLocationRef(parentEntity); + + deferredEntites.push({ + ...i.entity, + metadata: { + ...i.entity.metadata, + annotations: { + ...i.entity.metadata.annotations, + [ORIGIN_LOCATION_ANNOTATION]: originLocation, + [LOCATION_ANNOTATION]: stringifyLocationReference(i.location), + }, + }, + }); + } else if (i.type === 'location') { + deferredEntites.push( + locationSpecToLocationEntity(i.location, parentEntity), + ); } else if (i.type === 'relation') { relations.push(i.relation); } else if (i.type === 'error') { diff --git a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts index b56dfa1158..eaf5a78556 100644 --- a/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts +++ b/plugins/catalog-backend/src/next/DatabaseLocationProvider.ts @@ -15,9 +15,13 @@ */ import { Observable } from '@backstage/core'; +import { ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { EntityProvider, LocationStore, EntityMessage } from './types'; import ObservableImpl from 'zen-observable'; -import { locationToEntity, locationToEntityName } from './LocationToEntity'; +import { + locationSpecToLocationEntity, + locationSpecToMetadataName, +} from './util'; export class DatabaseLocationProvider implements EntityProvider { private subscribers = new Set< @@ -29,14 +33,16 @@ export class DatabaseLocationProvider implements EntityProvider { next: locations => { if ('all' in locations) { this.notify({ - all: locations.all.map(l => locationToEntity(l.type, l.target)), + all: locations.all.map(l => locationSpecToLocationEntity(l)), }); } else { this.notify({ - added: locations.added.map(l => locationToEntity(l.type, l.target)), - removed: locations.removed.map(l => - locationToEntityName(l.type, l.target), - ), + added: locations.added.map(l => locationSpecToLocationEntity(l)), + removed: locations.removed.map(l => ({ + kind: 'Location', + namespace: ENTITY_DEFAULT_NAMESPACE, + name: locationSpecToMetadataName(l), + })), }); } }, @@ -53,7 +59,7 @@ export class DatabaseLocationProvider implements EntityProvider { return new ObservableImpl(subscriber => { this.store.listLocations().then(locations => { subscriber.next({ - all: locations.map(l => locationToEntity(l.type, l.target)), + all: locations.map(l => locationSpecToLocationEntity(l)), }); this.subscribers.add(subscriber); }); diff --git a/plugins/catalog-backend/src/next/LocationServiceImpl.ts b/plugins/catalog-backend/src/next/LocationServiceImpl.ts index e8e8745f01..02a98fb2b9 100644 --- a/plugins/catalog-backend/src/next/LocationServiceImpl.ts +++ b/plugins/catalog-backend/src/next/LocationServiceImpl.ts @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { LocationSpec, Location, Entity } from '@backstage/catalog-model'; +import { + LocationSpec, + Location, + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; import { LocationService, LocationStore, @@ -37,6 +43,10 @@ export class LocationServiceImpl implements LocationService { metadata: { name: `${spec.type}:${spec.target}`, namespace: 'default', + annotations: { + [LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`, + [ORIGIN_LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`, + }, }, spec: { location: { type: spec.type, target: spec.target }, diff --git a/plugins/catalog-backend/src/next/LocationToEntity.ts b/plugins/catalog-backend/src/next/LocationToEntity.ts deleted file mode 100644 index 28695e14af..0000000000 --- a/plugins/catalog-backend/src/next/LocationToEntity.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, EntityName } from '@backstage/catalog-model'; - -export function locationToEntity(type: string, target: string): Entity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Location', - metadata: { - name: `${type}:${target}`, - namespace: 'default', - }, - spec: { - location: { type, target }, - }, - }; -} - -export function locationToEntityName(type: string, target: string): EntityName { - return { - kind: 'Location', - namespace: 'default', - name: `${type}:${target}`, - }; -} diff --git a/plugins/catalog-backend/src/next/util.ts b/plugins/catalog-backend/src/next/util.ts new file mode 100644 index 0000000000..6ff1cde2f0 --- /dev/null +++ b/plugins/catalog-backend/src/next/util.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + LocationSpec, + LocationEntityV1alpha1, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, + stringifyEntityRef, + stringifyLocationReference, +} from '@backstage/catalog-model'; +import { createHash } from 'crypto'; + +export function locationSpecToMetadataName(location: LocationSpec) { + const hash = createHash('sha1') + .update(`${location.type}:${location.target}`) + .digest('hex'); + + return `generated-${hash}`; +} + +export function locationSpecToLocationEntity( + location: LocationSpec, + parentEntity?: Entity, +): LocationEntityV1alpha1 { + let ownLocation: string; + let originLocation: string; + if (parentEntity) { + const maybeOwnLocation = + parentEntity.metadata.annotations?.[LOCATION_ANNOTATION]; + if (!maybeOwnLocation) { + throw new Error( + `Parent entity '${stringifyEntityRef( + parentEntity, + )}' of location '${stringifyLocationReference( + location, + )}' does not have a location annotation`, + ); + } + ownLocation = maybeOwnLocation; + const maybeOriginLocation = + parentEntity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION]; + if (!maybeOriginLocation) { + throw new Error( + `Parent entity '${stringifyEntityRef( + parentEntity, + )}' of location '${stringifyLocationReference( + location, + )}' does not have an origin location annotation`, + ); + } + originLocation = maybeOriginLocation; + } else { + ownLocation = stringifyLocationReference(location); + originLocation = ownLocation; + } + + const result: LocationEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: locationSpecToMetadataName(location), + annotations: { + [LOCATION_ANNOTATION]: ownLocation, + [ORIGIN_LOCATION_ANNOTATION]: originLocation, + }, + }, + spec: { + type: location.type, + target: location.target, + optional: location.presence === 'optional' ? true : undefined, + }, + }; + + return result; +}