paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Backwards compatible processing

Browse Source 
Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Co-authored-by: Fredrik Adelöw <freben@users.noreply.github.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>
This commit is contained in:
Johan Haals
2021-04-13 11:25:48 +02:00
parent bfa51b286f
commit 07de96af3b
5 changed files with 254 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/next/CatalogProcessingOrchestratorImpl.ts
+140 -12
View File
@@ -19,7 +19,12 @@ import {
EntityRelationSpec,
stringifyEntityRef,
LOCATION_ANNOTATION,
LocationSpec,
LocationEntity,
EntityPolicy,
ORIGIN_LOCATION_ANNOTATION,
stringifyLocationReference,
parseLocationReference,
} from '@backstage/catalog-model';
import {
CatalogProcessor,
@@ -33,12 +38,56 @@ import {
} from './types';
import { Logger } from 'winston';
import { InputError } from '@backstage/errors';
import { locationSpecToLocationEntity } from './util';
import path from 'path';
import * as results from '../ingestion/processors/results';
import { ScmIntegrationRegistry } from '@backstage/integration';
function isLocationEntity(entity: Entity): entity is LocationEntity {
return entity.kind === 'Location';
}
function getEntityOriginLocationRef(entity: Entity): string {
const ref = entity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION];
if (!ref) {
const entityRef = stringifyEntityRef(entity);
throw new InputError(
`Entity '${entityRef}' does not have an origin location`,
);
}
return ref;
}
function toAbsoluteUrl(
integrations: ScmIntegrationRegistry,
base: LocationSpec,
type: string,
target: string,
): string {
if (base.type !== type) {
return target;
}
try {
if (type === 'file') {
if (target.startsWith('.')) {
return path.join(path.dirname(base.target), target);
}
return target;
} else if (type === 'url') {
return integrations.resolveUrl({ url: target, base: base.target });
}
return target;
} catch (e) {
return target;
}
}
export class CatalogProcessingOrchestratorImpl
implements CatalogProcessingOrchestrator {
constructor(
private readonly options: {
processors: CatalogProcessor[];
integrations: ScmIntegrationRegistry;
logger: Logger;
parser: CatalogProcessorParser;
policy: EntityPolicy;
@@ -52,7 +101,7 @@ export class CatalogProcessingOrchestratorImpl
const result = await this.processSingleEntity(entity);
console.log('result', JSON.stringify(result));
console.log('result', JSON.stringify(result, undefined, 2));
return result;
}
@@ -64,19 +113,27 @@ export class CatalogProcessingOrchestratorImpl
// TODO: which one do we actually use here? source-location? - maybe probably doesn't exist yet?
const locationRef =
unprocessedEntity.metadata?.annotations?.[LOCATION_ANNOTATION];
if (!locationRef) {
throw new InputError(`Entity '${entityRef}' does not have a location`);
}
const location = parseLocationReference(locationRef);
const originLocation = parseLocationReference(
getEntityOriginLocationRef(unprocessedEntity),
);
const emitter = createEmitter(this.options.logger);
const emitter = createEmitter(this.options.logger, unprocessedEntity);
try {
// Pre-process phase, used to populate entities with data that is required during main processing step
let entity = unprocessedEntity;
for (const processor of this.options.processors) {
if (processor.preProcessEntity) {
try {
entity = await processor.preProcessEntity({
entity = await processor.preProcessEntity(
entity,
emit: emitter.emit,
});
location,
emitter.emit,
originLocation,
);
} catch (e) {
throw new Error(
`Processor ${processor.constructor.name} threw an error while preprocessing entity ${entityRef} at ${locationRef}, ${e}`,
@@ -123,14 +180,69 @@ export class CatalogProcessingOrchestratorImpl
);
}
// Backwards compatible processing of location entites
if (isLocationEntity(entity)) {
const { type = location.type, optional = false } = entity.spec;
const targets = new Array<string>();
if (entity.spec.target) {
targets.push(entity.spec.target);
}
if (entity.spec.targets) {
targets.push(...entity.spec.targets);
}
for (const maybeRelativeTarget of targets) {
if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) {
emitter.emit(
results.inputError(
location,
`LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`,
),
);
continue;
}
const target = toAbsoluteUrl(
this.options.integrations,
location,
type,
maybeRelativeTarget,
);
for (const processor of this.options.processors) {
if (processor.readLocation) {
try {
handled = await processor.readLocation(
{
type,
target,
presence: optional ? 'optional' : 'required',
},
Boolean(entity.spec?.optional),
emitter.emit,
this.options.parser,
);
if (handled) {
break;
}
} catch (e) {
throw new Error(
`Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`,
);
}
}
}
}
}
// Main processing step of the entity
for (const processor of this.options.processors) {
if (processor.processEntity) {
if (processor.postProcessEntity) {
try {
entity = await processor.processEntity({
entity = await processor.postProcessEntity(
entity,
emit: emitter.emit,
});
location,
emitter.emit,
);
} catch (e) {
throw new Error(
`Processor ${processor.constructor.name} threw an error while postprocessing entity ${entityRef} at ${locationRef}, ${e}`,
@@ -152,7 +264,7 @@ export class CatalogProcessingOrchestratorImpl
}
}
function createEmitter(logger: Logger) {
function createEmitter(logger: Logger, parentEntity: Entity) {
let done = false;
const errors = new Array<Error>();
@@ -170,7 +282,23 @@ function createEmitter(logger: Logger) {
return;
}
if (i.type === 'entity') {
deferredEntites.push(i.entity);
const originLocation = getEntityOriginLocationRef(parentEntity);
deferredEntites.push({
...i.entity,
metadata: {
...i.entity.metadata,
annotations: {
...i.entity.metadata.annotations,
[ORIGIN_LOCATION_ANNOTATION]: originLocation,
[LOCATION_ANNOTATION]: stringifyLocationReference(i.location),
},
},
});
} else if (i.type === 'location') {
deferredEntites.push(
locationSpecToLocationEntity(i.location, parentEntity),
);
} else if (i.type === 'relation') {
relations.push(i.relation);
} else if (i.type === 'error') {
plugins/catalog-backend/src/next/DatabaseLocationProvider.ts
+13 -7
View File
@@ -15,9 +15,13 @@
*/
import { Observable } from '@backstage/core';
import { ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { EntityProvider, LocationStore, EntityMessage } from './types';
import ObservableImpl from 'zen-observable';
import { locationToEntity, locationToEntityName } from './LocationToEntity';
import {
locationSpecToLocationEntity,
locationSpecToMetadataName,
} from './util';
export class DatabaseLocationProvider implements EntityProvider {
private subscribers = new Set<
@@ -29,14 +33,16 @@ export class DatabaseLocationProvider implements EntityProvider {
next: locations => {
if ('all' in locations) {
this.notify({
all: locations.all.map(l => locationToEntity(l.type, l.target)),
all: locations.all.map(l => locationSpecToLocationEntity(l)),
});
} else {
this.notify({
added: locations.added.map(l => locationToEntity(l.type, l.target)),
removed: locations.removed.map(l =>
locationToEntityName(l.type, l.target),
),
added: locations.added.map(l => locationSpecToLocationEntity(l)),
removed: locations.removed.map(l => ({
kind: 'Location',
namespace: ENTITY_DEFAULT_NAMESPACE,
name: locationSpecToMetadataName(l),
})),
});
}
},
@@ -53,7 +59,7 @@ export class DatabaseLocationProvider implements EntityProvider {
return new ObservableImpl(subscriber => {
this.store.listLocations().then(locations => {
subscriber.next({
all: locations.map(l => locationToEntity(l.type, l.target)),
all: locations.map(l => locationSpecToLocationEntity(l)),
});
this.subscribers.add(subscriber);
});
plugins/catalog-backend/src/next/LocationServiceImpl.ts
+11 -1
View File
@@ -13,7 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec, Location, Entity } from '@backstage/catalog-model';
import {
LocationSpec,
Location,
Entity,
LOCATION_ANNOTATION,
ORIGIN_LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import {
LocationService,
LocationStore,
@@ -37,6 +43,10 @@ export class LocationServiceImpl implements LocationService {
metadata: {
name: `${spec.type}:${spec.target}`,
namespace: 'default',
annotations: {
[LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`,
[ORIGIN_LOCATION_ANNOTATION]: `${spec.type}:${spec.target}`,
},
},
spec: {
location: { type: spec.type, target: spec.target },
plugins/catalog-backend/src/next/LocationToEntity.ts
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, EntityName } from '@backstage/catalog-model';
export function locationToEntity(type: string, target: string): Entity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
name: `${type}:${target}`,
namespace: 'default',
},
spec: {
location: { type, target },
},
};
}
export function locationToEntityName(type: string, target: string): EntityName {
return {
kind: 'Location',
namespace: 'default',
name: `${type}:${target}`,
};
}
plugins/catalog-backend/src/next/util.ts
+90
View File
@@ -0,0 +1,90 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
LocationSpec,
LocationEntityV1alpha1,
LOCATION_ANNOTATION,
ORIGIN_LOCATION_ANNOTATION,
stringifyEntityRef,
stringifyLocationReference,
} from '@backstage/catalog-model';
import { createHash } from 'crypto';
export function locationSpecToMetadataName(location: LocationSpec) {
const hash = createHash('sha1')
.update(`${location.type}:${location.target}`)
.digest('hex');
return `generated-${hash}`;
}
export function locationSpecToLocationEntity(
location: LocationSpec,
parentEntity?: Entity,
): LocationEntityV1alpha1 {
let ownLocation: string;
let originLocation: string;
if (parentEntity) {
const maybeOwnLocation =
parentEntity.metadata.annotations?.[LOCATION_ANNOTATION];
if (!maybeOwnLocation) {
throw new Error(
`Parent entity '${stringifyEntityRef(
parentEntity,
)}' of location '${stringifyLocationReference(
location,
)}' does not have a location annotation`,
);
}
ownLocation = maybeOwnLocation;
const maybeOriginLocation =
parentEntity.metadata.annotations?.[ORIGIN_LOCATION_ANNOTATION];
if (!maybeOriginLocation) {
throw new Error(
`Parent entity '${stringifyEntityRef(
parentEntity,
)}' of location '${stringifyLocationReference(
location,
)}' does not have an origin location annotation`,
);
}
originLocation = maybeOriginLocation;
} else {
ownLocation = stringifyLocationReference(location);
originLocation = ownLocation;
}
const result: LocationEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
name: locationSpecToMetadataName(location),
annotations: {
[LOCATION_ANNOTATION]: ownLocation,
[ORIGIN_LOCATION_ANNOTATION]: originLocation,
},
},
spec: {
type: location.type,
target: location.target,
optional: location.presence === 'optional' ? true : undefined,
},
};
return result;
}