diff --git a/.changeset/olive-parrots-retire.md b/.changeset/olive-parrots-retire.md new file mode 100644 index 0000000000..8e4ae1a7d4 --- /dev/null +++ b/.changeset/olive-parrots-retire.md @@ -0,0 +1,6 @@ +--- +'@backstage/catalog-model': minor +'@backstage/plugin-catalog-backend': patch +--- + +Add support for relative targets and implicit types in Location entities. diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 438f874990..19c5c74f68 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -31,6 +31,7 @@ we recommend that you name them `catalog-info.yaml`. - [Kind: Resource](#kind-resource) - [Kind: System](#kind-system) - [Kind: Domain](#kind-domain) +- [Kind: Location](#kind-location) ## Overall Shape Of An Entity @@ -884,3 +885,58 @@ This kind is not yet defined, but is reserved [for future use](system-model.md). ## Kind: Domain This kind is not yet defined, but is reserved [for future use](system-model.md). + +## Kind: Location + +Describes the following entity kind: + +| Field | Value | +| ------------ | ----------------------- | +| `apiVersion` | `backstage.io/v1alpha1` | +| `kind` | `Location` | + +A location is a marker that references other places to look for catalog data. + +Descriptor files for this kind may look as follows. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: Location +metadata: + name: org-data +spec: + type: url + targets: + - http://github.com/myorg/myproject/org-data-dump/catalog-info-staff.yaml + - http://github.com/myorg/myproject/org-data-dump/catalog-info-consultants.yaml +``` + +In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) +shape, this kind has the following structure. + +### `apiVersion` and `kind` [required] + +Exactly equal to `backstage.io/v1alpha1` and `Location`, respectively. + +### `spec.type` [optional] + +The single location type, that's common to the targets specified in the spec. If +it is left out, it is inherited from the location type that originally read the +entity data. For example, if you have a `url` type location, that when read +results in a `Location` kind entity with no `spec.type`, then the referenced +targets in the entity will implicitly also be of `url` type. This is useful +because you can define a hierarchy of things in a directory structure using +relative target paths (see below), and it will work out no matter if it's +consumed locally on disk from a `file` location, or as uploaded on a VCS. + +### `spec.target` [optional] + +A single target as a string. Can be either an absolute path/URL (depending on +the type), or a relative path such as `./details/catalog-info.yaml` which is +resolved relative to the location of this Location entity itself. + +### `spec.targets` [optional] + +A list of targets as strings. They can all be either absolute paths/URLs +(depending on the type), or relative paths such as `./details/catalog-info.yaml` +which are resolved relative to the location of this Location entity itself. diff --git a/packages/catalog-model/examples/acme-corp.yaml b/packages/catalog-model/examples/acme-corp.yaml index e8047fc143..6449d548e0 100644 --- a/packages/catalog-model/examples/acme-corp.yaml +++ b/packages/catalog-model/examples/acme-corp.yaml @@ -4,6 +4,5 @@ metadata: name: acme-corp description: A collection of all Backstage example Groups spec: - type: github targets: - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/org.yaml + - ./acme/org.yaml diff --git a/packages/catalog-model/examples/acme/org.yaml b/packages/catalog-model/examples/acme/org.yaml index 8c562aaf89..fe368962b1 100644 --- a/packages/catalog-model/examples/acme/org.yaml +++ b/packages/catalog-model/examples/acme/org.yaml @@ -16,12 +16,11 @@ metadata: name: example-groups description: A collection of all Backstage example Groups spec: - type: github targets: - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/infrastructure-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/boxoffice-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/backstage-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/team-a-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/team-b-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/team-c-group.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme/team-d-group.yaml + - ./infrastructure-group.yaml + - ./boxoffice-group.yaml + - ./backstage-group.yaml + - ./team-a-group.yaml + - ./team-b-group.yaml + - ./team-c-group.yaml + - ./team-d-group.yaml diff --git a/packages/catalog-model/examples/all-apis.yaml b/packages/catalog-model/examples/all-apis.yaml index 0278fc3078..20752faf97 100644 --- a/packages/catalog-model/examples/all-apis.yaml +++ b/packages/catalog-model/examples/all-apis.yaml @@ -4,10 +4,9 @@ metadata: name: example-apis description: A collection of all Backstage example APIs spec: - type: github targets: - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/apis/hello-world-api.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/apis/petstore-api.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/apis/spotify-api.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/apis/streetlights-api.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/apis/swapi-graphql.yaml + - ./apis/hello-world-api.yaml + - ./apis/petstore-api.yaml + - ./apis/spotify-api.yaml + - ./apis/streetlights-api.yaml + - ./apis/swapi-graphql.yaml diff --git a/packages/catalog-model/examples/all-components.yaml b/packages/catalog-model/examples/all-components.yaml index 06c44b59d7..f29a7378a0 100644 --- a/packages/catalog-model/examples/all-components.yaml +++ b/packages/catalog-model/examples/all-components.yaml @@ -4,14 +4,13 @@ metadata: name: example-components description: A collection of all Backstage example components spec: - type: github targets: - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/petstore-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/playback-order-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/podcast-api-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/queue-proxy-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/searcher-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/playback-lib-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/www-artist-component.yaml - - https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/shuffle-api-component.yaml + - ./components/artist-lookup-component.yaml + - ./components/petstore-component.yaml + - ./components/playback-order-component.yaml + - ./components/podcast-api-component.yaml + - ./components/queue-proxy-component.yaml + - ./components/searcher-component.yaml + - ./components/playback-lib-component.yaml + - ./components/www-artist-component.yaml + - ./components/shuffle-api-component.yaml diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts index d9c1e9185f..2451df8e64 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts @@ -54,9 +54,9 @@ describe('LocationV1alpha1Validator', () => { await expect(validator.check(entity)).resolves.toBe(false); }); - it('rejects missing type', async () => { + it('accepts missing type', async () => { delete (entity as any).spec.type; - await expect(validator.check(entity)).rejects.toThrow(/type/); + await expect(validator.check(entity)).resolves.toBe(true); }); it('rejects wrong type', async () => { diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index e536be7922..9cd767de94 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -26,7 +26,7 @@ const schema = yup.object>({ kind: yup.string().required().equals([KIND]), spec: yup .object({ - type: yup.string().required().min(1), + type: yup.string().notRequired().min(1), target: yup.string().notRequired().min(1), targets: yup.array(yup.string().required()).notRequired(), }) @@ -37,7 +37,7 @@ export interface LocationEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; spec: { - type: string; + type?: string; target?: string; targets?: string[]; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts new file mode 100644 index 0000000000..998dfb62a2 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { toAbsoluteUrl } from './LocationEntityProcessor'; +import path from 'path'; + +describe('LocationEntityProcessor', () => { + describe('toAbsoluteUrl', () => { + it('handles files', () => { + const base: LocationSpec = { + type: 'file', + target: `some${path.sep}path${path.sep}catalog-info.yaml`, + }; + expect(toAbsoluteUrl(base, `.${path.sep}c`)).toBe( + `some${path.sep}path${path.sep}c`, + ); + expect(toAbsoluteUrl(base, `${path.sep}c`)).toBe(`${path.sep}c`); + }); + + it('handles urls', () => { + const base: LocationSpec = { + type: 'url', + target: 'http://a.com/b/catalog-info.yaml', + }; + expect(toAbsoluteUrl(base, './c/d')).toBe('http://a.com/b/c/d'); + expect(toAbsoluteUrl(base, 'c/d')).toBe('http://a.com/b/c/d'); + expect(toAbsoluteUrl(base, 'http://b.com/z')).toBe('http://b.com/z'); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts index 86e17f07f2..1c7e13f1c4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts @@ -17,27 +17,52 @@ import { Entity, LocationEntity, LocationSpec } from '@backstage/catalog-model'; import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import path from 'path'; + +export function toAbsoluteUrl(base: LocationSpec, target: string): string { + try { + if (base.type === 'file') { + if (target.startsWith('.')) { + return path.join(path.dirname(base.target), target); + } + return target; + } + return new URL(target, base.target).toString(); + } catch (e) { + return target; + } +} export class LocationRefProcessor implements CatalogProcessor { async postProcessEntity( entity: Entity, - _location: LocationSpec, + location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { if (entity.kind === 'Location') { - const location = entity as LocationEntity; - if (location.spec.target) { + const locationEntity = entity as LocationEntity; + + const type = locationEntity.spec.type || location.type; + if (type === 'file' && location.target.endsWith(path.sep)) { emit( - result.location( - { type: location.spec.type, target: location.spec.target }, - false, + result.inputError( + location, + `LocationRefProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, ), ); } - if (location.spec.targets) { - for (const target of location.spec.targets) { - emit(result.location({ type: location.spec.type, target }, false)); - } + + const targets = new Array(); + if (locationEntity.spec.target) { + targets.push(locationEntity.spec.target); + } + if (locationEntity.spec.targets) { + targets.push(...locationEntity.spec.targets); + } + + for (const maybeRelativeTarget of targets) { + const target = toAbsoluteUrl(location, maybeRelativeTarget); + emit(result.location({ type, target }, false)); } }