diff --git a/.changeset/five-cats-hunt.md b/.changeset/five-cats-hunt.md new file mode 100644 index 0000000000..cf3364a23a --- /dev/null +++ b/.changeset/five-cats-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed the handling of optional locations so that the catalog no longer logs `NotFoundError`s for missing optional locations. diff --git a/.changeset/sweet-camels-learn.md b/.changeset/sweet-camels-learn.md new file mode 100644 index 0000000000..60481b9363 --- /dev/null +++ b/.changeset/sweet-camels-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Added an optional `presence` field to Location spec, which describes whether the target of a location is required to exist or not. It defaults to `'required'`, which is the current behaviour of the catalog. diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 52f36493b5..9c0c33fa0e 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -325,6 +325,7 @@ interface LocationEntityV1alpha1 extends Entity { type?: string; target?: string; targets?: string[]; + presence?: 'required' | 'optional'; }; } export { LocationEntityV1alpha1 as LocationEntity }; diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts index 7efec38395..6e78c1b96e 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts @@ -102,4 +102,23 @@ describe('LocationV1alpha1Validator', () => { (entity as any).spec.targets = 7; await expect(validator.check(entity)).rejects.toThrow(/targets/); }); + + it('accepts good presence', async () => { + (entity as any).spec.presence = 'required'; + await expect(validator.check(entity)).resolves.toBe(true); + (entity as any).spec.presence = 'optional'; + await expect(validator.check(entity)).resolves.toBe(true); + }); + + it('rejects empty presence', async () => { + (entity as any).spec.presence = ''; + await expect(validator.check(entity)).rejects.toThrow(/presence/); + }); + + it('rejects wrong presence', async () => { + (entity as any).spec.presence = 7; + await expect(validator.check(entity)).rejects.toThrow(/presence/); + (entity as any).spec.presence = 'nope'; + await expect(validator.check(entity)).rejects.toThrow(/presence/); + }); }); diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index 1872e43526..93a218aa2d 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -30,6 +30,7 @@ export interface LocationEntityV1alpha1 extends Entity { type?: string; target?: string; targets?: string[]; + presence?: 'required' | 'optional'; }; } diff --git a/packages/catalog-model/src/schema/kinds/Location.v1alpha1.schema.json b/packages/catalog-model/src/schema/kinds/Location.v1alpha1.schema.json index d633d30229..b70df9e70e 100644 --- a/packages/catalog-model/src/schema/kinds/Location.v1alpha1.schema.json +++ b/packages/catalog-model/src/schema/kinds/Location.v1alpha1.schema.json @@ -59,6 +59,13 @@ ], "minLength": 1 } + }, + "presence": { + "type": "string", + "description": "Whether the presence of the location target is required and it should be considered an error if it can not be found", + "default": "required", + "examples": ["required"], + "enum": ["required", "optional"] } } } diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts index ab053e8748..463d982ca2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -165,6 +165,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -174,6 +175,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -211,6 +213,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -249,6 +252,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index 58ee2155e9..aeac64f54d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -95,6 +95,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { { type: 'url', target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 2a63605b0d..3f8d0ba0fb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -208,6 +208,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -217,6 +218,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -242,6 +244,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -263,6 +266,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -291,6 +295,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -334,6 +339,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -343,6 +349,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -370,6 +377,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -379,6 +387,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -406,6 +415,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -415,6 +425,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -441,6 +452,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -468,6 +480,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -493,6 +506,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -528,6 +542,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index a0e44d7648..311de05673 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -201,6 +201,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + presence: 'optional', }, optional: true, }, @@ -209,6 +210,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }, @@ -246,6 +248,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }, diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index 445f02829e..70ccd44793 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -113,6 +113,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw // URL here won't work either. target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, + presence: 'optional', }, true, ), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 12d9e66659..59c25bb051 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -157,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/backstage/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -165,6 +166,7 @@ describe('GithubDiscoveryProcessor', () => { location: { type: 'url', target: 'https://github.com/backstage/demo/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -197,6 +199,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/tech-docs/blob/main/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -251,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -306,6 +310,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -315,6 +320,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -370,6 +376,7 @@ describe('GithubDiscoveryProcessor', () => { location: { type: 'url', target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index b104d87542..9bd6cc775e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -125,6 +125,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { { type: 'url', target: `${repository.url}${path}`, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts index bc6c5a540f..397d259bae 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts @@ -29,6 +29,7 @@ describe('BitbucketRepositoryParser', () => { { type: 'url', target: `${browseUrl}${path}`, + presence: 'optional', }, true, ), diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts index 27f07d58a2..34a9284b3e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts @@ -30,6 +30,7 @@ export const defaultRepositoryParser: BitbucketRepositoryParser = { type: 'url', target: target, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index b42c8839cf..756b6a3639 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -284,7 +284,7 @@ export class DefaultCatalogProcessingOrchestrator entity: LocationEntity, context: Context, ): Promise { - const { type = context.location.type } = entity.spec; + const { type = context.location.type, presence = 'required' } = entity.spec; const targets = new Array(); if (entity.spec.target) { targets.push(entity.spec.target); @@ -318,9 +318,9 @@ export class DefaultCatalogProcessingOrchestrator { type, target, - presence: 'required', + presence, }, - false, + presence === 'optional', context.collector.onEmit, this.options.parser, context.cache.forProcessor(processor, target), diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 436038db66..b0c46b4b9f 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -105,8 +105,9 @@ export class ProcessorOutputCollector { this.deferredEntities.push({ entity, locationKey: location }); } else if (i.type === 'location') { + const presence = i.optional ? 'optional' : 'required'; const entity = locationSpecToLocationEntity( - i.location, + { presence, ...i.location }, this.parentEntity, ); const locationKey = getEntityLocationRef(entity); diff --git a/plugins/catalog-backend/src/util/conversion.ts b/plugins/catalog-backend/src/util/conversion.ts index a834f81255..559a472c25 100644 --- a/plugins/catalog-backend/src/util/conversion.ts +++ b/plugins/catalog-backend/src/util/conversion.ts @@ -81,6 +81,7 @@ export function locationSpecToLocationEntity( spec: { type: location.type, target: location.target, + presence: location.presence, }, };