From 3368f27aef46b2e8198f3b8a2805f890c2f1aecc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 2 Dec 2021 19:53:59 +0100 Subject: [PATCH] catalog-backend: fix handling of optional locations Signed-off-by: Patrik Oldsberg --- .changeset/five-cats-hunt.md | 5 +++++ .../AzureDevOpsDiscoveryProcessor.test.ts | 4 ++++ .../processors/AzureDevOpsDiscoveryProcessor.ts | 1 + .../BitbucketDiscoveryProcessor.test.ts | 15 +++++++++++++++ .../processors/GitLabDiscoveryProcessor.test.ts | 3 +++ .../processors/GitLabDiscoveryProcessor.ts | 1 + .../processors/GithubDiscoveryProcessor.test.ts | 7 +++++++ .../processors/GithubDiscoveryProcessor.ts | 1 + .../bitbucket/BitbucketRepositoryParser.test.ts | 1 + .../bitbucket/BitbucketRepositoryParser.ts | 1 + .../DefaultCatalogProcessingOrchestrator.ts | 6 +++--- .../src/processing/ProcessorOutputCollector.ts | 3 ++- plugins/catalog-backend/src/util/conversion.ts | 1 + 13 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 .changeset/five-cats-hunt.md diff --git a/.changeset/five-cats-hunt.md b/.changeset/five-cats-hunt.md new file mode 100644 index 0000000000..cf3364a23a --- /dev/null +++ b/.changeset/five-cats-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed the handling of optional locations so that the catalog no longer logs `NotFoundError`s for missing optional locations. diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts index ab053e8748..463d982ca2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -165,6 +165,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -174,6 +175,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -211,6 +213,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -249,6 +252,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { type: 'url', target: 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index 58ee2155e9..aeac64f54d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -95,6 +95,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { { type: 'url', target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts index 2a63605b0d..3f8d0ba0fb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts @@ -208,6 +208,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -217,6 +218,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -242,6 +244,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -263,6 +266,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/test/browse/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -291,6 +295,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -334,6 +339,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -343,6 +349,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -370,6 +377,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -379,6 +387,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -406,6 +415,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -415,6 +425,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -441,6 +452,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -468,6 +480,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -493,6 +506,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -528,6 +542,7 @@ describe('BitbucketDiscoveryProcessor', () => { type: 'url', target: 'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index a0e44d7648..311de05673 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -201,6 +201,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + presence: 'optional', }, optional: true, }, @@ -209,6 +210,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }, @@ -246,6 +248,7 @@ describe('GitlabDiscoveryProcessor', () => { location: { type: 'url', target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }, diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index 445f02829e..70ccd44793 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -113,6 +113,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw // URL here won't work either. target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, + presence: 'optional', }, true, ), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 12d9e66659..59c25bb051 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -157,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/backstage/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -165,6 +166,7 @@ describe('GithubDiscoveryProcessor', () => { location: { type: 'url', target: 'https://github.com/backstage/demo/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -197,6 +199,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/tech-docs/blob/main/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -251,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + presence: 'optional', }, optional: true, }); @@ -306,6 +310,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -315,6 +320,7 @@ describe('GithubDiscoveryProcessor', () => { type: 'url', target: 'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); @@ -370,6 +376,7 @@ describe('GithubDiscoveryProcessor', () => { location: { type: 'url', target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + presence: 'optional', }, optional: true, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index b104d87542..9bd6cc775e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -125,6 +125,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { { type: 'url', target: `${repository.url}${path}`, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts index bc6c5a540f..397d259bae 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts @@ -29,6 +29,7 @@ describe('BitbucketRepositoryParser', () => { { type: 'url', target: `${browseUrl}${path}`, + presence: 'optional', }, true, ), diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts index 27f07d58a2..34a9284b3e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts @@ -30,6 +30,7 @@ export const defaultRepositoryParser: BitbucketRepositoryParser = { type: 'url', target: target, + presence: 'optional', }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index b42c8839cf..756b6a3639 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -284,7 +284,7 @@ export class DefaultCatalogProcessingOrchestrator entity: LocationEntity, context: Context, ): Promise { - const { type = context.location.type } = entity.spec; + const { type = context.location.type, presence = 'required' } = entity.spec; const targets = new Array(); if (entity.spec.target) { targets.push(entity.spec.target); @@ -318,9 +318,9 @@ export class DefaultCatalogProcessingOrchestrator { type, target, - presence: 'required', + presence, }, - false, + presence === 'optional', context.collector.onEmit, this.options.parser, context.cache.forProcessor(processor, target), diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 436038db66..b0c46b4b9f 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -105,8 +105,9 @@ export class ProcessorOutputCollector { this.deferredEntities.push({ entity, locationKey: location }); } else if (i.type === 'location') { + const presence = i.optional ? 'optional' : 'required'; const entity = locationSpecToLocationEntity( - i.location, + { presence, ...i.location }, this.parentEntity, ); const locationKey = getEntityLocationRef(entity); diff --git a/plugins/catalog-backend/src/util/conversion.ts b/plugins/catalog-backend/src/util/conversion.ts index a834f81255..559a472c25 100644 --- a/plugins/catalog-backend/src/util/conversion.ts +++ b/plugins/catalog-backend/src/util/conversion.ts @@ -81,6 +81,7 @@ export function locationSpecToLocationEntity( spec: { type: location.type, target: location.target, + presence: location.presence, }, };