From 83a83381b09890e0c2f6f7a6a5b4edcda7ae2982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 27 Feb 2022 19:16:01 +0100 Subject: [PATCH] rearrange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/lemon-needles-applaud.md | 5 ++ .changeset/long-weeks-thank.md | 5 ++ .changeset/old-waves-wash.md | 8 +++ .../software-catalog/external-integrations.md | 4 +- .../AwsOrganizationCloudAccountProcessor.ts | 4 +- .../src/processors/LdapOrgReaderProcessor.ts | 6 +- .../MicrosoftGraphOrgReaderProcessor.ts | 6 +- plugins/catalog-backend/api-report.md | 37 ++++++++-- plugins/catalog-backend/src/api/common.ts | 56 +++++++++++++++ .../results.ts => api/deprecatedResult.ts} | 36 +++++++--- plugins/catalog-backend/src/api/index.ts | 38 ++++++++++ .../src/api/processingResult.ts | 71 +++++++++++++++++++ .../processors/types.ts => api/processor.ts} | 19 +---- .../{providers/types.ts => api/provider.ts} | 2 +- plugins/catalog-backend/src/database/types.ts | 3 +- plugins/catalog-backend/src/index.ts | 3 +- .../src/ingestion/CatalogRules.test.ts | 2 +- .../src/ingestion/CatalogRules.ts | 2 +- .../catalog-backend/src/ingestion/index.ts | 2 - .../catalog-backend/src/ingestion/types.ts | 2 +- .../aws}/AwsS3DiscoveryProcessor.test.ts | 13 ++-- .../aws}/AwsS3DiscoveryProcessor.ts | 8 +-- .../aws/__fixtures__}/awsS3-mock-object.txt | 0 .../providers => modules/aws}/index.ts | 4 +- .../AzureDevOpsDiscoveryProcessor.test.ts | 6 +- .../azure}/AzureDevOpsDiscoveryProcessor.ts | 12 ++-- .../src/{providers => modules/azure}/index.ts | 8 +-- .../azure => modules/azure/lib}/azure.test.ts | 0 .../azure => modules/azure/lib}/azure.ts | 0 .../azure => modules/azure/lib}/index.ts | 0 .../BitbucketDiscoveryProcessor.test.ts | 12 ++-- .../bitbucket}/BitbucketDiscoveryProcessor.ts | 5 +- .../src/modules/bitbucket/index.ts | 18 +++++ .../lib}/BitbucketRepositoryParser.test.ts | 5 +- .../lib}/BitbucketRepositoryParser.ts | 31 ++++---- .../bitbucket/lib}/client.ts | 2 +- .../bitbucket/lib}/index.ts | 1 + .../bitbucket/lib}/types.ts | 0 .../codeowners}/CodeOwnersProcessor.test.ts | 2 +- .../codeowners}/CodeOwnersProcessor.ts | 5 +- .../src/modules/codeowners/index.ts | 17 +++++ .../codeowners/lib}/index.ts | 0 .../codeowners/lib}/read.test.ts | 0 .../codeowners/lib}/read.ts | 0 .../codeowners/lib}/resolve.test.ts | 0 .../codeowners/lib}/resolve.ts | 0 .../codeowners/lib}/scm.ts | 0 .../AnnotateLocationEntityProcessor.test.ts | 2 +- .../core}/AnnotateLocationEntityProcessor.ts | 6 +- .../AnnotateScmSlugEntityProcessor.test.ts | 2 +- .../core}/AnnotateScmSlugEntityProcessor.ts | 2 +- .../core}/BuiltinKindsEntityProcessor.test.ts | 0 .../core}/BuiltinKindsEntityProcessor.ts | 12 ++-- .../ConfigLocationEntityProvider.test.ts | 2 +- .../core}/ConfigLocationEntityProvider.ts | 6 +- .../core}/DefaultLocationStore.test.ts | 2 +- .../core}/DefaultLocationStore.ts | 10 +-- .../core}/FileReaderProcessor.test.ts | 10 ++- .../core}/FileReaderProcessor.ts | 8 +-- .../core}/LocationEntityProcessor.test.ts | 2 +- .../core}/LocationEntityProcessor.ts | 12 ++-- .../core}/PlaceholderProcessor.test.ts | 0 .../core}/PlaceholderProcessor.ts | 2 +- .../core}/StaticLocationProcessor.ts | 9 ++- .../core}/UrlReaderProcessor.test.ts | 4 +- .../core}/UrlReaderProcessor.ts | 8 +-- .../awsS3/awsS3-mock-object.txt | 1 + .../fileReaderProcessor/component.yaml | 0 .../fileReaderProcessor/dir/api.yaml | 0 .../__fixtures__/fileReaderProcessor/test.txt | 0 .../processors => modules/core}/index.ts | 17 +---- .../github}/GitHubOrgEntityProvider.test.ts | 8 ++- .../github}/GitHubOrgEntityProvider.ts | 7 +- .../github}/GithubDiscoveryProcessor.test.ts | 10 +-- .../github}/GithubDiscoveryProcessor.ts | 12 ++-- .../github}/GithubMultiOrgReaderProcessor.ts | 16 +++-- .../github}/GithubOrgReaderProcessor.test.ts | 3 +- .../github}/GithubOrgReaderProcessor.ts | 16 +++-- .../src/modules/github/index.ts | 21 ++++++ .../github/lib}/config.test.ts | 0 .../github => modules/github/lib}/config.ts | 0 .../github/lib}/github.test.ts | 0 .../github => modules/github/lib}/github.ts | 0 .../github => modules/github/lib}/index.ts | 0 .../github/lib}/util.test.ts | 1 + .../github => modules/github/lib}/util.ts | 1 + .../gitlab}/GitLabDiscoveryProcessor.test.ts | 4 +- .../gitlab}/GitLabDiscoveryProcessor.ts | 22 +++--- .../src/modules/gitlab/index.ts | 17 +++++ .../gitlab/lib}/client.test.ts | 2 +- .../gitlab => modules/gitlab/lib}/client.ts | 0 .../gitlab => modules/gitlab/lib}/index.ts | 0 .../gitlab => modules/gitlab/lib}/types.ts | 0 plugins/catalog-backend/src/modules/index.ts | 23 ++++++ .../processors => modules}/util/org.test.ts | 0 .../processors => modules}/util/org.ts | 0 .../processors => modules}/util/parse.test.ts | 21 +++--- .../processors => modules}/util/parse.ts | 12 ++-- ...faultCatalogProcessingOrchestrator.test.ts | 12 ++-- .../DefaultCatalogProcessingOrchestrator.ts | 22 +++--- .../processing/ProcessorCacheManager.test.ts | 2 +- .../src/processing/ProcessorCacheManager.ts | 3 +- .../processing/ProcessorOutputCollector.ts | 4 +- .../src/processing/connectEntityProviders.ts | 2 +- .../catalog-backend/src/processing/index.ts | 1 - .../catalog-backend/src/processing/types.ts | 25 +------ .../catalog-backend/src/processing/util.ts | 2 +- .../src/service/AuthorizedRefreshService.ts | 1 + .../src/service/CatalogBuilder.ts | 18 ++--- .../src/service/DefaultLocationService.ts | 1 + .../catalog-backend/src/util/conversion.ts | 2 +- .../processor/ScaffolderEntitiesProcessor.ts | 6 +- 112 files changed, 594 insertions(+), 280 deletions(-) create mode 100644 .changeset/lemon-needles-applaud.md create mode 100644 .changeset/long-weeks-thank.md create mode 100644 .changeset/old-waves-wash.md create mode 100644 plugins/catalog-backend/src/api/common.ts rename plugins/catalog-backend/src/{ingestion/processors/results.ts => api/deprecatedResult.ts} (69%) create mode 100644 plugins/catalog-backend/src/api/index.ts create mode 100644 plugins/catalog-backend/src/api/processingResult.ts rename plugins/catalog-backend/src/{ingestion/processors/types.ts => api/processor.ts} (92%) rename plugins/catalog-backend/src/{providers/types.ts => api/provider.ts} (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/aws}/AwsS3DiscoveryProcessor.test.ts (89%) rename plugins/catalog-backend/src/{ingestion/processors => modules/aws}/AwsS3DiscoveryProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors/__fixtures__/fileReaderProcessor/awsS3 => modules/aws/__fixtures__}/awsS3-mock-object.txt (100%) rename plugins/catalog-backend/src/{ingestion/providers => modules/aws}/index.ts (83%) rename plugins/catalog-backend/src/{ingestion/processors => modules/azure}/AzureDevOpsDiscoveryProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/azure}/AzureDevOpsDiscoveryProcessor.ts (95%) rename plugins/catalog-backend/src/{providers => modules/azure}/index.ts (79%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/azure.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/azure.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/azure => modules/azure/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/bitbucket}/BitbucketDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/bitbucket}/BitbucketDiscoveryProcessor.ts (99%) create mode 100644 plugins/catalog-backend/src/modules/bitbucket/index.ts rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/BitbucketRepositoryParser.test.ts (94%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/BitbucketRepositoryParser.ts (62%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/client.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/index.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors/bitbucket => modules/bitbucket/lib}/types.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/codeowners}/CodeOwnersProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/codeowners}/CodeOwnersProcessor.ts (95%) create mode 100644 plugins/catalog-backend/src/modules/codeowners/index.ts rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/read.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/read.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/resolve.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/resolve.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/codeowners => modules/codeowners/lib}/scm.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateLocationEntityProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateLocationEntityProcessor.ts (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateScmSlugEntityProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/AnnotateScmSlugEntityProcessor.ts (97%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/BuiltinKindsEntityProcessor.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/BuiltinKindsEntityProcessor.ts (97%) rename plugins/catalog-backend/src/{providers => modules/core}/ConfigLocationEntityProvider.test.ts (98%) rename plugins/catalog-backend/src/{providers => modules/core}/ConfigLocationEntityProvider.ts (90%) rename plugins/catalog-backend/src/{providers => modules/core}/DefaultLocationStore.test.ts (98%) rename plugins/catalog-backend/src/{providers => modules/core}/DefaultLocationStore.ts (93%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/FileReaderProcessor.test.ts (94%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/FileReaderProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/LocationEntityProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/LocationEntityProcessor.ts (92%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/PlaceholderProcessor.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/PlaceholderProcessor.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/StaticLocationProcessor.ts (91%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/UrlReaderProcessor.test.ts (98%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/UrlReaderProcessor.ts (95%) create mode 100644 plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/component.yaml (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/dir/api.yaml (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/__fixtures__/fileReaderProcessor/test.txt (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules/core}/index.ts (63%) rename plugins/catalog-backend/src/{ingestion/providers => modules/github}/GitHubOrgEntityProvider.test.ts (97%) rename plugins/catalog-backend/src/{ingestion/providers => modules/github}/GitHubOrgEntityProvider.ts (97%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubDiscoveryProcessor.ts (96%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubMultiOrgReaderProcessor.ts (95%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubOrgReaderProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/github}/GithubOrgReaderProcessor.ts (92%) create mode 100644 plugins/catalog-backend/src/modules/github/index.ts rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/config.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/config.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/github.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/github.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/util.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors/github => modules/github/lib}/util.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/gitlab}/GitLabDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend/src/{ingestion/processors => modules/gitlab}/GitLabDiscoveryProcessor.ts (92%) create mode 100644 plugins/catalog-backend/src/modules/gitlab/index.ts rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/client.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/client.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/index.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors/gitlab => modules/gitlab/lib}/types.ts (100%) create mode 100644 plugins/catalog-backend/src/modules/index.ts rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/org.test.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/org.ts (100%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/parse.test.ts (91%) rename plugins/catalog-backend/src/{ingestion/processors => modules}/util/parse.ts (87%) diff --git a/.changeset/lemon-needles-applaud.md b/.changeset/lemon-needles-applaud.md new file mode 100644 index 0000000000..a77d3fd33c --- /dev/null +++ b/.changeset/lemon-needles-applaud.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATED**: The `results` export, and instead adding `processingResult` with the same shape and purpose. diff --git a/.changeset/long-weeks-thank.md b/.changeset/long-weeks-thank.md new file mode 100644 index 0000000000..a6c3a0719f --- /dev/null +++ b/.changeset/long-weeks-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Internal restructuring to collect the various provider files in a `modules` folder while waiting to be externalized diff --git a/.changeset/old-waves-wash.md b/.changeset/old-waves-wash.md new file mode 100644 index 0000000000..34ba0760b5 --- /dev/null +++ b/.changeset/old-waves-wash.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +'@backstage/plugin-catalog-backend-module-ldap': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Use the new `processingResult` export from the catalog backend diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 59caf9475c..9f592e7cca 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -57,7 +57,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/providers/types.ts) +[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/provider.ts) subclass that can be added to this catalog builder. Let's make a simple provider that can refresh a set of entities based on a @@ -355,7 +355,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/ingestion/processors/types.ts) +[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/processor.ts) subclass that can be added to this catalog builder. It is up to you where you put the code for this new processor class. For quick diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts index e5e72d5fa8..1018e49a41 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts @@ -20,7 +20,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import AWS, { Credentials, Organizations } from 'aws-sdk'; import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; @@ -112,7 +112,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { return true; }) .forEach(entity => { - emit(results.entity(location, entity)); + emit(processingResult.entity(location, entity)); }); return true; diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts index 6c31d5a27d..bec24a1744 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts @@ -28,7 +28,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; /** @@ -119,10 +119,10 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 4db3a9b1a0..942bde19eb 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -19,7 +19,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { @@ -125,10 +125,10 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index c9e08964b6..00e09d1169 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -524,7 +524,7 @@ export type EntitiesSearchFilter = { values?: string[]; }; -// @public (undocumented) +// @public @deprecated (undocumented) function entity( atLocation: LocationSpec, newEntity: Entity, @@ -641,7 +641,7 @@ export class FileReaderProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function generalError( atLocation: LocationSpec, message: string, @@ -776,13 +776,13 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function inputError( atLocation: LocationSpec, message: string, ): CatalogProcessorResult; -// @public (undocumented) +// @public @deprecated (undocumented) function location_2( newLocation: LocationSpec, optional?: boolean, @@ -876,7 +876,7 @@ export interface LocationStore { listLocations(): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function notFoundError( atLocation: LocationSpec, message: string, @@ -963,6 +963,31 @@ export type PlaceholderResolverResolveUrl = ( base: string, ) => string; +// @public +export const processingResult: Readonly<{ + readonly notFoundError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly inputError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly generalError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly location: ( + newLocation: LocationSpec, + optional?: boolean | undefined, + ) => CatalogProcessorResult; + readonly entity: ( + atLocation: LocationSpec, + newEntity: Entity, + ) => CatalogProcessorResult; + readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; +}>; + // @public export type RecursivePartial = { [P in keyof T]?: T[P] extends (infer U)[] @@ -986,7 +1011,7 @@ export interface RefreshService { refresh(options: RefreshOptions): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function relation(spec: EntityRelationSpec): CatalogProcessorResult; declare namespace results { diff --git a/plugins/catalog-backend/src/api/common.ts b/plugins/catalog-backend/src/api/common.ts new file mode 100644 index 0000000000..1fd09aec32 --- /dev/null +++ b/plugins/catalog-backend/src/api/common.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityName } from '@backstage/catalog-model'; + +/** + * Holds the entity location information. + * + * @remarks + * + * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. + * This flag is then set to indicate that the file can be not present. + * default value: 'required'. + * + * @public + */ +export type LocationSpec = { + type: string; + target: string; + presence?: 'optional' | 'required'; +}; + +/** + * Holds the relation data for entities. + * + * @public + */ +export type EntityRelationSpec = { + /** + * The source entity of this relation. + */ + source: EntityName; + + /** + * The type of the relation. + */ + type: string; + + /** + * The target entity of this relation. + */ + target: EntityName; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/results.ts b/plugins/catalog-backend/src/api/deprecatedResult.ts similarity index 69% rename from plugins/catalog-backend/src/ingestion/processors/results.ts rename to plugins/catalog-backend/src/api/deprecatedResult.ts index 01583678df..ecacc62c65 100644 --- a/plugins/catalog-backend/src/ingestion/processors/results.ts +++ b/plugins/catalog-backend/src/api/deprecatedResult.ts @@ -16,10 +16,15 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Entity } from '@backstage/catalog-model'; -import { CatalogProcessorResult, LocationSpec } from './types'; -import { EntityRelationSpec } from '../../processing/types'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; -/** @public */ +// NOTE: This entire file is deprecated and should be eventually removed along with the `result` export + +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function notFoundError( atLocation: LocationSpec, message: string, @@ -31,7 +36,10 @@ export function notFoundError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function inputError( atLocation: LocationSpec, message: string, @@ -43,7 +51,10 @@ export function inputError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function generalError( atLocation: LocationSpec, message: string, @@ -51,7 +62,10 @@ export function generalError( return { type: 'error', location: atLocation, error: new Error(message) }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function location( newLocation: LocationSpec, optional?: boolean, @@ -59,7 +73,10 @@ export function location( return { type: 'location', location: newLocation, optional }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function entity( atLocation: LocationSpec, newEntity: Entity, @@ -67,7 +84,10 @@ export function entity( return { type: 'entity', location: atLocation, entity: newEntity }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; } diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts new file mode 100644 index 0000000000..3f1c627743 --- /dev/null +++ b/plugins/catalog-backend/src/api/index.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as results from './deprecatedResult'; + +export { results }; + +export { processingResult } from './processingResult'; +export type { EntityRelationSpec, LocationSpec } from './common'; +export type { + CatalogProcessor, + CatalogProcessorParser, + CatalogProcessorCache, + CatalogProcessorEmit, + CatalogProcessorLocationResult, + CatalogProcessorEntityResult, + CatalogProcessorRelationResult, + CatalogProcessorErrorResult, + CatalogProcessorResult, +} from './processor'; +export type { + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, +} from './provider'; diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts new file mode 100644 index 0000000000..5ad908dc8b --- /dev/null +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError, NotFoundError } from '@backstage/errors'; +import { Entity } from '@backstage/catalog-model'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; + +/** + * Factory functions for the standard processing result types. + * + * @public + */ +export const processingResult = Object.freeze({ + notFoundError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new NotFoundError(message), + }; + }, + + inputError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new InputError(message), + }; + }, + + generalError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { type: 'error', location: atLocation, error: new Error(message) }; + }, + + location( + newLocation: LocationSpec, + optional?: boolean, + ): CatalogProcessorResult { + return { type: 'location', location: newLocation, optional }; + }, + + entity(atLocation: LocationSpec, newEntity: Entity): CatalogProcessorResult { + return { type: 'entity', location: atLocation, entity: newEntity }; + }, + + relation(spec: EntityRelationSpec): CatalogProcessorResult { + return { type: 'relation', relation: spec }; + }, +} as const); diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/api/processor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/types.ts rename to plugins/catalog-backend/src/api/processor.ts index 9b3bb05e2d..12a008edc7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -16,24 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; -import { EntityRelationSpec } from '../../processing/types'; - -/** - * Holds the entity location information. - * - * @remarks - * - * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. - * This flag is then set to indicate that the file can be not present. - * default value: 'required'. - * - * @public - */ -export type LocationSpec = { - type: string; - target: string; - presence?: 'optional' | 'required'; -}; +import { EntityRelationSpec, LocationSpec } from './common'; /** * @public diff --git a/plugins/catalog-backend/src/providers/types.ts b/plugins/catalog-backend/src/api/provider.ts similarity index 96% rename from plugins/catalog-backend/src/providers/types.ts rename to plugins/catalog-backend/src/api/provider.ts index 226c021e3c..fa7659ee32 100644 --- a/plugins/catalog-backend/src/providers/types.ts +++ b/plugins/catalog-backend/src/api/provider.ts @@ -44,7 +44,7 @@ export interface EntityProviderConnection { * @public */ export interface EntityProvider { - /** Unique name provider name used internally for caching. */ + /** Unique provider name used internally for caching. */ getProviderName(): string; /** Connect is called upon initialization by the catalog engine. */ connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 08201323ea..1affb14b45 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -17,7 +17,8 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; -import { DeferredEntity, EntityRelationSpec } from '../processing/types'; +import { EntityRelationSpec } from '../api'; +import { DeferredEntity } from '../processing/types'; /** * An abstraction for transactions of the underlying database technology. diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index efdaa4b8c5..4c1e753300 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -20,11 +20,12 @@ * @packageDocumentation */ +export * from './api'; export * from './catalog'; export * from './ingestion'; +export * from './modules'; export * from './search'; export * from './util'; export * from './processing'; -export * from './providers'; export * from './service'; export * from './permissions'; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index d29a445a8c..13430f51dd 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultCatalogRulesEnforcer } from './CatalogRules'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; const entity = { user: { diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index b4ad394168..e4bfdb8d0f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** * Rules to apply to catalog entities. diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index fcb634914b..1f4729b041 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -16,8 +16,6 @@ export { DefaultCatalogRulesEnforcer } from './CatalogRules'; export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; -export * from './processors'; -export * from './providers'; export type { AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index eccf583e4b..37d12d12b1 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { RecursivePartial } from '../util/RecursivePartial'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** @public */ export type LocationAnalyzer = { diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts index 064a2ad724..249de71591 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; -import { defaultEntityDataParser } from './util/parse'; +import { + CatalogProcessorEntityResult, + CatalogProcessorResult, +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; @@ -36,10 +40,7 @@ AWSMock.mock( 'getObject', Buffer.from( require('fs').readFileSync( - path.resolve( - __dirname, - '__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt', - ), + path.resolve(__dirname, '__fixtures__/awsS3-mock-object.txt'), ), ), ); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts index 0dc44cc157..737f8f7c2f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts @@ -17,13 +17,13 @@ import { UrlReader } from '@backstage/backend-common'; import { isError } from '@backstage/errors'; import limiterFactory from 'p-limit'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; /** @public */ export class AwsS3DiscoveryProcessor implements CatalogProcessor { @@ -58,10 +58,10 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { if (isError(error) && error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt rename to plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog-backend/src/modules/aws/index.ts similarity index 83% rename from plugins/catalog-backend/src/ingestion/providers/index.ts rename to plugins/catalog-backend/src/modules/aws/index.ts index cd1bc5cb36..9477d4104d 100644 --- a/plugins/catalog-backend/src/ingestion/providers/index.ts +++ b/plugins/catalog-backend/src/modules/aws/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts index a406feaa74..450a0482ed 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts @@ -16,14 +16,14 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { codeSearch } from './azure'; +import { codeSearch } from './lib'; import { AzureDevOpsDiscoveryProcessor, parseUrl, } from './AzureDevOpsDiscoveryProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; -jest.mock('./azure'); +jest.mock('./lib'); const mockCodeSearch = codeSearch as jest.MockedFunction; describe('AzureDevOpsDiscoveryProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts index bc74621f1d..8e36e810b8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { codeSearch } from './azure'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { codeSearch } from './lib'; /** * Extracts repositories out of an Azure DevOps org. @@ -102,7 +106,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { for (const file of files) { emit( - results.location({ + processingResult.location({ type: 'url', target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/providers/index.ts b/plugins/catalog-backend/src/modules/azure/index.ts similarity index 79% rename from plugins/catalog-backend/src/providers/index.ts rename to plugins/catalog-backend/src/modules/azure/index.ts index 3b4a1c4b7e..9c76336804 100644 --- a/plugins/catalog-backend/src/providers/index.ts +++ b/plugins/catalog-backend/src/modules/azure/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,4 @@ * limitations under the License. */ -export type { - EntityProvider, - EntityProviderConnection, - EntityProviderMutation, -} from './types'; +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/modules/azure/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/index.ts rename to plugins/catalog-backend/src/modules/azure/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts index 4a261386c0..857f999a70 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts @@ -13,18 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; -import { - BitbucketRepository20, - PagedResponse, - PagedResponse20, -} from './bitbucket'; -import { LocationSpec } from './types'; -import { results } from './index'; import { RequestHandler, rest } from 'msw'; import { setupServer } from 'msw/node'; +import { BitbucketRepository20, PagedResponse, PagedResponse20 } from './lib'; +import { LocationSpec, processingResult } from '../../api'; const server = setupServer(); @@ -755,7 +751,7 @@ describe('BitbucketDiscoveryProcessor', () => { }), { parser: async function* customRepositoryParser({}) { - yield results.location({ + yield processingResult.location({ type: 'custom-location-type', target: 'custom-target', presence: 'optional', diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts index 25f52b7dd7..daf24e2427 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Logger } from 'winston'; import { Config } from '@backstage/config'; @@ -28,13 +29,13 @@ import { paginated20, BitbucketRepository, BitbucketRepository20, -} from './bitbucket'; +} from './lib'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorResult, LocationSpec, -} from './types'; +} from '../../api'; const DEFAULT_BRANCH = 'master'; const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; diff --git a/plugins/catalog-backend/src/modules/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/index.ts new file mode 100644 index 0000000000..0e39083520 --- /dev/null +++ b/plugins/catalog-backend/src/modules/bitbucket/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; +export type { BitbucketRepositoryParser } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts similarity index 94% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts index e010865041..9f2d8b2daa 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { processingResult } from '../../../api'; import { defaultRepositoryParser } from './BitbucketRepositoryParser'; -import { results } from '../index'; describe('BitbucketRepositoryParser', () => { describe('defaultRepositoryParser', () => { @@ -23,7 +24,7 @@ describe('BitbucketRepositoryParser', () => { 'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse'; const path = '/catalog-info.yaml'; const expected = [ - results.location({ + processingResult.location({ type: 'url', target: `${browseUrl}${path}`, presence: 'optional', diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts similarity index 62% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts index 6671735f57..c72e560c21 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { CatalogProcessorResult } from '../types'; -import { results } from '../index'; -import { Logger } from 'winston'; + import { BitbucketIntegration } from '@backstage/integration'; +import { Logger } from 'winston'; +import { CatalogProcessorResult, processingResult } from '../../../api'; /** * @public @@ -28,17 +28,14 @@ export type BitbucketRepositoryParser = (options: { logger: Logger; }) => AsyncIterable; -export const defaultRepositoryParser = async function* defaultRepositoryParser({ - target, -}: { - target: string; -}) { - yield results.location({ - type: 'url', - target: target, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - presence: 'optional', - }); -}; +export const defaultRepositoryParser = + async function* defaultRepositoryParser(options: { target: string }) { + yield processingResult.location({ + type: 'url', + target: options.target, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + presence: 'optional', + }); + }; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/client.ts index 0c132bb61b..65cd28d5ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fetch from 'node-fetch'; +import fetch from 'node-fetch'; import { BitbucketIntegrationConfig, getBitbucketRequestOptions, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/index.ts index 4b28c7f115..a819bb3c64 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export { BitbucketClient, paginated, paginated20 } from './client'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; export type { PagedResponse, PagedResponse20 } from './client'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts index 8d6d1f8ff7..8cf250de2a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { CodeOwnersProcessor } from './CodeOwnersProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; const mockCodeOwnersText = () => ` * @acme/team-foo @acme/team-bar diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts index 93e3c1618a..6f6aee7161 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts @@ -22,11 +22,10 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import { findCodeOwnerByTarget } from './codeowners'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; +import { findCodeOwnerByTarget } from './lib'; const ALLOWED_KINDS = ['API', 'Component', 'Domain', 'Resource', 'System']; - const ALLOWED_LOCATION_TYPES = ['url']; /** @public */ diff --git a/plugins/catalog-backend/src/modules/codeowners/index.ts b/plugins/catalog-backend/src/modules/codeowners/index.ts new file mode 100644 index 0000000000..6b7569ff07 --- /dev/null +++ b/plugins/catalog-backend/src/modules/codeowners/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CodeOwnersProcessor } from './CodeOwnersProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts b/plugins/catalog-backend/src/modules/codeowners/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts b/plugins/catalog-backend/src/modules/codeowners/lib/scm.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/scm.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts index 5f5b20d1c2..3911f7f743 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; import { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; describe('AnnotateLocationEntityProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts index b8c31ed5a2..4af0b8cf61 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts @@ -25,7 +25,11 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** @public */ export class AnnotateLocationEntityProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts index 311dd13789..1e553a44f8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('AnnotateScmSlugEntityProcessor', () => { describe('github', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts index 4d3b3aff6c..e44db4d44d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import parseGitUrl from 'git-url-parse'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug'; diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts index 89b0bb1bb5..548cdf3115 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts @@ -52,8 +52,12 @@ import { TemplateEntityV1beta2, templateEntityV1beta2Validator, } from '@backstage/plugin-scaffolder-common'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** @public */ export class BuiltinKindsEntityProcessor implements CatalogProcessor { @@ -107,7 +111,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { for (const target of [targets].flat()) { const targetRef = parseEntityRef(target, context); emit( - result.relation({ + processingResult.relation({ source: selfRef, type: outgoingRelation, target: { @@ -118,7 +122,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { }), ); emit( - result.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace, diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts index 0663ad8d04..d61342e3d0 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts @@ -17,7 +17,7 @@ import { ConfigReader } from '@backstage/config'; import path from 'path'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { EntityProviderConnection } from './types'; +import { EntityProviderConnection } from '../../api'; describe('ConfigLocationEntityProvider', () => { it('should apply mutation with the correct paths in the config', async () => { diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts similarity index 90% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts index 389cd8d9ff..9529a5b608 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts @@ -16,9 +16,9 @@ import { Config } from '@backstage/config'; import path from 'path'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; export class ConfigLocationEntityProvider implements EntityProvider { constructor(private readonly config: Config) {} diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index b9d129de5f..dfbc37f6c9 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; -import { applyDatabaseMigrations } from '../database/migrations'; +import { applyDatabaseMigrations } from '../../database/migrations'; import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts similarity index 93% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts index 9d5971508f..d637f4eda0 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts @@ -18,11 +18,11 @@ import { Location } from '@backstage/catalog-client'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DbLocationsRow } from '../database/tables'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; -import { LocationInput, LocationStore } from '../service'; +import { DbLocationsRow } from '../../database/tables'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; +import { LocationInput, LocationStore } from '../../service'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts similarity index 94% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index c2cce66bc6..a3eb5e4542 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -19,12 +19,16 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; import path from 'path'; -import { defaultEntityDataParser } from './util/parse'; +import { defaultEntityDataParser } from '../util/parse'; describe('FileReaderProcessor', () => { - const fixturesRoot = path.join(__dirname, '__fixtures__/fileReaderProcessor'); + const fixturesRoot = path.join( + __dirname, + '__fixtures__', + 'fileReaderProcessor', + ); it('should load from file', async () => { const processor = new FileReaderProcessor(); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 996beff24a..66c6479fa7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -18,13 +18,13 @@ import fs from 'fs-extra'; import g from 'glob'; import path from 'path'; import { promisify } from 'util'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const glob = promisify(g); @@ -65,11 +65,11 @@ export class FileReaderProcessor implements CatalogProcessor { } } else if (!optional) { const message = `${location.type} ${location.target} does not exist`; - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } catch (e) { const message = `${location.type} ${location.target} could not be read, ${e}`; - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts index 83f1ca1925..92ba1fcebf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import path from 'path'; import { toAbsoluteUrl } from './LocationEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('LocationEntityProcessor', () => { describe('toAbsoluteUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts index 533a366f57..c168f54bf1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts @@ -17,8 +17,12 @@ import { Entity, LocationEntity } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; export function toAbsoluteUrl( integrations: ScmIntegrationRegistry, @@ -62,7 +66,7 @@ export class LocationEntityProcessor implements CatalogProcessor { const type = locationEntity.spec.type || location.type; if (type === 'file' && location.target.endsWith(path.sep)) { emit( - result.inputError( + processingResult.inputError( location, `LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, ), @@ -83,7 +87,7 @@ export class LocationEntityProcessor implements CatalogProcessor { location, maybeRelativeTarget, ); - emit(result.location({ type, target })); + emit(processingResult.location({ type, target })); } } diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 0d572a024d..ddd5f951db 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; diff --git a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts rename to plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts index 1c2c27f425..e22d011695 100644 --- a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts @@ -15,8 +15,11 @@ */ import { Config } from '@backstage/config'; -import * as result from './results'; -import { CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** * @deprecated no longer in use, replaced by the ConfigLocationEntityProvider. @@ -48,7 +51,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor { } for (const staticLocation of this.staticLocations) { - emit(result.location(staticLocation)); + emit(processingResult.location(staticLocation)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index b7dc0084f2..a96d392c52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -28,9 +28,9 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import { UrlReaderProcessor } from './UrlReaderProcessor'; -import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index c2be454bcf..7f27bcd19d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -20,7 +20,6 @@ import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorCache, @@ -29,7 +28,8 @@ import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const CACHE_KEY = 'v1'; @@ -102,10 +102,10 @@ export class UrlReaderProcessor implements CatalogProcessor { } } else if (error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt @@ -0,0 +1 @@ +site_name: Test diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/modules/core/index.ts similarity index 63% rename from plugins/catalog-backend/src/ingestion/processors/index.ts rename to plugins/catalog-backend/src/modules/core/index.ts index 84a92fb1ba..9d582bfb34 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/modules/core/index.ts @@ -14,20 +14,10 @@ * limitations under the License. */ -import * as results from './results'; - export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; -export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; -export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; -export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; -export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; -export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export type { LocationEntityProcessorOptions } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; @@ -39,10 +29,5 @@ export type { PlaceholderResolverResolveUrl, } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; -export * from './types'; export { UrlReaderProcessor } from './UrlReaderProcessor'; -export { parseEntityYaml } from './util/parse'; -export { results }; - -export type { BitbucketRepositoryParser } from './bitbucket'; -export type { GithubMultiOrgConfig } from './github'; +export { parseEntityYaml } from '../util/parse'; diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts index e7aec1520d..dd3bf490fb 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts @@ -20,10 +20,12 @@ import { GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; -import { GitHubOrgEntityProvider } from '.'; -import { EntityProviderConnection } from '../../providers'; -import { withLocations } from './GitHubOrgEntityProvider'; import { graphql } from '@octokit/graphql'; +import { EntityProviderConnection } from '../../api'; +import { + GitHubOrgEntityProvider, + withLocations, +} from './GitHubOrgEntityProvider'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts index a20f7fdb0b..76389968ae 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -29,13 +30,13 @@ import { import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; -import { EntityProvider, EntityProviderConnection } from '../../providers'; +import { EntityProvider, EntityProviderConnection } from '../../api'; import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from '../processors/github'; -import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; +} from './lib'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; // TODO: Consider supporting an (optional) webhook that reacts on org changes /** @public */ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts index 55e5560451..5440539b27 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts @@ -15,16 +15,16 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; -import { getOrganizationRepositories } from './github'; -import { LocationSpec } from './types'; import { ConfigReader } from '@backstage/config'; import { - ScmIntegrations, DefaultGithubCredentialsProvider, + ScmIntegrations, } from '@backstage/integration'; +import { LocationSpec } from '../../api'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './lib'; -jest.mock('./github'); +jest.mock('./lib'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts index e790134011..4aff470f52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts @@ -23,9 +23,13 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationRepositories } from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { getOrganizationRepositories } from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** * Extracts repositories out of a GitHub org. @@ -141,7 +145,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { const path = `/blob/${branchName}${catalogPath}`; emit( - results.location({ + processingResult.location({ type: 'url', target: `${repository.url}${path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts index 383b82c019..339119137a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts @@ -30,10 +30,14 @@ import { getOrganizationUsers, GithubMultiOrgConfig, readGithubMultiOrgConfig, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { buildOrgHierarchy } from '../util/org'; /** * @alpha @@ -158,7 +162,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { buildOrgHierarchy(groups); for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } } catch (e) { this.logger.error( @@ -169,7 +173,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const allUsers = Array.from(allUsersMap.values()); for (const user of allUsers) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts index c01db88d3a..9ce278ac44 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { @@ -21,7 +22,7 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts index 8b5914cc80..e85e7b874d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts @@ -28,10 +28,14 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { assignGroupsToUsers, buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; type GraphQL = typeof graphql; @@ -106,10 +110,10 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/modules/github/index.ts b/plugins/catalog-backend/src/modules/github/index.ts new file mode 100644 index 0000000000..7958818096 --- /dev/null +++ b/plugins/catalog-backend/src/modules/github/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts b/plugins/catalog-backend/src/modules/github/lib/config.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.test.ts rename to plugins/catalog-backend/src/modules/github/lib/config.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.ts b/plugins/catalog-backend/src/modules/github/lib/config.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.ts rename to plugins/catalog-backend/src/modules/github/lib/config.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/modules/github/lib/github.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.test.ts rename to plugins/catalog-backend/src/modules/github/lib/github.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/modules/github/lib/github.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.ts rename to plugins/catalog-backend/src/modules/github/lib/github.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/modules/github/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/index.ts rename to plugins/catalog-backend/src/modules/github/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/modules/github/lib/util.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.test.ts rename to plugins/catalog-backend/src/modules/github/lib/util.test.ts index 02f54e0675..c73c11bd2d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { parseGitHubOrgUrl } from './util'; describe('parseGitHubOrgUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/modules/github/lib/util.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.ts rename to plugins/catalog-backend/src/modules/github/lib/util.ts index d8df376038..a38225ff98 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export function parseGitHubOrgUrl(urlString: string): { org: string } { const path = new URL(urlString).pathname.substr(1).split('/'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts index 8261690e8f..cc0f368aee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts @@ -19,8 +19,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from './gitlab'; -import { LocationSpec } from './types'; +import { GitLabProject } from './lib'; +import { LocationSpec } from '../../api'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts index b534d50044..d3916f45d1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { GitLabClient, GitLabProject, paginated } from './lib'; import { CacheClient, CacheManager, @@ -95,12 +99,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { page: 1, }); - const result: Result = { + const res: Result = { scanned: 0, matches: [], }; for await (const project of projects) { - result.scanned++; + res.scanned++; if (project.archived) { continue; @@ -110,14 +114,14 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { continue; } - result.matches.push(project); + res.matches.push(project); } - for (const project of result.matches) { + for (const project of res.matches) { const project_branch = branch === '*' ? project.default_branch : branch; emit( - results.location({ + processingResult.location({ type: 'url', // The format expected by the GitLabUrlReader: // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath @@ -133,7 +137,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( - `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + `Read ${res.scanned} GitLab repositories in ${duration} seconds`, ); return true; diff --git a/plugins/catalog-backend/src/modules/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/index.ts new file mode 100644 index 0000000000..f42a901f8f --- /dev/null +++ b/plugins/catalog-backend/src/modules/gitlab/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts index b249137740..a7c0d5161f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ConfigReader } from '@backstage/config'; import { setupRequestMockHandlers } from '@backstage/test-utils'; import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; import { rest } from 'msw'; import { setupServer, SetupServerApi } from 'msw/node'; - import { GitLabClient, paginated } from './client'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/modules/gitlab/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/types.ts diff --git a/plugins/catalog-backend/src/modules/index.ts b/plugins/catalog-backend/src/modules/index.ts new file mode 100644 index 0000000000..6acf057ca3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './aws'; +export * from './azure'; +export * from './bitbucket'; +export * from './codeowners'; +export * from './core'; +export * from './github'; +export * from './gitlab'; diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/modules/util/org.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.test.ts rename to plugins/catalog-backend/src/modules/util/org.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/modules/util/org.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.ts rename to plugins/catalog-backend/src/modules/util/org.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts b/plugins/catalog-backend/src/modules/util/parse.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts rename to plugins/catalog-backend/src/modules/util/parse.test.ts index a42d160725..e87de1c9a3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts +++ b/plugins/catalog-backend/src/modules/util/parse.test.ts @@ -15,7 +15,7 @@ */ import { parseEntityYaml } from './parse'; -import * as result from '../results'; +import { processingResult } from '../../api'; const testLoc = { target: 'my-loc-target', @@ -47,7 +47,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -92,7 +92,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -102,7 +102,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -137,7 +137,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -157,7 +157,7 @@ describe('parseEntityYaml', () => { // Parse errors are always per document expect(results).toEqual([ - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Plain value cannot start with reserved character `', ), @@ -186,7 +186,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -196,7 +196,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Nested mappings are not allowed in compact mappings', ), @@ -209,7 +209,10 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.generalError(testLoc, 'Expected object at root, got string'), + processingResult.generalError( + testLoc, + 'Expected object at root, got string', + ), ]); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/modules/util/parse.ts similarity index 87% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.ts rename to plugins/catalog-backend/src/modules/util/parse.ts index 0beaf1cd4c..65fad5986f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/modules/util/parse.ts @@ -17,12 +17,12 @@ import { Entity, stringifyLocationRef } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; -import * as result from '../results'; import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from '../types'; + processingResult, +} from '../../api'; /** @public */ export function* parseEntityYaml( @@ -35,7 +35,7 @@ export function* parseEntityYaml( } catch (e) { const loc = stringifyLocationRef(location); const message = `Failed to parse YAML at ${loc}, ${e}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); return; } @@ -43,17 +43,17 @@ export function* parseEntityYaml( if (document.errors?.length) { const loc = stringifyLocationRef(location); const message = `YAML error at ${loc}, ${document.errors[0]}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } else { const json = document.toJSON(); if (lodash.isPlainObject(json)) { - yield result.entity(location, json as Entity); + yield processingResult.entity(location, json as Entity); } else if (json === null) { // Ignore null values, these happen if there is an empty document in the // YAML file, for example if --- is added to the end of the file. } else { const message = `Expected object at root, got ${typeof json}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } } } diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 0853484408..6e494d784e 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -29,11 +29,11 @@ import { CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, - results, -} from '../ingestion'; + processingResult, +} from '../api'; import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { ConfigReader } from '@backstage/config'; class FooBarProcessor implements CatalogProcessor { @@ -51,7 +51,7 @@ class FooBarProcessor implements CatalogProcessor { ) { if (await cache.get('emit')) { emit( - results.entity( + processingResult.entity( { type: 'url', target: './new-place' }, { apiVersion: 'my-api/v1', @@ -63,7 +63,7 @@ class FooBarProcessor implements CatalogProcessor { ), ); emit( - results.relation({ + processingResult.relation({ type: 'my-type', source: { kind: 'foobar', name: 'my-source', namespace: 'default' }, target: { kind: 'foobar', name: 'my-target', namespace: 'default' }, @@ -211,7 +211,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { getProcessorName: jest.fn(), validateEntityKind: jest.fn(async () => true), readLocation: jest.fn(async (_l, _o, emit) => { - emit(results.entity({ type: 't', target: 't' }, entity)); + emit(processingResult.entity({ type: 't', target: 't' }, entity)); return true; }), }; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index 9649bc937b..c050779a8b 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -36,8 +36,8 @@ import { CatalogProcessor, CatalogProcessorParser, LocationSpec, -} from '../ingestion/processors'; -import * as results from '../ingestion/processors/results'; + processingResult, +} from '../api'; import { CatalogProcessingOrchestrator, EntityProcessingRequest, @@ -178,13 +178,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.preProcessEntity) { try { - result = await processor.preProcessEntity( - result, + res = await processor.preProcessEntity( + res, context.location, context.collector.onEmit, context.originLocation, @@ -199,7 +199,7 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } /** @@ -295,7 +295,7 @@ export class DefaultCatalogProcessingOrchestrator for (const maybeRelativeTarget of targets) { if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { context.collector.onEmit( - results.inputError( + processingResult.inputError( context.location, `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`, ), @@ -351,13 +351,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.postProcessEntity) { try { - result = await processor.postProcessEntity( - result, + res = await processor.postProcessEntity( + res, context.location, context.collector.onEmit, context.cache.forProcessor(processor), @@ -371,6 +371,6 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } } diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts index 71631b13d8..602557a224 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { CatalogProcessor } from '../ingestion/processors'; +import { CatalogProcessor } from '../api'; import { ProcessorCacheManager } from './ProcessorCacheManager'; class MyProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts index ab95fcedb7..e54641f572 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts @@ -15,8 +15,7 @@ */ import { JsonObject, JsonValue } from '@backstage/types'; -import { CatalogProcessor } from '../ingestion/processors'; -import { CatalogProcessorCache } from '../ingestion/processors/types'; +import { CatalogProcessor, CatalogProcessorCache } from '../api'; import { isObject } from './util'; class SingleProcessorSubCache implements CatalogProcessorCache { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 351e68882b..c3cc6e288b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -22,9 +22,9 @@ import { } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; -import { CatalogProcessorResult } from '../ingestion'; +import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity, EntityRelationSpec } from './types'; +import { DeferredEntity } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts index 02bdc2aee3..d7015ea52e 100644 --- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts @@ -23,7 +23,7 @@ import { EntityProvider, EntityProviderConnection, EntityProviderMutation, -} from '../providers/types'; +} from '../api'; class Connection implements EntityProviderConnection { readonly validateEntityEnvelope = entityEnvelopeSchemaValidator(); diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index 83f8a9f566..dc8b07eceb 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -19,7 +19,6 @@ export type { CatalogProcessingEngine, EntityProcessingRequest, EntityProcessingResult, - EntityRelationSpec, DeferredEntity, } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b732df297e..484cd73939 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -14,30 +14,9 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; - -/** - * Holds the relation data for entities. - * - * @public - */ -export type EntityRelationSpec = { - /** - * The source entity of this relation. - */ - source: EntityName; - - /** - * The type of the relation. - */ - type: string; - - /** - * The target entity of this relation. - */ - target: EntityName; -}; +import { EntityRelationSpec } from '../api'; /** * The request to process an entity. diff --git a/plugins/catalog-backend/src/processing/util.ts b/plugins/catalog-backend/src/processing/util.ts index 94634f794a..b1948cf559 100644 --- a/plugins/catalog-backend/src/processing/util.ts +++ b/plugins/catalog-backend/src/processing/util.ts @@ -27,7 +27,7 @@ import { JsonObject, JsonValue } from '@backstage/types'; import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function isLocationEntity(entity: Entity): entity is LocationEntity { return entity.kind === 'Location'; diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 819451d854..17dfb58c54 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { NotAllowedError } from '@backstage/errors'; import { catalogEntityRefreshPermission } from '@backstage/plugin-catalog-common'; import { diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9004cb4c56..f62377decb 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,12 +38,15 @@ import { Router } from 'express'; import lodash, { keyBy } from 'lodash'; import { EntitiesCatalog, EntitiesSearchFilter } from '../catalog'; +import { + CatalogProcessor, + CatalogProcessorParser, + EntityProvider, +} from '../api'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, BuiltinKindsEntityProcessor, - CatalogProcessor, - CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, AzureDevOpsDiscoveryProcessor, @@ -53,23 +56,22 @@ import { PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, -} from '../ingestion'; +} from '../modules'; +import { ConfigLocationEntityProvider } from '../modules/core/ConfigLocationEntityProvider'; +import { DefaultLocationStore } from '../modules/core/DefaultLocationStore'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; import { jsonPlaceholderResolver, textPlaceholderResolver, yamlPlaceholderResolver, -} from '../ingestion/processors/PlaceholderProcessor'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +} from '../modules/core/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; -import { EntityProvider } from '../providers/types'; import { CatalogProcessingEngine } from '../processing/types'; -import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; -import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.ts b/plugins/catalog-backend/src/service/DefaultLocationService.ts index c6b5eaf162..7a1b923068 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, ANNOTATION_LOCATION, diff --git a/plugins/catalog-backend/src/util/conversion.ts b/plugins/catalog-backend/src/util/conversion.ts index 75ed4f5e96..63771d1028 100644 --- a/plugins/catalog-backend/src/util/conversion.ts +++ b/plugins/catalog-backend/src/util/conversion.ts @@ -23,7 +23,7 @@ import { stringifyLocationRef, } from '@backstage/catalog-model'; import { createHash } from 'crypto'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function locationSpecToMetadataName(location: LocationSpec) { const hash = createHash('sha1') diff --git a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts index 43559fcd5f..5fcd89841e 100644 --- a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts +++ b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts @@ -25,7 +25,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { TemplateEntityV1beta3, @@ -70,7 +70,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { defaultNamespace: selfRef.namespace, }); emit( - results.relation({ + processingResult.relation({ source: selfRef, type: RELATION_OWNED_BY, target: { @@ -81,7 +81,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { }), ); emit( - results.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace,