From e5bf3749ad0ed8789a861310acb0a7245c246699 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Thu, 12 Oct 2023 19:58:54 -0400 Subject: [PATCH] feat(catalog): support adding location analyzers in processing extension Signed-off-by: Phil Kuang --- .changeset/blue-bags-warn.md | 7 +++++++ .../api-report.md | 4 ++-- .../src/analyzers/GithubLocationAnalyzer.ts | 2 +- plugins/catalog-backend/api-report.md | 16 ++++------------ .../src/ingestion/LocationAnalyzer.ts | 3 ++- plugins/catalog-backend/src/ingestion/index.ts | 10 ++++++++-- plugins/catalog-backend/src/ingestion/types.ts | 16 ---------------- .../src/service/CatalogBuilder.ts | 3 ++- .../src/service/CatalogPlugin.ts | 13 +++++++++++++ plugins/catalog-node/alpha-api-report.md | 5 +++++ plugins/catalog-node/api-report.md | 15 +++++++++++++++ plugins/catalog-node/src/extensions.ts | 4 ++++ plugins/catalog-node/src/processing/index.ts | 2 ++ plugins/catalog-node/src/processing/types.ts | 18 ++++++++++++++++++ 14 files changed, 83 insertions(+), 35 deletions(-) create mode 100644 .changeset/blue-bags-warn.md diff --git a/.changeset/blue-bags-warn.md b/.changeset/blue-bags-warn.md new file mode 100644 index 0000000000..9d74aa3320 --- /dev/null +++ b/.changeset/blue-bags-warn.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-node': minor +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Support adding location analyzers in catalog processing extension point and move `AnalyzeOptions` and `ScmLocationAnalyzer` types to `@backstage/plugin-catalog-node` diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 4c44f95b61..eb43748aa1 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -3,7 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; +import { AnalyzeOptions } from '@backstage/plugin-catalog-node'; import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; @@ -21,7 +21,7 @@ import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ScmIntegrationRegistry } from '@backstage/integration'; -import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend'; +import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node'; import { TaskRunner } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; import { UserEntity } from '@backstage/catalog-model'; diff --git a/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.ts b/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.ts index 9341743413..3353254400 100644 --- a/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.ts +++ b/plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.ts @@ -27,7 +27,7 @@ import parseGitUrl from 'git-url-parse'; import { AnalyzeOptions, ScmLocationAnalyzer, -} from '@backstage/plugin-catalog-backend'; +} from '@backstage/plugin-catalog-node'; import { PluginEndpointDiscovery, TokenManager, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index e426ca1d75..d9c241e99c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -10,6 +10,7 @@ import { AnalyzeLocationExistingEntity as AnalyzeLocationExistingEntity_2 } from import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from '@backstage/plugin-catalog-common'; import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common'; import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common'; +import { AnalyzeOptions } from '@backstage/plugin-catalog-node'; import { CatalogApi } from '@backstage/catalog-client'; import type { CatalogCollatorEntityTransformer as CatalogCollatorEntityTransformer_2 } from '@backstage/plugin-search-backend-module-catalog'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; @@ -50,6 +51,7 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node'; import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; import { Validators } from '@backstage/catalog-model'; @@ -69,11 +71,7 @@ export type AnalyzeLocationRequest = AnalyzeLocationRequest_2; // @public @deprecated (undocumented) export type AnalyzeLocationResponse = AnalyzeLocationResponse_2; -// @public (undocumented) -export type AnalyzeOptions = { - url: string; - catalogFilename?: string; -}; +export { AnalyzeOptions }; // @public (undocumented) export class AnnotateLocationEntityProcessor implements CatalogProcessor_2 { @@ -468,13 +466,7 @@ export const processingResult: Readonly<{ readonly refresh: (key: string) => CatalogProcessorResult_2; }>; -// @public (undocumented) -export type ScmLocationAnalyzer = { - supports(url: string): boolean; - analyze(options: AnalyzeOptions): Promise<{ - existing: AnalyzeLocationExistingEntity[]; - }>; -}; +export { ScmLocationAnalyzer }; // @public (undocumented) export class UrlReaderProcessor implements CatalogProcessor_2 { diff --git a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts index 761a0f8c0a..ad752b770d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts +++ b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts @@ -18,11 +18,12 @@ import { Logger } from 'winston'; import parseGitUrl from 'git-url-parse'; import { Entity } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; -import { LocationAnalyzer, ScmLocationAnalyzer } from './types'; +import { LocationAnalyzer } from './types'; import { AnalyzeLocationRequest, AnalyzeLocationResponse, } from '@backstage/plugin-catalog-common'; +import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node'; export class RepoLocationAnalyzer implements LocationAnalyzer { private readonly logger: Logger; diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 0b00809043..86a8179d8d 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -21,6 +21,12 @@ export type { AnalyzeLocationRequest, AnalyzeLocationResponse, LocationAnalyzer, - ScmLocationAnalyzer, - AnalyzeOptions, } from './types'; + +/** + * @deprecated use the same type from `@backstage/plugin-catalog-node` instead + */ +export type { + AnalyzeOptions, + ScmLocationAnalyzer, +} from '@backstage/plugin-catalog-node'; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 78339b3b23..3875d3af03 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -78,19 +78,3 @@ export type LocationAnalyzer = { location: AnalyzeLocationRequest, ): Promise; }; -/** @public */ -export type AnalyzeOptions = { - url: string; - catalogFilename?: string; -}; - -/** @public */ -export type ScmLocationAnalyzer = { - /** The method that decides if this analyzer can work with the provided url */ - supports(url: string): boolean; - /** This function can return an array of already existing entities */ - analyze(options: AnalyzeOptions): Promise<{ - /** Existing entities in the analyzed location */ - existing: AnalyzeLocationExistingEntity[]; - }>; -}; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index c12178e888..5cf76ee1ac 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,6 +38,7 @@ import { CatalogProcessor, CatalogProcessorParser, EntityProvider, + ScmLocationAnalyzer, } from '@backstage/plugin-catalog-node'; import { AnnotateLocationEntityProcessor, @@ -57,7 +58,7 @@ import { yamlPlaceholderResolver, } from '../modules/core/PlaceholderProcessor'; import { defaultEntityDataParser } from '../modules/util/parse'; -import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types'; +import { LocationAnalyzer } from '../ingestion/types'; import { CatalogProcessingEngine } from '../processing'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index b8d5bdbeda..d112429fc4 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -25,6 +25,7 @@ import { import { CatalogProcessor, EntityProvider, + ScmLocationAnalyzer, } from '@backstage/plugin-catalog-node'; import { loggerToWinstonLogger } from '@backstage/backend-common'; import { PlaceholderResolver } from '../modules'; @@ -33,6 +34,7 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint { #processors = new Array(); #entityProviders = new Array(); #placeholderResolvers: Record = {}; + #locationAnalyzers = new Array(); addProcessor( ...processors: Array> @@ -54,6 +56,12 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint { this.#placeholderResolvers[key] = resolver; } + addLocationAnalyzers( + ...analyzers: Array> + ): void { + this.#locationAnalyzers.push(...analyzers.flat()); + } + get processors() { return this.#processors; } @@ -65,6 +73,10 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint { get placeholderResolvers() { return this.#placeholderResolvers; } + + get locationAnalyzers() { + return this.#locationAnalyzers; + } } /** @@ -116,6 +128,7 @@ export const catalogPlugin = createBackendPlugin({ Object.entries(processingExtensions.placeholderResolvers).forEach( ([key, resolver]) => builder.setPlaceholderResolver(key, resolver), ); + builder.addLocationAnalyzers(...processingExtensions.locationAnalyzers); const { processingEngine, router } = await builder.build(); diff --git a/plugins/catalog-node/alpha-api-report.md b/plugins/catalog-node/alpha-api-report.md index 082228a58e..f3de6a8b33 100644 --- a/plugins/catalog-node/alpha-api-report.md +++ b/plugins/catalog-node/alpha-api-report.md @@ -8,6 +8,7 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { PlaceholderResolver } from '@backstage/plugin-catalog-node'; +import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -17,6 +18,10 @@ export interface CatalogProcessingExtensionPoint { ...providers: Array> ): void; // (undocumented) + addLocationAnalyzers( + ...analyzers: Array> + ): void; + // (undocumented) addPlaceholderResolver(key: string, resolver: PlaceholderResolver): void; // (undocumented) addProcessor( diff --git a/plugins/catalog-node/api-report.md b/plugins/catalog-node/api-report.md index 34d7a4bc9b..ef2c09e61f 100644 --- a/plugins/catalog-node/api-report.md +++ b/plugins/catalog-node/api-report.md @@ -5,12 +5,19 @@ ```ts /// +import { AnalyzeLocationExistingEntity } from '@backstage/plugin-catalog-common'; import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { LocationEntityV1alpha1 } from '@backstage/catalog-model'; import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common'; +// @public (undocumented) +export type AnalyzeOptions = { + url: string; + catalogFilename?: string; +}; + // @public (undocumented) export type CatalogProcessor = { getProcessorName(): string; @@ -198,4 +205,12 @@ export const processingResult: Readonly<{ readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; readonly refresh: (key: string) => CatalogProcessorResult; }>; + +// @public (undocumented) +export type ScmLocationAnalyzer = { + supports(url: string): boolean; + analyze(options: AnalyzeOptions): Promise<{ + existing: AnalyzeLocationExistingEntity[]; + }>; +}; ``` diff --git a/plugins/catalog-node/src/extensions.ts b/plugins/catalog-node/src/extensions.ts index 5225ad7545..4e8f7127f1 100644 --- a/plugins/catalog-node/src/extensions.ts +++ b/plugins/catalog-node/src/extensions.ts @@ -18,6 +18,7 @@ import { EntityProvider, CatalogProcessor, PlaceholderResolver, + ScmLocationAnalyzer, } from '@backstage/plugin-catalog-node'; /** @@ -31,6 +32,9 @@ export interface CatalogProcessingExtensionPoint { ...providers: Array> ): void; addPlaceholderResolver(key: string, resolver: PlaceholderResolver): void; + addLocationAnalyzers( + ...analyzers: Array> + ): void; } /** diff --git a/plugins/catalog-node/src/processing/index.ts b/plugins/catalog-node/src/processing/index.ts index 40cee63e25..8cffac6961 100644 --- a/plugins/catalog-node/src/processing/index.ts +++ b/plugins/catalog-node/src/processing/index.ts @@ -15,9 +15,11 @@ */ export type { + AnalyzeOptions, DeferredEntity, PlaceholderResolver, PlaceholderResolverParams, PlaceholderResolverRead, PlaceholderResolverResolveUrl, + ScmLocationAnalyzer, } from './types'; diff --git a/plugins/catalog-node/src/processing/types.ts b/plugins/catalog-node/src/processing/types.ts index b98e2da1b7..b6785c1b91 100644 --- a/plugins/catalog-node/src/processing/types.ts +++ b/plugins/catalog-node/src/processing/types.ts @@ -15,6 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; +import { AnalyzeLocationExistingEntity } from '@backstage/plugin-catalog-common'; import { JsonValue } from '@backstage/types'; import { CatalogProcessorEmit } from '../api'; @@ -50,3 +51,20 @@ export type PlaceholderResolverParams = { export type PlaceholderResolver = ( params: PlaceholderResolverParams, ) => Promise; + +/** @public */ +export type AnalyzeOptions = { + url: string; + catalogFilename?: string; +}; + +/** @public */ +export type ScmLocationAnalyzer = { + /** The method that decides if this analyzer can work with the provided url */ + supports(url: string): boolean; + /** This function can return an array of already existing entities */ + analyze(options: AnalyzeOptions): Promise<{ + /** Existing entities in the analyzed location */ + existing: AnalyzeLocationExistingEntity[]; + }>; +};