feat(catalog): support adding location analyzers in processing extension

Signed-off-by: Phil Kuang <pkuang@factset.com>

.changeset/blue-bags-warn.md

@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-catalog-backend': minor
+'@backstage/plugin-catalog-node': minor
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+Support adding location analyzers in catalog processing extension point and move `AnalyzeOptions` and `ScmLocationAnalyzer` types to `@backstage/plugin-catalog-node`

plugins/catalog-backend-module-github/api-report.md

@@ -3,7 +3,7 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
-import { AnalyzeOptions } from '@backstage/plugin-catalog-backend';
+import { AnalyzeOptions } from '@backstage/plugin-catalog-node';
 import { CatalogProcessor } from '@backstage/plugin-catalog-node';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
 import { Config } from '@backstage/config';
@@ -21,7 +21,7 @@ import { Logger } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { ScmIntegrationRegistry } from '@backstage/integration';
-import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
+import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { TokenManager } from '@backstage/backend-common';
 import { UserEntity } from '@backstage/catalog-model';

plugins/catalog-backend-module-github/src/analyzers/GithubLocationAnalyzer.ts

@@ -27,7 +27,7 @@ import parseGitUrl from 'git-url-parse';
 import {
   AnalyzeOptions,
   ScmLocationAnalyzer,
-} from '@backstage/plugin-catalog-backend';
+} from '@backstage/plugin-catalog-node';
 import {
   PluginEndpointDiscovery,
   TokenManager,

plugins/catalog-backend/api-report.md

@@ -10,6 +10,7 @@ import { AnalyzeLocationExistingEntity as AnalyzeLocationExistingEntity_2 } from
 import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from '@backstage/plugin-catalog-common';
 import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common';
 import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common';
+import { AnalyzeOptions } from '@backstage/plugin-catalog-node';
 import { CatalogApi } from '@backstage/catalog-client';
 import type { CatalogCollatorEntityTransformer as CatalogCollatorEntityTransformer_2 } from '@backstage/plugin-search-backend-module-catalog';
 import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
@@ -50,6 +51,7 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { Router } from 'express';
 import { ScmIntegrationRegistry } from '@backstage/integration';
+import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node';
 import { TokenManager } from '@backstage/backend-common';
 import { UrlReader } from '@backstage/backend-common';
 import { Validators } from '@backstage/catalog-model';
@@ -69,11 +71,7 @@ export type AnalyzeLocationRequest = AnalyzeLocationRequest_2;
 // @public @deprecated (undocumented)
 export type AnalyzeLocationResponse = AnalyzeLocationResponse_2;
 
-// @public (undocumented)
-export type AnalyzeOptions = {
-  url: string;
-  catalogFilename?: string;
-};
+export { AnalyzeOptions };
 
 // @public (undocumented)
 export class AnnotateLocationEntityProcessor implements CatalogProcessor_2 {
@@ -468,13 +466,7 @@ export const processingResult: Readonly<{
   readonly refresh: (key: string) => CatalogProcessorResult_2;
 }>;
 
-// @public (undocumented)
-export type ScmLocationAnalyzer = {
-  supports(url: string): boolean;
-  analyze(options: AnalyzeOptions): Promise<{
-    existing: AnalyzeLocationExistingEntity[];
-  }>;
-};
+export { ScmLocationAnalyzer };
 
 // @public (undocumented)
 export class UrlReaderProcessor implements CatalogProcessor_2 {

plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts

@@ -18,11 +18,12 @@ import { Logger } from 'winston';
 import parseGitUrl from 'git-url-parse';
 import { Entity } from '@backstage/catalog-model';
 import { ScmIntegrationRegistry } from '@backstage/integration';
-import { LocationAnalyzer, ScmLocationAnalyzer } from './types';
+import { LocationAnalyzer } from './types';
 import {
   AnalyzeLocationRequest,
   AnalyzeLocationResponse,
 } from '@backstage/plugin-catalog-common';
+import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node';
 
 export class RepoLocationAnalyzer implements LocationAnalyzer {
   private readonly logger: Logger;

plugins/catalog-backend/src/ingestion/index.ts

@@ -21,6 +21,12 @@ export type {
   AnalyzeLocationRequest,
   AnalyzeLocationResponse,
   LocationAnalyzer,
-  ScmLocationAnalyzer,
-  AnalyzeOptions,
 } from './types';
+
+/**
+ * @deprecated use the same type from `@backstage/plugin-catalog-node` instead
+ */
+export type {
+  AnalyzeOptions,
+  ScmLocationAnalyzer,
+} from '@backstage/plugin-catalog-node';

plugins/catalog-backend/src/ingestion/types.ts

@@ -78,19 +78,3 @@ export type LocationAnalyzer = {
     location: AnalyzeLocationRequest,
   ): Promise<AnalyzeLocationResponse>;
 };
-/** @public */
-export type AnalyzeOptions = {
-  url: string;
-  catalogFilename?: string;
-};
-
-/** @public */
-export type ScmLocationAnalyzer = {
-  /** The method that decides if this analyzer can work with the provided url */
-  supports(url: string): boolean;
-  /** This function can return an array of already existing entities */
-  analyze(options: AnalyzeOptions): Promise<{
-    /** Existing entities in the analyzed location */
-    existing: AnalyzeLocationExistingEntity[];
-  }>;
-};

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -38,6 +38,7 @@ import {
   CatalogProcessor,
   CatalogProcessorParser,
   EntityProvider,
+  ScmLocationAnalyzer,
 } from '@backstage/plugin-catalog-node';
 import {
   AnnotateLocationEntityProcessor,
@@ -57,7 +58,7 @@ import {
   yamlPlaceholderResolver,
 } from '../modules/core/PlaceholderProcessor';
 import { defaultEntityDataParser } from '../modules/util/parse';
-import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types';
+import { LocationAnalyzer } from '../ingestion/types';
 import { CatalogProcessingEngine } from '../processing';
 import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
 import { applyDatabaseMigrations } from '../database/migrations';

plugins/catalog-backend/src/service/CatalogPlugin.ts

@@ -25,6 +25,7 @@ import {
 import {
   CatalogProcessor,
   EntityProvider,
+  ScmLocationAnalyzer,
 } from '@backstage/plugin-catalog-node';
 import { loggerToWinstonLogger } from '@backstage/backend-common';
 import { PlaceholderResolver } from '../modules';
@@ -33,6 +34,7 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint {
   #processors = new Array<CatalogProcessor>();
   #entityProviders = new Array<EntityProvider>();
   #placeholderResolvers: Record<string, PlaceholderResolver> = {};
+  #locationAnalyzers = new Array<ScmLocationAnalyzer>();
 
   addProcessor(
     ...processors: Array<CatalogProcessor | Array<CatalogProcessor>>
@@ -54,6 +56,12 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint {
     this.#placeholderResolvers[key] = resolver;
   }
 
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): void {
+    this.#locationAnalyzers.push(...analyzers.flat());
+  }
+
   get processors() {
     return this.#processors;
   }
@@ -65,6 +73,10 @@ class CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint {
   get placeholderResolvers() {
     return this.#placeholderResolvers;
   }
+
+  get locationAnalyzers() {
+    return this.#locationAnalyzers;
+  }
 }
 
 /**
@@ -116,6 +128,7 @@ export const catalogPlugin = createBackendPlugin({
         Object.entries(processingExtensions.placeholderResolvers).forEach(
           ([key, resolver]) => builder.setPlaceholderResolver(key, resolver),
         );
+        builder.addLocationAnalyzers(...processingExtensions.locationAnalyzers);
 
         const { processingEngine, router } = await builder.build();
 

plugins/catalog-node/alpha-api-report.md

@@ -8,6 +8,7 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-node';
 import { EntityProvider } from '@backstage/plugin-catalog-node';
 import { ExtensionPoint } from '@backstage/backend-plugin-api';
 import { PlaceholderResolver } from '@backstage/plugin-catalog-node';
+import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node';
 import { ServiceRef } from '@backstage/backend-plugin-api';
 
 // @alpha (undocumented)
@@ -17,6 +18,10 @@ export interface CatalogProcessingExtensionPoint {
     ...providers: Array<EntityProvider | Array<EntityProvider>>
   ): void;
   // (undocumented)
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): void;
+  // (undocumented)
   addPlaceholderResolver(key: string, resolver: PlaceholderResolver): void;
   // (undocumented)
   addProcessor(

plugins/catalog-node/api-report.md

@@ -5,12 +5,19 @@
 ```ts
 /// <reference types="node" />
 
+import { AnalyzeLocationExistingEntity } from '@backstage/plugin-catalog-common';
 import { CompoundEntityRef } from '@backstage/catalog-model';
 import { Entity } from '@backstage/catalog-model';
 import { JsonValue } from '@backstage/types';
 import { LocationEntityV1alpha1 } from '@backstage/catalog-model';
 import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common';
 
+// @public (undocumented)
+export type AnalyzeOptions = {
+  url: string;
+  catalogFilename?: string;
+};
+
 // @public (undocumented)
 export type CatalogProcessor = {
   getProcessorName(): string;
@@ -198,4 +205,12 @@ export const processingResult: Readonly<{
   readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult;
   readonly refresh: (key: string) => CatalogProcessorResult;
 }>;
+
+// @public (undocumented)
+export type ScmLocationAnalyzer = {
+  supports(url: string): boolean;
+  analyze(options: AnalyzeOptions): Promise<{
+    existing: AnalyzeLocationExistingEntity[];
+  }>;
+};
 ```

plugins/catalog-node/src/extensions.ts

@@ -18,6 +18,7 @@ import {
   EntityProvider,
   CatalogProcessor,
   PlaceholderResolver,
+  ScmLocationAnalyzer,
 } from '@backstage/plugin-catalog-node';
 
 /**
@@ -31,6 +32,9 @@ export interface CatalogProcessingExtensionPoint {
     ...providers: Array<EntityProvider | Array<EntityProvider>>
   ): void;
   addPlaceholderResolver(key: string, resolver: PlaceholderResolver): void;
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): void;
 }
 
 /**

plugins/catalog-node/src/processing/index.ts

@@ -15,9 +15,11 @@
  */
 
 export type {
+  AnalyzeOptions,
   DeferredEntity,
   PlaceholderResolver,
   PlaceholderResolverParams,
   PlaceholderResolverRead,
   PlaceholderResolverResolveUrl,
+  ScmLocationAnalyzer,
 } from './types';

plugins/catalog-node/src/processing/types.ts

@@ -15,6 +15,7 @@
  */
 
 import { Entity } from '@backstage/catalog-model';
+import { AnalyzeLocationExistingEntity } from '@backstage/plugin-catalog-common';
 import { JsonValue } from '@backstage/types';
 import { CatalogProcessorEmit } from '../api';
 
@@ -50,3 +51,20 @@ export type PlaceholderResolverParams = {
 export type PlaceholderResolver = (
   params: PlaceholderResolverParams,
 ) => Promise<JsonValue>;
+
+/** @public */
+export type AnalyzeOptions = {
+  url: string;
+  catalogFilename?: string;
+};
+
+/** @public */
+export type ScmLocationAnalyzer = {
+  /** The method that decides if this analyzer can work with the provided url */
+  supports(url: string): boolean;
+  /** This function can return an array of already existing entities */
+  analyze(options: AnalyzeOptions): Promise<{
+    /** Existing entities in the analyzed location */
+    existing: AnalyzeLocationExistingEntity[];
+  }>;
+};