move github deps to their own plugin

Signed-off-by: Kiss Miklos <miklos@roadie.io>

plugins/catalog-backend-module-github/package.json

@@ -36,6 +36,7 @@
     "@backstage/backend-common": "workspace:^",
     "@backstage/backend-plugin-api": "workspace:^",
     "@backstage/backend-tasks": "workspace:^",
+    "@backstage/catalog-client": "workspace:^",
     "@backstage/catalog-model": "workspace:^",
     "@backstage/config": "workspace:^",
     "@backstage/errors": "workspace:^",
@@ -44,6 +45,8 @@
     "@backstage/plugin-catalog-node": "workspace:^",
     "@backstage/types": "workspace:^",
     "@octokit/graphql": "^5.0.0",
+    "@octokit/rest": "^19.0.4",
+    "git-url-parse": "^13.1.0",
     "lodash": "^4.17.21",
     "msw": "^0.47.0",
     "node-fetch": "^2.6.7",

plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts

@@ -46,12 +46,17 @@ describe('GitHubLocationAnalyzer', () => {
     getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'),
     getExternalBaseUrl: jest.fn(),
   };
-  const integration = new GitHubIntegration({
-    host: 'h.com',
-    apiBaseUrl: 'a',
-    rawBaseUrl: 'r',
-    token: 't',
-  });
+  const integrations = {
+    list: jest.fn(),
+    byHost: jest.fn(),
+    byUrl: () =>
+      new GitHubIntegration({
+        host: 'h.com',
+        apiBaseUrl: 'a',
+        rawBaseUrl: 'r',
+        token: 't',
+      }),
+  };
 
   setupRequestMockHandlers(server);
 
@@ -112,9 +117,11 @@ describe('GitHubLocationAnalyzer', () => {
 
     const analyzer = new GitHubLocationAnalyzer({
       discovery: mockDiscoveryApi,
-      integration,
+      integrations,
+    });
+    const result = await analyzer.analyze({
+      url: 'https://github.com/foo/bar',
     });
-    const result = await analyzer.analyze('https://github.com/foo/bar');
 
     expect(result[0].isRegistered).toBeFalsy();
     expect(result[0].location).toEqual({
@@ -135,10 +142,12 @@ describe('GitHubLocationAnalyzer', () => {
 
     const analyzer = new GitHubLocationAnalyzer({
       discovery: mockDiscoveryApi,
-      integration,
+      integrations,
+    });
+    const result = await analyzer.analyze({
+      url: 'https://github.com/foo/bar',
       catalogFilename: 'anvil.yaml',
     });
-    const result = await analyzer.analyze('https://github.com/foo/bar');
 
     expect(result[0].location).toEqual({
       type: 'url',

plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts

@@ -15,39 +15,57 @@
  */
 
 import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
-import { GitHubIntegration } from '@backstage/integration';
-import { DiscoveryApi } from '@backstage/plugin-permission-common';
+import {
+  GitHubIntegration,
+  ScmIntegrationsGroup,
+} from '@backstage/integration';
 import { Octokit } from '@octokit/rest';
 import { trimEnd } from 'lodash';
 import parseGitUrl from 'git-url-parse';
-import { AnalyzeLocationExistingEntity, ScmLocationAnalyzer } from '../types';
+import {
+  AnalyzeLocationExistingEntity,
+  AnalyzeOptions,
+  ScmLocationAnalyzer,
+} from '@backstage/plugin-catalog-backend';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
 
 export type GitHubLocationAnalyzerOptions = {
-  integration: GitHubIntegration;
+  integrations: ScmIntegrationsGroup<GitHubIntegration>;
   catalogFilename?: string;
-  discovery: DiscoveryApi;
+  discovery: PluginEndpointDiscovery;
 };
 export class GitHubLocationAnalyzer implements ScmLocationAnalyzer {
-  private readonly catalogFilename: string;
-  private readonly discovery: DiscoveryApi;
-  private readonly octokitClient: Octokit;
   private readonly catalogClient: CatalogApi;
+  private readonly integrations: ScmIntegrationsGroup<GitHubIntegration>;
 
   constructor(options: GitHubLocationAnalyzerOptions) {
-    this.catalogFilename = options.catalogFilename || 'catalog-info.yaml';
-    this.discovery = options.discovery;
-    this.octokitClient = new Octokit({
-      auth: options.integration.config.token,
-      baseUrl: options.integration.config.apiBaseUrl,
-    });
-    this.catalogClient = new CatalogClient({ discoveryApi: this.discovery });
+    this.integrations = options.integrations;
+    this.catalogClient = new CatalogClient({ discoveryApi: options.discovery });
   }
-
-  async analyze(url: string): Promise<AnalyzeLocationExistingEntity[]> {
+  getIntegrationType() {
+    return 'github';
+  }
+  async analyze({
+    url,
+    catalogFilename,
+  }: AnalyzeOptions): Promise<AnalyzeLocationExistingEntity[]> {
     const { owner, name: repo } = parseGitUrl(url);
-    const query = `filename:${this.catalogFilename} repo:${owner}/${repo}`;
 
-    const searchResult = await this.octokitClient.search
+    const catalogFile = catalogFilename || 'catalog-info.yaml';
+
+    const query = `filename:${catalogFile} repo:${owner}/${repo}`;
+
+    const integration = this.integrations.byUrl(url);
+    if (!integration) {
+      throw new Error('Make sure you have a GitHub integration configured');
+    }
+
+    const octokitClient = new Octokit({
+      auth: integration.config.token,
+      baseUrl: integration.config.apiBaseUrl,
+    });
+
+    const searchResult = await octokitClient.search
       .code({ q: query })
       .catch(e => {
         throw new Error(`Couldn't search repository for metadata file, ${e}`);
@@ -55,7 +73,7 @@ export class GitHubLocationAnalyzer implements ScmLocationAnalyzer {
 
     const exists = searchResult.data.total_count > 0;
     if (exists) {
-      const repoInformation = await this.octokitClient.repos
+      const repoInformation = await octokitClient.repos
         .get({ owner, repo })
         .catch(e => {
           throw new Error(`Couldn't fetch repo data, ${e}`);

plugins/catalog-backend-module-github/src/index.ts

@@ -29,3 +29,4 @@ export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntity
 export type { GithubMultiOrgConfig } from './lib';
 export { githubEntityProviderCatalogModule } from './module';
 export type { GithubEntityProviderCatalogModuleOptions } from './module';
+export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer';

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -308,6 +308,39 @@ export async function getOrganizationRepositories(
   return { repositories };
 }
 
+export async function getRepository(
+  client: typeof graphql,
+  org: string,
+  name: string,
+): Promise<Repository> {
+  const query = `
+  query repositories($org: String!, $name: String!) {
+    repository(name: $name, owner: $org) {
+      name
+      url
+      isArchived
+      repositoryTopics(first: 100) {
+        nodes {
+          ... on RepositoryTopic {
+            topic {
+              name
+            }
+          }
+        }
+      }
+      defaultBranchRef {
+        name
+      }
+    }
+  }`;
+
+  const repository: Repository = await client(query, {
+    org,
+    name,
+  });
+  return repository;
+}
+
 /**
  * Gets all the users out of a GitHub organization.
  *

plugins/catalog-backend-module-github/src/lib/index.ts

@@ -20,6 +20,7 @@ export {
   getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
+  getRepository,
 } from './github';
 export { assignGroupsToUsers, buildOrgHierarchy } from './org';
 export { parseGitHubOrgUrl } from './util';

plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts

@@ -25,23 +25,22 @@ import {
   AnalyzeLocationRequest,
   AnalyzeLocationResponse,
   LocationAnalyzer,
+  ScmLocationAnalyzer,
 } from './types';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
-import { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer';
 
 export class RepoLocationAnalyzer implements LocationAnalyzer {
   private readonly logger: Logger;
   private readonly scmIntegrations: ScmIntegrationRegistry;
-  private readonly discovery: PluginEndpointDiscovery;
+  private readonly analyzers: ScmLocationAnalyzer[];
 
   constructor(
     logger: Logger,
     scmIntegrations: ScmIntegrationRegistry,
-    discovery: PluginEndpointDiscovery,
+    analyzers: ScmLocationAnalyzer[],
   ) {
     this.logger = logger;
     this.scmIntegrations = scmIntegrations;
-    this.discovery = discovery;
+    this.analyzers = analyzers;
   }
   async analyzeLocation(
     request: AnalyzeLocationRequest,
@@ -52,7 +51,6 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
     const { owner, name } = parseGitUrl(request.location.target);
 
     let annotationPrefix;
-    let analyzer;
     switch (integration?.type) {
       case 'azure':
         annotationPrefix = 'dev.azure.com';
@@ -62,11 +60,6 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
         break;
       case 'github':
         annotationPrefix = 'github.com';
-        analyzer = new GitHubLocationAnalyzer({
-          integration,
-          discovery: this.discovery,
-          catalogFilename: request.catalogFilename,
-        });
         break;
       case 'gitlab':
         annotationPrefix = 'gitlab.com';
@@ -75,10 +68,13 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
         break;
     }
 
+    const analyzer = this.analyzers.find(
+      a => a.getIntegrationType() === integration.type,
+    );
     if (analyzer) {
-      const existingEntityFiles = await analyzer.analyze(
-        request.location.target,
-      );
+      const existingEntityFiles = await analyzer.analyze({
+        url: request.location.target,
+      });
       if (existingEntityFiles.length > 0) {
         this.logger.debug(
           `entity for ${request.location.target} already exists.`,

plugins/catalog-backend/src/ingestion/index.ts

@@ -21,4 +21,6 @@ export type {
   AnalyzeLocationRequest,
   AnalyzeLocationResponse,
   LocationAnalyzer,
+  ScmLocationAnalyzer,
+  AnalyzeOptions,
 } from './types';

plugins/catalog-backend/src/ingestion/types.ts

@@ -100,10 +100,14 @@ export type AnalyzeLocationEntityField = {
   description: string;
 };
 
-export interface ScmLocationAnalyzer {
-  analyze(
-    owner: string,
-    repo: string,
-    url: string,
-  ): Promise<AnalyzeLocationExistingEntity[]>;
-}
+export type AnalyzeOptions = {
+  url: string;
+  catalogFilename?: string;
+};
+/** @public */
+export type ScmLocationAnalyzer = {
+  /** The integration type this location analyzer can work with */
+  getIntegrationType(): string;
+  /** This function is responsible to figure out if the catalog file is already present in the repository */
+  analyze(options: AnalyzeOptions): Promise<AnalyzeLocationExistingEntity[]>;
+};

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -60,7 +60,7 @@ import {
   yamlPlaceholderResolver,
 } from '../modules/core/PlaceholderProcessor';
 import { defaultEntityDataParser } from '../modules/util/parse';
-import { LocationAnalyzer } from '../ingestion/types';
+import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types';
 import { CatalogProcessingEngine } from '../processing';
 import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
 import { applyDatabaseMigrations } from '../database/migrations';
@@ -120,6 +120,9 @@ export type CatalogEnvironment = {
  *   after the processors' pre-processing steps. All policies are given the
  *   chance to inspect the entity, and all of them have to pass in order for
  *   the entity to be considered valid from an overall point of view.
+ * - Location analyzers can be added. These are responsible to analyze the
+ *   the existence of a catalog-info.yaml file int he provided git repository
+ *   when you use the /catalog-import page with a repository url.
  * - Placeholder resolvers can be replaced or added. These run on the raw
  *   structured data between the parsing and pre-processing steps, to replace
  *   dollar-prefixed entries with their actual values (like $file).
@@ -140,6 +143,7 @@ export class CatalogBuilder {
   private fieldFormatValidators: Partial<Validators>;
   private entityProviders: EntityProvider[];
   private processors: CatalogProcessor[];
+  private locationAnalyzers: ScmLocationAnalyzer[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
   private onProcessingError?: (event: {
@@ -170,6 +174,7 @@ export class CatalogBuilder {
     this.fieldFormatValidators = {};
     this.entityProviders = [];
     this.processors = [];
+    this.locationAnalyzers = [];
     this.processorsReplace = false;
     this.parser = undefined;
     this.permissionRules = Object.values(catalogPermissionRules);
@@ -340,6 +345,20 @@ export class CatalogBuilder {
     ];
   }
 
+  /**
+   * Adds Location Analyzers. These are responsible for figuring out
+   * if the repository already contains a catalog-info.yaml file when
+   * you register a repostiroy in the /catalog-import page
+   *
+   * @param locationAnalyzers - One or more location analyzers
+   */
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): CatalogBuilder {
+    this.locationAnalyzers.push(...analyzers.flat());
+    return this;
+  }
+
   /**
    * Sets up the catalog to use a custom parser for entity data.
    *
@@ -484,7 +503,7 @@ export class CatalogBuilder {
 
     const locationAnalyzer =
       this.locationAnalyzer ??
-      new RepoLocationAnalyzer(logger, integrations, this.env.discovery);
+      new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers);
     const locationService = new AuthorizedLocationService(
       new DefaultLocationService(locationStore, orchestrator, {
         allowedLocationTypes: this.allowedLocationType,

yarn.lock

@@ -4550,6 +4550,7 @@ __metadata:
     "@backstage/plugin-catalog-node": "workspace:^"
     "@backstage/types": "workspace:^"
     "@octokit/graphql": ^5.0.0
+    "@octokit/rest": ^19.0.4
     "@types/lodash": ^4.14.151
     lodash: ^4.17.21
     msw: ^0.47.0