diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index 23d9cbecfc..950c7723e8 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -36,6 +36,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", @@ -44,6 +45,8 @@ "@backstage/plugin-catalog-node": "workspace:^", "@backstage/types": "workspace:^", "@octokit/graphql": "^5.0.0", + "@octokit/rest": "^19.0.4", + "git-url-parse": "^13.1.0", "lodash": "^4.17.21", "msw": "^0.47.0", "node-fetch": "^2.6.7", diff --git a/plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.test.ts b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.test.ts rename to plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts index fbc9ae3454..5ceb9cf85c 100644 --- a/plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.test.ts +++ b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts @@ -46,12 +46,17 @@ describe('GitHubLocationAnalyzer', () => { getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'), getExternalBaseUrl: jest.fn(), }; - const integration = new GitHubIntegration({ - host: 'h.com', - apiBaseUrl: 'a', - rawBaseUrl: 'r', - token: 't', - }); + const integrations = { + list: jest.fn(), + byHost: jest.fn(), + byUrl: () => + new GitHubIntegration({ + host: 'h.com', + apiBaseUrl: 'a', + rawBaseUrl: 'r', + token: 't', + }), + }; setupRequestMockHandlers(server); @@ -112,9 +117,11 @@ describe('GitHubLocationAnalyzer', () => { const analyzer = new GitHubLocationAnalyzer({ discovery: mockDiscoveryApi, - integration, + integrations, + }); + const result = await analyzer.analyze({ + url: 'https://github.com/foo/bar', }); - const result = await analyzer.analyze('https://github.com/foo/bar'); expect(result[0].isRegistered).toBeFalsy(); expect(result[0].location).toEqual({ @@ -135,10 +142,12 @@ describe('GitHubLocationAnalyzer', () => { const analyzer = new GitHubLocationAnalyzer({ discovery: mockDiscoveryApi, - integration, + integrations, + }); + const result = await analyzer.analyze({ + url: 'https://github.com/foo/bar', catalogFilename: 'anvil.yaml', }); - const result = await analyzer.analyze('https://github.com/foo/bar'); expect(result[0].location).toEqual({ type: 'url', diff --git a/plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.ts b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts similarity index 62% rename from plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.ts rename to plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts index 2e42f202f9..218d56410d 100644 --- a/plugins/catalog-backend/src/ingestion/analyzers/GitHubLocationAnalyzer.ts +++ b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts @@ -15,39 +15,57 @@ */ import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; -import { GitHubIntegration } from '@backstage/integration'; -import { DiscoveryApi } from '@backstage/plugin-permission-common'; +import { + GitHubIntegration, + ScmIntegrationsGroup, +} from '@backstage/integration'; import { Octokit } from '@octokit/rest'; import { trimEnd } from 'lodash'; import parseGitUrl from 'git-url-parse'; -import { AnalyzeLocationExistingEntity, ScmLocationAnalyzer } from '../types'; +import { + AnalyzeLocationExistingEntity, + AnalyzeOptions, + ScmLocationAnalyzer, +} from '@backstage/plugin-catalog-backend'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; export type GitHubLocationAnalyzerOptions = { - integration: GitHubIntegration; + integrations: ScmIntegrationsGroup; catalogFilename?: string; - discovery: DiscoveryApi; + discovery: PluginEndpointDiscovery; }; export class GitHubLocationAnalyzer implements ScmLocationAnalyzer { - private readonly catalogFilename: string; - private readonly discovery: DiscoveryApi; - private readonly octokitClient: Octokit; private readonly catalogClient: CatalogApi; + private readonly integrations: ScmIntegrationsGroup; constructor(options: GitHubLocationAnalyzerOptions) { - this.catalogFilename = options.catalogFilename || 'catalog-info.yaml'; - this.discovery = options.discovery; - this.octokitClient = new Octokit({ - auth: options.integration.config.token, - baseUrl: options.integration.config.apiBaseUrl, - }); - this.catalogClient = new CatalogClient({ discoveryApi: this.discovery }); + this.integrations = options.integrations; + this.catalogClient = new CatalogClient({ discoveryApi: options.discovery }); } - - async analyze(url: string): Promise { + getIntegrationType() { + return 'github'; + } + async analyze({ + url, + catalogFilename, + }: AnalyzeOptions): Promise { const { owner, name: repo } = parseGitUrl(url); - const query = `filename:${this.catalogFilename} repo:${owner}/${repo}`; - const searchResult = await this.octokitClient.search + const catalogFile = catalogFilename || 'catalog-info.yaml'; + + const query = `filename:${catalogFile} repo:${owner}/${repo}`; + + const integration = this.integrations.byUrl(url); + if (!integration) { + throw new Error('Make sure you have a GitHub integration configured'); + } + + const octokitClient = new Octokit({ + auth: integration.config.token, + baseUrl: integration.config.apiBaseUrl, + }); + + const searchResult = await octokitClient.search .code({ q: query }) .catch(e => { throw new Error(`Couldn't search repository for metadata file, ${e}`); @@ -55,7 +73,7 @@ export class GitHubLocationAnalyzer implements ScmLocationAnalyzer { const exists = searchResult.data.total_count > 0; if (exists) { - const repoInformation = await this.octokitClient.repos + const repoInformation = await octokitClient.repos .get({ owner, repo }) .catch(e => { throw new Error(`Couldn't fetch repo data, ${e}`); diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index d793d48f1e..e856277873 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -29,3 +29,4 @@ export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntity export type { GithubMultiOrgConfig } from './lib'; export { githubEntityProviderCatalogModule } from './module'; export type { GithubEntityProviderCatalogModuleOptions } from './module'; +export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer'; diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 6a944af330..ec904a52e0 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -308,6 +308,39 @@ export async function getOrganizationRepositories( return { repositories }; } +export async function getRepository( + client: typeof graphql, + org: string, + name: string, +): Promise { + const query = ` + query repositories($org: String!, $name: String!) { + repository(name: $name, owner: $org) { + name + url + isArchived + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } + defaultBranchRef { + name + } + } + }`; + + const repository: Repository = await client(query, { + org, + name, + }); + return repository; +} + /** * Gets all the users out of a GitHub organization. * diff --git a/plugins/catalog-backend-module-github/src/lib/index.ts b/plugins/catalog-backend-module-github/src/lib/index.ts index 26ec3e8516..28cbe898d6 100644 --- a/plugins/catalog-backend-module-github/src/lib/index.ts +++ b/plugins/catalog-backend-module-github/src/lib/index.ts @@ -20,6 +20,7 @@ export { getOrganizationRepositories, getOrganizationTeams, getOrganizationUsers, + getRepository, } from './github'; export { assignGroupsToUsers, buildOrgHierarchy } from './org'; export { parseGitHubOrgUrl } from './util'; diff --git a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts index 24c5821d92..7992903b65 100644 --- a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts +++ b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts @@ -25,23 +25,22 @@ import { AnalyzeLocationRequest, AnalyzeLocationResponse, LocationAnalyzer, + ScmLocationAnalyzer, } from './types'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer'; export class RepoLocationAnalyzer implements LocationAnalyzer { private readonly logger: Logger; private readonly scmIntegrations: ScmIntegrationRegistry; - private readonly discovery: PluginEndpointDiscovery; + private readonly analyzers: ScmLocationAnalyzer[]; constructor( logger: Logger, scmIntegrations: ScmIntegrationRegistry, - discovery: PluginEndpointDiscovery, + analyzers: ScmLocationAnalyzer[], ) { this.logger = logger; this.scmIntegrations = scmIntegrations; - this.discovery = discovery; + this.analyzers = analyzers; } async analyzeLocation( request: AnalyzeLocationRequest, @@ -52,7 +51,6 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { const { owner, name } = parseGitUrl(request.location.target); let annotationPrefix; - let analyzer; switch (integration?.type) { case 'azure': annotationPrefix = 'dev.azure.com'; @@ -62,11 +60,6 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { break; case 'github': annotationPrefix = 'github.com'; - analyzer = new GitHubLocationAnalyzer({ - integration, - discovery: this.discovery, - catalogFilename: request.catalogFilename, - }); break; case 'gitlab': annotationPrefix = 'gitlab.com'; @@ -75,10 +68,13 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { break; } + const analyzer = this.analyzers.find( + a => a.getIntegrationType() === integration.type, + ); if (analyzer) { - const existingEntityFiles = await analyzer.analyze( - request.location.target, - ); + const existingEntityFiles = await analyzer.analyze({ + url: request.location.target, + }); if (existingEntityFiles.length > 0) { this.logger.debug( `entity for ${request.location.target} already exists.`, diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index c97d029b5c..0b00809043 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -21,4 +21,6 @@ export type { AnalyzeLocationRequest, AnalyzeLocationResponse, LocationAnalyzer, + ScmLocationAnalyzer, + AnalyzeOptions, } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 8dc833abd8..961be78572 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -100,10 +100,14 @@ export type AnalyzeLocationEntityField = { description: string; }; -export interface ScmLocationAnalyzer { - analyze( - owner: string, - repo: string, - url: string, - ): Promise; -} +export type AnalyzeOptions = { + url: string; + catalogFilename?: string; +}; +/** @public */ +export type ScmLocationAnalyzer = { + /** The integration type this location analyzer can work with */ + getIntegrationType(): string; + /** This function is responsible to figure out if the catalog file is already present in the repository */ + analyze(options: AnalyzeOptions): Promise; +}; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f1d29f015c..97379ac491 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -60,7 +60,7 @@ import { yamlPlaceholderResolver, } from '../modules/core/PlaceholderProcessor'; import { defaultEntityDataParser } from '../modules/util/parse'; -import { LocationAnalyzer } from '../ingestion/types'; +import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types'; import { CatalogProcessingEngine } from '../processing'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; @@ -120,6 +120,9 @@ export type CatalogEnvironment = { * after the processors' pre-processing steps. All policies are given the * chance to inspect the entity, and all of them have to pass in order for * the entity to be considered valid from an overall point of view. + * - Location analyzers can be added. These are responsible to analyze the + * the existence of a catalog-info.yaml file int he provided git repository + * when you use the /catalog-import page with a repository url. * - Placeholder resolvers can be replaced or added. These run on the raw * structured data between the parsing and pre-processing steps, to replace * dollar-prefixed entries with their actual values (like $file). @@ -140,6 +143,7 @@ export class CatalogBuilder { private fieldFormatValidators: Partial; private entityProviders: EntityProvider[]; private processors: CatalogProcessor[]; + private locationAnalyzers: ScmLocationAnalyzer[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; private onProcessingError?: (event: { @@ -170,6 +174,7 @@ export class CatalogBuilder { this.fieldFormatValidators = {}; this.entityProviders = []; this.processors = []; + this.locationAnalyzers = []; this.processorsReplace = false; this.parser = undefined; this.permissionRules = Object.values(catalogPermissionRules); @@ -340,6 +345,20 @@ export class CatalogBuilder { ]; } + /** + * Adds Location Analyzers. These are responsible for figuring out + * if the repository already contains a catalog-info.yaml file when + * you register a repostiroy in the /catalog-import page + * + * @param locationAnalyzers - One or more location analyzers + */ + addLocationAnalyzers( + ...analyzers: Array> + ): CatalogBuilder { + this.locationAnalyzers.push(...analyzers.flat()); + return this; + } + /** * Sets up the catalog to use a custom parser for entity data. * @@ -484,7 +503,7 @@ export class CatalogBuilder { const locationAnalyzer = this.locationAnalyzer ?? - new RepoLocationAnalyzer(logger, integrations, this.env.discovery); + new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers); const locationService = new AuthorizedLocationService( new DefaultLocationService(locationStore, orchestrator, { allowedLocationTypes: this.allowedLocationType, diff --git a/yarn.lock b/yarn.lock index c3155dbfee..718736394b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4550,6 +4550,7 @@ __metadata: "@backstage/plugin-catalog-node": "workspace:^" "@backstage/types": "workspace:^" "@octokit/graphql": ^5.0.0 + "@octokit/rest": ^19.0.4 "@types/lodash": ^4.14.151 lodash: ^4.17.21 msw: ^0.47.0