diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f5ab216a7f..1e2d732f35 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -38,6 +38,7 @@ "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/core-plugin-api": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", @@ -47,6 +48,7 @@ "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", + "@octokit/rest": "^19.0.4", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/ingestion/GitHubLocationAnalyzer.ts b/plugins/catalog-backend/src/ingestion/GitHubLocationAnalyzer.ts new file mode 100644 index 0000000000..b96b50649e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/GitHubLocationAnalyzer.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogClient } from '@backstage/catalog-client'; +import { GitHubIntegration } from '@backstage/integration'; +import { DiscoveryApi } from '@backstage/plugin-permission-common'; +import { Octokit } from '@octokit/rest'; +import { trimEnd } from 'lodash'; +import { AnalyzeLocationExistingEntity, BaseLocationAnalyzer } from './types'; + +export type GitHubLocationAnalyzerOptions = { + integration: GitHubIntegration; + catalogFilename?: string; + + discovery: DiscoveryApi; +}; +export class GitHubLocationAnalyzer implements BaseLocationAnalyzer { + private readonly integration: GitHubIntegration; + private readonly catalogFilename: string; + private readonly discovery: DiscoveryApi; + + constructor(options: GitHubLocationAnalyzerOptions) { + this.integration = options.integration; + this.catalogFilename = options.catalogFilename || 'catalog-info.yaml'; + this.discovery = options.discovery; + } + + async analyze( + owner: string, + repo: string, + url: string, + ): Promise { + const octo = new Octokit({ + auth: this.integration.config.token, + baseUrl: this.integration.config.apiBaseUrl, + }); + const query = `filename:${this.catalogFilename} repo:${owner}/${repo} `; + + const catalogClient = new CatalogClient({ discoveryApi: this.discovery }); + + const searchResult = await octo.search.code({ q: query }).catch(e => { + throw new Error(`Couldn't search repository for metadata file, ${e}`); + }); + + const exists = searchResult.data.total_count > 0; + if (exists) { + const repoInformation = await octo.repos.get({ owner, repo }).catch(e => { + throw new Error(`Couldn't fetch repo data, ${e}`); + }); + const defaultBranch = repoInformation.data.default_branch; + + return await Promise.all( + searchResult.data.items + .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`) + .map(async target => { + const result = await catalogClient.addLocation({ + type: 'url', + target, + dryRun: true, + }); + return { + target, + exists: result.exists, + entities: result.entities.map(e => ({ + kind: e.kind, + namespace: e.metadata.namespace ?? 'default', + name: e.metadata.name, + })), + }; + }), + ); + } + return []; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts index b84dff7de8..c7354a40bc 100644 --- a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts +++ b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts @@ -17,25 +17,40 @@ import { Logger } from 'winston'; import parseGitUrl from 'git-url-parse'; import { Entity } from '@backstage/catalog-model'; -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GitHubIntegration, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { AnalyzeLocationRequest, AnalyzeLocationResponse, LocationAnalyzer, } from './types'; +import { DiscoveryApi } from '@backstage/plugin-permission-common'; +import { GitHubLocationAnalyzer } from './GitHubLocationAnalyzer'; export class RepoLocationAnalyzer implements LocationAnalyzer { private readonly logger: Logger; private readonly scmIntegrations: ScmIntegrationRegistry; + private readonly discovery: DiscoveryApi; - constructor(logger: Logger, scmIntegrations: ScmIntegrationRegistry) { + constructor( + logger: Logger, + scmIntegrations: ScmIntegrationRegistry, + discovery: DiscoveryApi, + ) { this.logger = logger; this.scmIntegrations = scmIntegrations; + this.discovery = discovery; } async analyzeLocation( request: AnalyzeLocationRequest, ): Promise { + const integration = this.scmIntegrations.byUrl( + request.location.target, + ) as GitHubIntegration; const { owner, name } = parseGitUrl(request.location.target); + const entity: Entity = { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', @@ -45,8 +60,8 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { spec: { type: 'other', lifecycle: 'unknown' }, }; - const integration = this.scmIntegrations.byUrl(request.location.target); let annotationPrefix; + let analyzer; switch (integration?.type) { case 'azure': annotationPrefix = 'dev.azure.com'; @@ -56,6 +71,10 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { break; case 'github': annotationPrefix = 'github.com'; + analyzer = new GitHubLocationAnalyzer({ + integration, + discovery: this.discovery, + }); break; case 'gitlab': annotationPrefix = 'gitlab.com'; @@ -63,6 +82,22 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { default: break; } + if (analyzer) { + const existingEntityFiles = await analyzer.analyze( + owner, + name, + request.location.target, + ); + if (existingEntityFiles.length > 0) { + this.logger.debug( + `entity for ${request.location.target} already exists.`, + ); + return { + existingEntityFiles, + generateEntities: [], + }; + } + } if (annotationPrefix) { entity.metadata.annotations = { diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 2602bd080f..149795d166 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; import { RecursivePartial } from '../util/RecursivePartial'; import { LocationSpec } from '@backstage/plugin-catalog-node'; @@ -50,9 +50,9 @@ export type AnalyzeLocationResponse = { * @public */ export type AnalyzeLocationExistingEntity = { - location: LocationSpec; - isRegistered: boolean; - entity: Entity; + target: string; + exists: boolean | undefined; + entities: CompoundEntityRef[]; }; /** @@ -98,3 +98,11 @@ export type AnalyzeLocationEntityField = { */ description: string; }; + +export interface BaseLocationAnalyzer { + analyze( + owner: string, + repo: string, + url: string, + ): Promise; +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index d47ace1454..2e20b0d4c3 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -96,6 +96,7 @@ import { RESOURCE_TYPE_CATALOG_ENTITY, } from '@backstage/plugin-catalog-common'; import { AuthorizedLocationService } from './AuthorizedLocationService'; +import { DiscoveryApi } from '@backstage/core-plugin-api'; /** @public */ export type CatalogEnvironment = { @@ -104,6 +105,7 @@ export type CatalogEnvironment = { config: Config; reader: UrlReader; permissions: PermissionEvaluator | PermissionAuthorizer; + discovery: DiscoveryApi; }; /** @@ -478,7 +480,8 @@ export class CatalogBuilder { ); const locationAnalyzer = - this.locationAnalyzer ?? new RepoLocationAnalyzer(logger, integrations); + this.locationAnalyzer ?? + new RepoLocationAnalyzer(logger, integrations, this.env.discovery); const locationService = new AuthorizedLocationService( new DefaultLocationService(locationStore, orchestrator, { allowedLocationTypes: this.allowedLocationType, diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index e48515fb8b..51195ccdd7 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -70,6 +70,7 @@ export async function startStandaloneServer( config, reader, permissions, + discovery, }); const catalog = await builder.build(); diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 2a4ea4642c..055955c9c7 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -40,6 +40,7 @@ "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/integration-react": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-react": "workspace:^", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index c12e391ec9..0533b558c8 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -15,7 +15,6 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi, DiscoveryApi, @@ -28,11 +27,10 @@ import { import { ScmAuthApi } from '@backstage/integration-react'; import { Octokit } from '@octokit/rest'; import { Base64 } from 'js-base64'; -import { PartialEntity } from '../types'; import { AnalyzeResult, CatalogImportApi } from './CatalogImportApi'; import { getGithubIntegrationConfig } from './GitHub'; -import { trimEnd } from 'lodash'; import { getBranchName, getCatalogFilename } from '../components/helpers'; +import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-backend'; /** * The default implementation of the {@link CatalogImportApi}. @@ -105,16 +103,14 @@ export class CatalogImportClient implements CatalogImportApi { ); } - // TODO: this could be part of the analyze-location endpoint - const locations = await this.checkGitHubForExistingCatalogInfo({ - ...ghConfig, - url, + const analyzation = await this.analyzeLocation({ + repo: url, }); - if (locations.length > 0) { + if (analyzation.existingEntityFiles.length > 0) { return { type: 'locations', - locations, + locations: analyzation.existingEntityFiles, }; } @@ -122,9 +118,7 @@ export class CatalogImportClient implements CatalogImportApi { type: 'repository', integrationType: 'github', url: url, - generatedEntities: await this.generateEntityDefinitions({ - repo: url, - }), + generatedEntities: analyzation.generateEntities.map((x: any) => x.entity), }; } @@ -174,9 +168,9 @@ the component will become available.\n\nFor more information, read an \ } // TODO: this could be part of the catalog api - private async generateEntityDefinitions(options: { + private async analyzeLocation(options: { repo: string; - }): Promise { + }): Promise { const { token } = await this.identityApi.getCredentials(); const response = await fetch( `${await this.discoveryApi.getBaseUrl('catalog')}/analyze-location`, @@ -200,69 +194,7 @@ the component will become available.\n\nFor more information, read an \ } const payload = await response.json(); - return payload.generateEntities.map((x: any) => x.entity); - } - - // TODO: this response should better be part of the analyze-locations response and scm-independent / implemented per scm - private async checkGitHubForExistingCatalogInfo(options: { - url: string; - owner: string; - repo: string; - githubIntegrationConfig: GitHubIntegrationConfig; - }): Promise< - Array<{ - target: string; - entities: CompoundEntityRef[]; - }> - > { - const { url, owner, repo, githubIntegrationConfig } = options; - - const { token } = await this.scmAuthApi.getCredentials({ url }); - const octo = new Octokit({ - auth: token, - baseUrl: githubIntegrationConfig.apiBaseUrl, - }); - const catalogFilename = getCatalogFilename(this.configApi); - const query = `repo:${owner}/${repo}+filename:${catalogFilename}`; - - const searchResult = await octo.search.code({ q: query }).catch(e => { - throw new Error( - formatHttpErrorMessage( - "Couldn't search repository for metadata file.", - e, - ), - ); - }); - const exists = searchResult.data.total_count > 0; - if (exists) { - const repoInformation = await octo.repos.get({ owner, repo }).catch(e => { - throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e)); - }); - const defaultBranch = repoInformation.data.default_branch; - - return await Promise.all( - searchResult.data.items - .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`) - .map(async target => { - const result = await this.catalogApi.addLocation({ - type: 'url', - target, - dryRun: true, - }); - return { - target, - exists: result.exists, - entities: result.entities.map(e => ({ - kind: e.kind, - namespace: e.metadata.namespace ?? 'default', - name: e.metadata.name, - })), - }; - }), - ); - } - - return []; + return payload; } // TODO: extract this function and implement for non-github diff --git a/yarn.lock b/yarn.lock index 1178b3d940..fbb8955c8f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4659,6 +4659,7 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/core-plugin-api": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" @@ -4669,6 +4670,7 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" + "@octokit/rest": ^19.0.4 "@types/core-js": ^2.5.4 "@types/express": ^4.17.6 "@types/git-url-parse": ^9.0.0 @@ -4796,6 +4798,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/integration-react": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@material-ui/core": ^4.12.2 @@ -11576,7 +11579,7 @@ __metadata: languageName: node linkType: hard -"@octokit/rest@npm:^19.0.3": +"@octokit/rest@npm:^19.0.3, @octokit/rest@npm:^19.0.4": version: 19.0.4 resolution: "@octokit/rest@npm:19.0.4" dependencies: