paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

move github search to the backend

Browse Source 
Signed-off-by: Kiss Miklos <miklos@roadie.io>
This commit is contained in:
Kiss Miklos
2022-09-22 00:17:52 +02:00
parent 856146d755
commit fbf2d1256b
9 changed files with 159 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/package.json
+2
View File
@@ -38,6 +38,7 @@
"@backstage/catalog-client": "workspace:^",
"@backstage/catalog-model": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/core-plugin-api": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/plugin-catalog-common": "workspace:^",
@@ -47,6 +48,7 @@
"@backstage/plugin-scaffolder-common": "workspace:^",
"@backstage/plugin-search-common": "workspace:^",
"@backstage/types": "workspace:^",
"@octokit/rest": "^19.0.4",
"@types/express": "^4.17.6",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
plugins/catalog-backend/src/ingestion/GitHubLocationAnalyzer.ts
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CatalogClient } from '@backstage/catalog-client';
import { GitHubIntegration } from '@backstage/integration';
import { DiscoveryApi } from '@backstage/plugin-permission-common';
import { Octokit } from '@octokit/rest';
import { trimEnd } from 'lodash';
import { AnalyzeLocationExistingEntity, BaseLocationAnalyzer } from './types';
export type GitHubLocationAnalyzerOptions = {
integration: GitHubIntegration;
catalogFilename?: string;
discovery: DiscoveryApi;
};
export class GitHubLocationAnalyzer implements BaseLocationAnalyzer {
private readonly integration: GitHubIntegration;
private readonly catalogFilename: string;
private readonly discovery: DiscoveryApi;
constructor(options: GitHubLocationAnalyzerOptions) {
this.integration = options.integration;
this.catalogFilename = options.catalogFilename || 'catalog-info.yaml';
this.discovery = options.discovery;
}
async analyze(
owner: string,
repo: string,
url: string,
): Promise<AnalyzeLocationExistingEntity[]> {
const octo = new Octokit({
auth: this.integration.config.token,
baseUrl: this.integration.config.apiBaseUrl,
});
const query = `filename:${this.catalogFilename} repo:${owner}/${repo} `;
const catalogClient = new CatalogClient({ discoveryApi: this.discovery });
const searchResult = await octo.search.code({ q: query }).catch(e => {
throw new Error(`Couldn't search repository for metadata file, ${e}`);
});
const exists = searchResult.data.total_count > 0;
if (exists) {
const repoInformation = await octo.repos.get({ owner, repo }).catch(e => {
throw new Error(`Couldn't fetch repo data, ${e}`);
});
const defaultBranch = repoInformation.data.default_branch;
return await Promise.all(
searchResult.data.items
.map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`)
.map(async target => {
const result = await catalogClient.addLocation({
type: 'url',
target,
dryRun: true,
});
return {
target,
exists: result.exists,
entities: result.entities.map(e => ({
kind: e.kind,
namespace: e.metadata.namespace ?? 'default',
name: e.metadata.name,
})),
};
}),
);
}
return [];
}
}
plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts
+38 -3
View File
@@ -17,25 +17,40 @@
import { Logger } from 'winston';
import parseGitUrl from 'git-url-parse';
import { Entity } from '@backstage/catalog-model';
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
GitHubIntegration,
ScmIntegrationRegistry,
} from '@backstage/integration';
import {
AnalyzeLocationRequest,
AnalyzeLocationResponse,
LocationAnalyzer,
} from './types';
import { DiscoveryApi } from '@backstage/plugin-permission-common';
import { GitHubLocationAnalyzer } from './GitHubLocationAnalyzer';
export class RepoLocationAnalyzer implements LocationAnalyzer {
private readonly logger: Logger;
private readonly scmIntegrations: ScmIntegrationRegistry;
private readonly discovery: DiscoveryApi;
constructor(logger: Logger, scmIntegrations: ScmIntegrationRegistry) {
constructor(
logger: Logger,
scmIntegrations: ScmIntegrationRegistry,
discovery: DiscoveryApi,
) {
this.logger = logger;
this.scmIntegrations = scmIntegrations;
this.discovery = discovery;
}
async analyzeLocation(
request: AnalyzeLocationRequest,
): Promise<AnalyzeLocationResponse> {
const integration = this.scmIntegrations.byUrl(
request.location.target,
) as GitHubIntegration;
const { owner, name } = parseGitUrl(request.location.target);
const entity: Entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
@@ -45,8 +60,8 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
spec: { type: 'other', lifecycle: 'unknown' },
};
const integration = this.scmIntegrations.byUrl(request.location.target);
let annotationPrefix;
let analyzer;
switch (integration?.type) {
case 'azure':
annotationPrefix = 'dev.azure.com';
@@ -56,6 +71,10 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
break;
case 'github':
annotationPrefix = 'github.com';
analyzer = new GitHubLocationAnalyzer({
integration,
discovery: this.discovery,
});
break;
case 'gitlab':
annotationPrefix = 'gitlab.com';
@@ -63,6 +82,22 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
default:
break;
}
if (analyzer) {
const existingEntityFiles = await analyzer.analyze(
owner,
name,
request.location.target,
);
if (existingEntityFiles.length > 0) {
this.logger.debug(
`entity for ${request.location.target} already exists.`,
);
return {
existingEntityFiles,
generateEntities: [],
};
}
}
if (annotationPrefix) {
entity.metadata.annotations = {
plugins/catalog-backend/src/ingestion/types.ts
+12 -4
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { CompoundEntityRef, Entity } from '@backstage/catalog-model';
import { RecursivePartial } from '../util/RecursivePartial';
import { LocationSpec } from '@backstage/plugin-catalog-node';
@@ -50,9 +50,9 @@ export type AnalyzeLocationResponse = {
* @public
*/
export type AnalyzeLocationExistingEntity = {
location: LocationSpec;
isRegistered: boolean;
entity: Entity;
target: string;
exists: boolean | undefined;
entities: CompoundEntityRef[];
};
/**
@@ -98,3 +98,11 @@ export type AnalyzeLocationEntityField = {
*/
description: string;
};
export interface BaseLocationAnalyzer {
analyze(
owner: string,
repo: string,
url: string,
): Promise<AnalyzeLocationExistingEntity[]>;
}
plugins/catalog-backend/src/service/CatalogBuilder.ts
+4 -1
View File
@@ -96,6 +96,7 @@ import {
RESOURCE_TYPE_CATALOG_ENTITY,
} from '@backstage/plugin-catalog-common';
import { AuthorizedLocationService } from './AuthorizedLocationService';
import { DiscoveryApi } from '@backstage/core-plugin-api';
/** @public */
export type CatalogEnvironment = {
@@ -104,6 +105,7 @@ export type CatalogEnvironment = {
config: Config;
reader: UrlReader;
permissions: PermissionEvaluator | PermissionAuthorizer;
discovery: DiscoveryApi;
};
/**
@@ -478,7 +480,8 @@ export class CatalogBuilder {
);
const locationAnalyzer =
this.locationAnalyzer ?? new RepoLocationAnalyzer(logger, integrations);
this.locationAnalyzer ??
new RepoLocationAnalyzer(logger, integrations, this.env.discovery);
const locationService = new AuthorizedLocationService(
new DefaultLocationService(locationStore, orchestrator, {
allowedLocationTypes: this.allowedLocationType,
plugins/catalog-backend/src/service/standaloneServer.ts
+1
View File
@@ -70,6 +70,7 @@ export async function startStandaloneServer(
config,
reader,
permissions,
discovery,
});
const catalog = await builder.build();
plugins/catalog-import/package.json
+1
View File
@@ -40,6 +40,7 @@
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/integration-react": "workspace:^",
"@backstage/plugin-catalog-backend": "workspace:^",
"@backstage/plugin-catalog-react": "workspace:^",
"@material-ui/core": "^4.12.2",
"@material-ui/icons": "^4.9.1",
plugins/catalog-import/src/api/CatalogImportClient.ts
+9 -77
View File
@@ -15,7 +15,6 @@
*/
import { CatalogApi } from '@backstage/catalog-client';
import { CompoundEntityRef } from '@backstage/catalog-model';
import {
ConfigApi,
DiscoveryApi,
@@ -28,11 +27,10 @@ import {
import { ScmAuthApi } from '@backstage/integration-react';
import { Octokit } from '@octokit/rest';
import { Base64 } from 'js-base64';
import { PartialEntity } from '../types';
import { AnalyzeResult, CatalogImportApi } from './CatalogImportApi';
import { getGithubIntegrationConfig } from './GitHub';
import { trimEnd } from 'lodash';
import { getBranchName, getCatalogFilename } from '../components/helpers';
import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-backend';
/**
* The default implementation of the {@link CatalogImportApi}.
@@ -105,16 +103,14 @@ export class CatalogImportClient implements CatalogImportApi {
);
}
// TODO: this could be part of the analyze-location endpoint
const locations = await this.checkGitHubForExistingCatalogInfo({
...ghConfig,
url,
const analyzation = await this.analyzeLocation({
repo: url,
});
if (locations.length > 0) {
if (analyzation.existingEntityFiles.length > 0) {
return {
type: 'locations',
locations,
locations: analyzation.existingEntityFiles,
};
}
@@ -122,9 +118,7 @@ export class CatalogImportClient implements CatalogImportApi {
type: 'repository',
integrationType: 'github',
url: url,
generatedEntities: await this.generateEntityDefinitions({
repo: url,
}),
generatedEntities: analyzation.generateEntities.map((x: any) => x.entity),
};
}
@@ -174,9 +168,9 @@ the component will become available.\n\nFor more information, read an \
}
// TODO: this could be part of the catalog api
private async generateEntityDefinitions(options: {
private async analyzeLocation(options: {
repo: string;
}): Promise<PartialEntity[]> {
}): Promise<AnalyzeLocationResponse> {
const { token } = await this.identityApi.getCredentials();
const response = await fetch(
`${await this.discoveryApi.getBaseUrl('catalog')}/analyze-location`,
@@ -200,69 +194,7 @@ the component will become available.\n\nFor more information, read an \
}
const payload = await response.json();
return payload.generateEntities.map((x: any) => x.entity);
}
// TODO: this response should better be part of the analyze-locations response and scm-independent / implemented per scm
private async checkGitHubForExistingCatalogInfo(options: {
url: string;
owner: string;
repo: string;
githubIntegrationConfig: GitHubIntegrationConfig;
}): Promise<
Array<{
target: string;
entities: CompoundEntityRef[];
}>
> {
const { url, owner, repo, githubIntegrationConfig } = options;
const { token } = await this.scmAuthApi.getCredentials({ url });
const octo = new Octokit({
auth: token,
baseUrl: githubIntegrationConfig.apiBaseUrl,
});
const catalogFilename = getCatalogFilename(this.configApi);
const query = `repo:${owner}/${repo}+filename:${catalogFilename}`;
const searchResult = await octo.search.code({ q: query }).catch(e => {
throw new Error(
formatHttpErrorMessage(
"Couldn't search repository for metadata file.",
e,
),
);
});
const exists = searchResult.data.total_count > 0;
if (exists) {
const repoInformation = await octo.repos.get({ owner, repo }).catch(e => {
throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e));
});
const defaultBranch = repoInformation.data.default_branch;
return await Promise.all(
searchResult.data.items
.map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`)
.map(async target => {
const result = await this.catalogApi.addLocation({
type: 'url',
target,
dryRun: true,
});
return {
target,
exists: result.exists,
entities: result.entities.map(e => ({
kind: e.kind,
namespace: e.metadata.namespace ?? 'default',
name: e.metadata.name,
})),
};
}),
);
}
return [];
return payload;
}
// TODO: extract this function and implement for non-github
yarn.lock
+4 -1
View File
@@ -4659,6 +4659,7 @@ __metadata:
"@backstage/catalog-model": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
"@backstage/core-plugin-api": "workspace:^"
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/plugin-catalog-common": "workspace:^"
@@ -4669,6 +4670,7 @@ __metadata:
"@backstage/plugin-search-backend-node": "workspace:^"
"@backstage/plugin-search-common": "workspace:^"
"@backstage/types": "workspace:^"
"@octokit/rest": ^19.0.4
"@types/core-js": ^2.5.4
"@types/express": ^4.17.6
"@types/git-url-parse": ^9.0.0
@@ -4796,6 +4798,7 @@ __metadata:
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/integration-react": "workspace:^"
"@backstage/plugin-catalog-backend": "workspace:^"
"@backstage/plugin-catalog-react": "workspace:^"
"@backstage/test-utils": "workspace:^"
"@material-ui/core": ^4.12.2
@@ -11576,7 +11579,7 @@ __metadata:
languageName: node
linkType: hard
"@octokit/rest@npm:^19.0.3":
"@octokit/rest@npm:^19.0.3, @octokit/rest@npm:^19.0.4":
version: 19.0.4
resolution: "@octokit/rest@npm:19.0.4"
dependencies: