diff --git a/.changeset/brave-goats-rush.md b/.changeset/brave-goats-rush.md new file mode 100644 index 0000000000..8a14f8ea00 --- /dev/null +++ b/.changeset/brave-goats-rush.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-node': minor +--- + +Deprecated the `LocationSpec` type. It got moved from this package to the `@backstage/plugin-catalog-common` so make sure imports are updated. diff --git a/.changeset/curvy-pets-wash.md b/.changeset/curvy-pets-wash.md new file mode 100644 index 0000000000..4c824f9ef4 --- /dev/null +++ b/.changeset/curvy-pets-wash.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added a new method `addLocationAnalyzers` to the `CatalogBuilder`. With this you can add location analyzers to your catalog. These analyzers will be used by the /analyze-location endpoint to decide if the provided URL contains any catalog-info.yaml files already or not. + +Moved the following types from this package to `@backstage/plugin-catalog-backend`. + +- AnalyzeLocationResponse +- AnalyzeLocationRequest +- AnalyzeLocationExistingEntity +- AnalyzeLocationGenerateEntity +- AnalyzeLocationEntityField diff --git a/.changeset/five-tables-grow.md b/.changeset/five-tables-grow.md new file mode 100644 index 0000000000..982b17b97b --- /dev/null +++ b/.changeset/five-tables-grow.md @@ -0,0 +1,11 @@ +--- +'@backstage/plugin-catalog-common': patch +--- + +Moved the following types from `@backstage/plugin-catalog-backend` to this package. + +- AnalyzeLocationResponse +- AnalyzeLocationRequest +- AnalyzeLocationExistingEntity +- AnalyzeLocationGenerateEntity +- AnalyzeLocationEntityField diff --git a/.changeset/fuzzy-dolls-shake.md b/.changeset/fuzzy-dolls-shake.md new file mode 100644 index 0000000000..35ebcf6103 --- /dev/null +++ b/.changeset/fuzzy-dolls-shake.md @@ -0,0 +1,21 @@ +--- +'@backstage/plugin-catalog-import': minor +--- + +**Breaking** +Moved the code search for the existing catalog-info.yaml files to the backend from the frontend. It means it will use the configured GitHub integration's credentials. + +Add the following to your `CatalogBuilder` to have the repo URL ingestion working again. + +```ts +// catalog.ts +import { GitHubLocationAnalyzer } from '@backstage/plugin-catalog-backend-module-github'; +... + builder.addLocationAnalyzers( + new GitHubLocationAnalyzer({ + discovery: env.discovery, + config: env.config, + }), + ); +... +``` diff --git a/.changeset/moody-carrots-shout.md b/.changeset/moody-carrots-shout.md new file mode 100644 index 0000000000..486590e0ba --- /dev/null +++ b/.changeset/moody-carrots-shout.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added `GitHubLocationAnalyzer`. This can be used to add to the `CatalogBuilder`. When added this will be used by `RepoLocationAnalyzer` to figure out if the given URL that you are trying to import from the /catalog-import page already contains catalog-info.yaml files. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 8e8055bafc..c89a34c259 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -3,17 +3,21 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; +import { Entity } from '@backstage/catalog-model'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GitHubIntegrationConfig } from '@backstage/integration'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend'; import { TaskRunner } from '@backstage/backend-tasks'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; @@ -70,6 +74,30 @@ export type GithubEntityProviderCatalogModuleOptions = { schedule?: TaskScheduleDefinition; }; +// @public (undocumented) +export class GitHubLocationAnalyzer implements ScmLocationAnalyzer { + constructor(options: GitHubLocationAnalyzerOptions); + // (undocumented) + analyze({ url, catalogFilename }: AnalyzeOptions): Promise<{ + existing: { + location: { + type: string; + target: string; + }; + isRegistered: boolean; + entity: Entity; + }[]; + }>; + // (undocumented) + supports(url: string): boolean; +} + +// @public (undocumented) +export type GitHubLocationAnalyzerOptions = { + config: Config; + discovery: PluginEndpointDiscovery; +}; + // @public export type GithubMultiOrgConfig = Array<{ name: string; diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index 686b45f13c..1fb82a648a 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -36,6 +36,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", @@ -44,6 +45,8 @@ "@backstage/plugin-catalog-node": "workspace:^", "@backstage/types": "workspace:^", "@octokit/graphql": "^5.0.0", + "@octokit/rest": "^19.0.3", + "git-url-parse": "^13.0.0", "lodash": "^4.17.21", "msw": "^0.47.0", "node-fetch": "^2.6.7", diff --git a/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts new file mode 100644 index 0000000000..0dcc131502 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const octokit = { + search: { + code: jest.fn(), + }, + repos: { + get: jest.fn(), + }, +}; + +jest.mock('@octokit/rest', () => { + class Octokit { + constructor() { + return octokit; + } + } + return { Octokit }; +}); + +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { GitHubLocationAnalyzer } from './GitHubLocationAnalyzer'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; +import { ConfigReader } from '@backstage/config'; + +const server = setupServer(); + +describe('GitHubLocationAnalyzer', () => { + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'), + getExternalBaseUrl: jest.fn(), + }; + const config = new ConfigReader({ + integrations: { + github: [ + { + host: 'h.com', + token: 't', + }, + ], + }, + }); + + setupRequestMockHandlers(server); + + beforeEach(() => { + server.use( + rest.post('http://localhost:7007/locations', async (_, res, ctx) => { + return res( + ctx.status(201), + ctx.json({ + location: 'test', + exists: false, + entities: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + name: 'test-entity', + }, + spec: { + type: 'url', + target: 'whatever', + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity-2', + description: 'The expected description 2', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, + ], + }), + ); + }), + ); + + octokit.repos.get.mockResolvedValue({ + data: { default_branch: 'my_default_branch' }, + }); + }); + + it('should analyze', async () => { + octokit.search.code.mockImplementation((opts: { q: string }) => { + if (opts.q === 'filename:catalog-info.yaml repo:foo/bar') { + return Promise.resolve({ + data: { items: [{ path: 'catalog-info.yaml' }], total_count: 1 }, + }); + } + return Promise.reject(); + }); + + const analyzer = new GitHubLocationAnalyzer({ + discovery: mockDiscoveryApi, + config, + }); + const result = await analyzer.analyze({ + url: 'https://github.com/foo/bar', + }); + + expect(result.existing[0].isRegistered).toBeFalsy(); + expect(result.existing[0].location).toEqual({ + type: 'url', + target: + 'https://github.com/foo/bar/blob/my_default_branch/catalog-info.yaml', + }); + }); + it('should use the provided entity filename for search', async () => { + octokit.search.code.mockImplementation((opts: { q: string }) => { + if (opts.q === 'filename:anvil.yaml repo:foo/bar') { + return Promise.resolve({ + data: { items: [{ path: 'anvil.yaml' }], total_count: 1 }, + }); + } + return Promise.reject(); + }); + + const analyzer = new GitHubLocationAnalyzer({ + discovery: mockDiscoveryApi, + config, + }); + const result = await analyzer.analyze({ + url: 'https://github.com/foo/bar', + catalogFilename: 'anvil.yaml', + }); + + expect(result.existing[0].location).toEqual({ + type: 'url', + target: 'https://github.com/foo/bar/blob/my_default_branch/anvil.yaml', + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts new file mode 100644 index 0000000000..d1112d2cc0 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/analyzers/GitHubLocationAnalyzer.ts @@ -0,0 +1,103 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; +import { ScmIntegrations } from '@backstage/integration'; +import { Octokit } from '@octokit/rest'; +import { trimEnd } from 'lodash'; +import parseGitUrl from 'git-url-parse'; +import { + AnalyzeOptions, + ScmLocationAnalyzer, +} from '@backstage/plugin-catalog-backend'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; + +/** @public */ +export type GitHubLocationAnalyzerOptions = { + config: Config; + discovery: PluginEndpointDiscovery; +}; +/** @public */ +export class GitHubLocationAnalyzer implements ScmLocationAnalyzer { + private readonly catalogClient: CatalogApi; + private readonly config: Config; + + constructor(options: GitHubLocationAnalyzerOptions) { + this.config = options.config; + this.catalogClient = new CatalogClient({ discoveryApi: options.discovery }); + } + supports(url: string) { + const integrations = ScmIntegrations.fromConfig(this.config); + const integration = integrations.byUrl(url); + return integration?.type === 'github'; + } + async analyze({ url, catalogFilename }: AnalyzeOptions) { + const { owner, name: repo } = parseGitUrl(url); + + const catalogFile = catalogFilename || 'catalog-info.yaml'; + + const query = `filename:${catalogFile} repo:${owner}/${repo}`; + + const integration = ScmIntegrations.fromConfig(this.config).github.byUrl( + url, + ); + if (!integration) { + throw new Error('Make sure you have a GitHub integration configured'); + } + + const octokitClient = new Octokit({ + auth: integration.config.token, + baseUrl: integration.config.apiBaseUrl, + }); + + const searchResult = await octokitClient.search + .code({ q: query }) + .catch(e => { + throw new Error(`Couldn't search repository for metadata file, ${e}`); + }); + + const exists = searchResult.data.total_count > 0; + if (exists) { + const repoInformation = await octokitClient.repos + .get({ owner, repo }) + .catch(e => { + throw new Error(`Couldn't fetch repo data, ${e}`); + }); + const defaultBranch = repoInformation.data.default_branch; + + const result = await Promise.all( + searchResult.data.items + .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`) + .map(async target => { + const addLocationResult = await this.catalogClient.addLocation({ + type: 'url', + target, + dryRun: true, + }); + return addLocationResult.entities.map(e => ({ + location: { type: 'url', target }, + isRegistered: !!addLocationResult.exists, + entity: e, + })); + }), + ); + + return { existing: result.flat() }; + } + return { existing: [] }; + } +} diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index d793d48f1e..ca7c72f11b 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -29,3 +29,5 @@ export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntity export type { GithubMultiOrgConfig } from './lib'; export { githubEntityProviderCatalogModule } from './module'; export type { GithubEntityProviderCatalogModuleOptions } from './module'; +export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer'; +export type { GitHubLocationAnalyzerOptions } from './analyzers/GitHubLocationAnalyzer'; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index f18d396114..d23c6f7519 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -5,6 +5,11 @@ ```ts /// +import { AnalyzeLocationEntityField as AnalyzeLocationEntityField_2 } from '@backstage/plugin-catalog-common'; +import { AnalyzeLocationExistingEntity as AnalyzeLocationExistingEntity_2 } from '@backstage/plugin-catalog-common'; +import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from '@backstage/plugin-catalog-common'; +import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common'; +import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogApi } from '@backstage/catalog-client'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; @@ -51,39 +56,25 @@ import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; import { Validators } from '@backstage/catalog-model'; -// @public (undocumented) -export type AnalyzeLocationEntityField = { - field: string; - state: - | 'analysisSuggestedValue' - | 'analysisSuggestedNoValue' - | 'needsUserInput'; - value: string | null; - description: string; -}; +// @public @deprecated +export type AnalyzeLocationEntityField = AnalyzeLocationEntityField_2; -// @public -export type AnalyzeLocationExistingEntity = { - location: LocationSpec; - isRegistered: boolean; - entity: Entity; -}; +// @public @deprecated +export type AnalyzeLocationExistingEntity = AnalyzeLocationExistingEntity_2; -// @public -export type AnalyzeLocationGenerateEntity = { - entity: RecursivePartial; - fields: AnalyzeLocationEntityField[]; -}; +// @public @deprecated +export type AnalyzeLocationGenerateEntity = AnalyzeLocationGenerateEntity_2; + +// @public @deprecated (undocumented) +export type AnalyzeLocationRequest = AnalyzeLocationRequest_2; + +// @public @deprecated (undocumented) +export type AnalyzeLocationResponse = AnalyzeLocationResponse_2; // @public (undocumented) -export type AnalyzeLocationRequest = { - location: LocationSpec; -}; - -// @public (undocumented) -export type AnalyzeLocationResponse = { - existingEntityFiles: AnalyzeLocationExistingEntity[]; - generateEntities: AnalyzeLocationGenerateEntity[]; +export type AnalyzeOptions = { + url: string; + catalogFilename?: string; }; // @public (undocumented) @@ -133,6 +124,9 @@ export class CatalogBuilder { addEntityProvider( ...providers: Array> ): CatalogBuilder; + addLocationAnalyzers( + ...analyzers: Array> + ): CatalogBuilder; // @alpha addPermissionRules( ...permissionRules: Array< @@ -531,6 +525,14 @@ export type ProcessingIntervalFunction = () => number; export { processingResult }; +// @public (undocumented) +export type ScmLocationAnalyzer = { + supports(url: string): boolean; + analyze(options: AnalyzeOptions): Promise<{ + existing: AnalyzeLocationExistingEntity[]; + }>; +}; + // @public (undocumented) export class UrlReaderProcessor implements CatalogProcessor { constructor(options: { reader: UrlReader; logger: Logger }); diff --git a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts index b84dff7de8..761a0f8c0a 100644 --- a/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts +++ b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts @@ -18,34 +18,32 @@ import { Logger } from 'winston'; import parseGitUrl from 'git-url-parse'; import { Entity } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { LocationAnalyzer, ScmLocationAnalyzer } from './types'; import { AnalyzeLocationRequest, AnalyzeLocationResponse, - LocationAnalyzer, -} from './types'; +} from '@backstage/plugin-catalog-common'; export class RepoLocationAnalyzer implements LocationAnalyzer { private readonly logger: Logger; private readonly scmIntegrations: ScmIntegrationRegistry; + private readonly analyzers: ScmLocationAnalyzer[]; - constructor(logger: Logger, scmIntegrations: ScmIntegrationRegistry) { + constructor( + logger: Logger, + scmIntegrations: ScmIntegrationRegistry, + analyzers: ScmLocationAnalyzer[], + ) { this.logger = logger; this.scmIntegrations = scmIntegrations; + this.analyzers = analyzers; } async analyzeLocation( request: AnalyzeLocationRequest, ): Promise { - const { owner, name } = parseGitUrl(request.location.target); - const entity: Entity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: name, - }, - spec: { type: 'other', lifecycle: 'unknown' }, - }; - const integration = this.scmIntegrations.byUrl(request.location.target); + const { owner, name } = parseGitUrl(request.location.target); + let annotationPrefix; switch (integration?.type) { case 'azure': @@ -64,6 +62,33 @@ export class RepoLocationAnalyzer implements LocationAnalyzer { break; } + const analyzer = this.analyzers.find(a => + a.supports(request.location.target), + ); + if (analyzer) { + const analyzerResult = await analyzer.analyze({ + url: request.location.target, + }); + if (analyzerResult.existing.length > 0) { + this.logger.debug( + `entity for ${request.location.target} already exists.`, + ); + return { + existingEntityFiles: analyzerResult.existing, + generateEntities: [], + }; + } + } + + const entity: Entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: name, + }, + spec: { type: 'other', lifecycle: 'unknown' }, + }; + if (annotationPrefix) { entity.metadata.annotations = { [`${annotationPrefix}/project-slug`]: `${owner}/${name}`, diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index c97d029b5c..0b00809043 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -21,4 +21,6 @@ export type { AnalyzeLocationRequest, AnalyzeLocationResponse, LocationAnalyzer, + ScmLocationAnalyzer, + AnalyzeOptions, } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 2602bd080f..78339b3b23 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,9 +14,57 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; -import { RecursivePartial } from '../util/RecursivePartial'; -import { LocationSpec } from '@backstage/plugin-catalog-node'; +import { + AnalyzeLocationRequest as NonDeprecatedAnalyzeLocationRequest, + AnalyzeLocationResponse as NonDeprecatedAnalyzeLocationResponse, + AnalyzeLocationExistingEntity as NonDeprecatedAnalyzeLocationExistingEntity, + AnalyzeLocationGenerateEntity as NonDeprecatedAnalyzeLocationGenerateEntity, + AnalyzeLocationEntityField as NonDeprecatedAnalyzeLocationEntityField, +} from '@backstage/plugin-catalog-common'; + +/** + * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead + */ +export type AnalyzeLocationRequest = NonDeprecatedAnalyzeLocationRequest; +/** + * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead + */ +export type AnalyzeLocationResponse = NonDeprecatedAnalyzeLocationResponse; + +/** + * If the folder pointed to already contained catalog info yaml files, they are + * read and emitted like this so that the frontend can inform the user that it + * located them and can make sure to register them as well if they weren't + * already + * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead + */ +export type AnalyzeLocationExistingEntity = + NonDeprecatedAnalyzeLocationExistingEntity; +/** + * This is some form of representation of what the analyzer could deduce. + * We should probably have a chat about how this can best be conveyed to + * the frontend. It'll probably contain a (possibly incomplete) entity, plus + * enough info for the frontend to know what form data to show to the user + * for overriding/completing the info. + * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead + */ +export type AnalyzeLocationGenerateEntity = + NonDeprecatedAnalyzeLocationGenerateEntity; + +/** + * + * This is where I get really vague. Something like this perhaps? Or it could be + * something like a json-schema that contains enough info for the frontend to + * be able to present a form and explanations + * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead + */ +export type AnalyzeLocationEntityField = + NonDeprecatedAnalyzeLocationEntityField; /** @public */ export type LocationAnalyzer = { @@ -30,71 +78,19 @@ export type LocationAnalyzer = { location: AnalyzeLocationRequest, ): Promise; }; - /** @public */ -export type AnalyzeLocationRequest = { - location: LocationSpec; +export type AnalyzeOptions = { + url: string; + catalogFilename?: string; }; /** @public */ -export type AnalyzeLocationResponse = { - existingEntityFiles: AnalyzeLocationExistingEntity[]; - generateEntities: AnalyzeLocationGenerateEntity[]; -}; - -/** - * If the folder pointed to already contained catalog info yaml files, they are - * read and emitted like this so that the frontend can inform the user that it - * located them and can make sure to register them as well if they weren't - * already - * @public - */ -export type AnalyzeLocationExistingEntity = { - location: LocationSpec; - isRegistered: boolean; - entity: Entity; -}; - -/** - * This is some form of representation of what the analyzer could deduce. - * We should probably have a chat about how this can best be conveyed to - * the frontend. It'll probably contain a (possibly incomplete) entity, plus - * enough info for the frontend to know what form data to show to the user - * for overriding/completing the info. - * @public - */ -export type AnalyzeLocationGenerateEntity = { - // Some form of partial representation of the entity - entity: RecursivePartial; - // Lists the suggestions that the user may want to override - fields: AnalyzeLocationEntityField[]; -}; - -// This is where I get really vague. Something like this perhaps? Or it could be -// something like a json-schema that contains enough info for the frontend to -// be able to present a form and explanations -/** @public */ -export type AnalyzeLocationEntityField = { - /** - * e.g. "spec.owner"? The frontend needs to know how to "inject" the field into the - * entity again if the user wants to change it - */ - field: string; - - /** The outcome of the analysis for this particular field */ - state: - | 'analysisSuggestedValue' - | 'analysisSuggestedNoValue' - | 'needsUserInput'; - - // If the analysis did suggest a value, this is where it would be. Not sure if we want - // to limit this to strings or if we want it to be any JsonValue - value: string | null; - /** - * A text to show to the user to inform about the choices made. Like, it could say - * "Found a CODEOWNERS file that covers this target, so we suggest leaving this - * field empty; which would currently make it owned by X" where X is taken from the - * codeowners file. - */ - description: string; +export type ScmLocationAnalyzer = { + /** The method that decides if this analyzer can work with the provided url */ + supports(url: string): boolean; + /** This function can return an array of already existing entities */ + analyze(options: AnalyzeOptions): Promise<{ + /** Existing entities in the analyzed location */ + existing: AnalyzeLocationExistingEntity[]; + }>; }; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index d47ace1454..d5bc7057dd 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -56,7 +56,7 @@ import { yamlPlaceholderResolver, } from '../modules/core/PlaceholderProcessor'; import { defaultEntityDataParser } from '../modules/util/parse'; -import { LocationAnalyzer } from '../ingestion/types'; +import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types'; import { CatalogProcessingEngine } from '../processing'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; @@ -115,6 +115,10 @@ export type CatalogEnvironment = { * after the processors' pre-processing steps. All policies are given the * chance to inspect the entity, and all of them have to pass in order for * the entity to be considered valid from an overall point of view. + * - Location analyzers can be added. These are responsible for analyzing + * repositories when onboarding them into the catalog, by finding + * catalog-info.yaml files and other artifacts that can help automatically + * register or create catalog data on the user's behalf. * - Placeholder resolvers can be replaced or added. These run on the raw * structured data between the parsing and pre-processing steps, to replace * dollar-prefixed entries with their actual values (like $file). @@ -135,6 +139,7 @@ export class CatalogBuilder { private fieldFormatValidators: Partial; private entityProviders: EntityProvider[]; private processors: CatalogProcessor[]; + private locationAnalyzers: ScmLocationAnalyzer[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; private onProcessingError?: (event: { @@ -165,6 +170,7 @@ export class CatalogBuilder { this.fieldFormatValidators = {}; this.entityProviders = []; this.processors = []; + this.locationAnalyzers = []; this.processorsReplace = false; this.parser = undefined; this.permissionRules = Object.values(catalogPermissionRules); @@ -335,6 +341,21 @@ export class CatalogBuilder { ]; } + /** + * Adds Location Analyzers. These are responsible for analyzing + * repositories when onboarding them into the catalog, by finding + * catalog-info.yaml files and other artifacts that can help automatically + * register or create catalog data on the user's behalf. + * + * @param locationAnalyzers - One or more location analyzers + */ + addLocationAnalyzers( + ...analyzers: Array> + ): CatalogBuilder { + this.locationAnalyzers.push(...analyzers.flat()); + return this; + } + /** * Sets up the catalog to use a custom parser for entity data. * @@ -478,7 +499,8 @@ export class CatalogBuilder { ); const locationAnalyzer = - this.locationAnalyzer ?? new RepoLocationAnalyzer(logger, integrations); + this.locationAnalyzer ?? + new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers); const locationService = new AuthorizedLocationService( new DefaultLocationService(locationStore, orchestrator, { allowedLocationTypes: this.allowedLocationType, diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 0002231bab..5267519c33 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -229,9 +229,15 @@ export async function createRouter( router.post('/analyze-location', async (req, res) => { const body = await validateRequestBody( req, - z.object({ location: locationInput }), + z.object({ + location: locationInput, + catalogFilename: z.string().optional(), + }), ); - const schema = z.object({ location: locationInput }); + const schema = z.object({ + location: locationInput, + catalogFilename: z.string().optional(), + }); const output = await locationAnalyzer.analyzeLocation(schema.parse(body)); res.status(200).json(output); }); diff --git a/plugins/catalog-common/api-report.md b/plugins/catalog-common/api-report.md index b7eaf99d8b..1bef626c79 100644 --- a/plugins/catalog-common/api-report.md +++ b/plugins/catalog-common/api-report.md @@ -4,9 +4,46 @@ ```ts import { BasicPermission } from '@backstage/plugin-permission-common'; +import { Entity } from '@backstage/catalog-model'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { ResourcePermission } from '@backstage/plugin-permission-common'; +// @public (undocumented) +export type AnalyzeLocationEntityField = { + field: string; + state: + | 'analysisSuggestedValue' + | 'analysisSuggestedNoValue' + | 'needsUserInput'; + value: string | null; + description: string; +}; + +// @public +export type AnalyzeLocationExistingEntity = { + location: LocationSpec; + isRegistered: boolean; + entity: Entity; +}; + +// @public +export type AnalyzeLocationGenerateEntity = { + entity: RecursivePartial; + fields: AnalyzeLocationEntityField[]; +}; + +// @public (undocumented) +export type AnalyzeLocationRequest = { + location: LocationSpec; + catalogFilename?: string; +}; + +// @public (undocumented) +export type AnalyzeLocationResponse = { + existingEntityFiles: AnalyzeLocationExistingEntity[]; + generateEntities: AnalyzeLocationGenerateEntity[]; +}; + // @alpha export const catalogEntityCreatePermission: BasicPermission; @@ -55,6 +92,13 @@ export const catalogPermissions: ( | ResourcePermission<'catalog-entity'> )[]; +// @public +export type LocationSpec = { + type: string; + target: string; + presence?: 'optional' | 'required'; +}; + // @alpha export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; ``` diff --git a/plugins/catalog-common/package.json b/plugins/catalog-common/package.json index 5be665b7d6..12dcbc4ccf 100644 --- a/plugins/catalog-common/package.json +++ b/plugins/catalog-common/package.json @@ -33,6 +33,7 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/catalog-model": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-search-common": "workspace:^" }, diff --git a/plugins/catalog-common/src/common.ts b/plugins/catalog-common/src/common.ts new file mode 100644 index 0000000000..c926a94b28 --- /dev/null +++ b/plugins/catalog-common/src/common.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Holds the entity location information. + * + * @remarks + * + * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. + * This flag is then set to indicate that the file can be not present. + * default value: 'required'. + * + * @public + */ +export type LocationSpec = { + type: string; + target: string; + presence?: 'optional' | 'required'; +}; diff --git a/plugins/catalog-common/src/index.ts b/plugins/catalog-common/src/index.ts index 614948e56c..95bd65bc92 100644 --- a/plugins/catalog-common/src/index.ts +++ b/plugins/catalog-common/src/index.ts @@ -35,3 +35,5 @@ export { export type { CatalogEntityPermission } from './permissions'; export * from './search'; +export * from './ingestion'; +export type { LocationSpec } from './common'; diff --git a/plugins/catalog-common/src/ingestion/LocationAnalyzer.ts b/plugins/catalog-common/src/ingestion/LocationAnalyzer.ts new file mode 100644 index 0000000000..08ae5ef0cb --- /dev/null +++ b/plugins/catalog-common/src/ingestion/LocationAnalyzer.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '../common'; +import { Entity } from '@backstage/catalog-model'; +import { RecursivePartial } from './RecursivePartial'; + +/** @public */ +export type AnalyzeLocationRequest = { + location: LocationSpec; + catalogFilename?: string; +}; + +/** @public */ +export type AnalyzeLocationResponse = { + existingEntityFiles: AnalyzeLocationExistingEntity[]; + generateEntities: AnalyzeLocationGenerateEntity[]; +}; + +/** + * If the folder pointed to already contained catalog info yaml files, they are + * read and emitted like this so that the frontend can inform the user that it + * located them and can make sure to register them as well if they weren't + * already + * @public + */ +export type AnalyzeLocationExistingEntity = { + location: LocationSpec; + isRegistered: boolean; + entity: Entity; +}; + +/** + * This is some form of representation of what the analyzer could deduce. + * We should probably have a chat about how this can best be conveyed to + * the frontend. It'll probably contain a (possibly incomplete) entity, plus + * enough info for the frontend to know what form data to show to the user + * for overriding/completing the info. + * @public + */ +export type AnalyzeLocationGenerateEntity = { + // Some form of partial representation of the entity + entity: RecursivePartial; + // Lists the suggestions that the user may want to override + fields: AnalyzeLocationEntityField[]; +}; + +// This is where I get really vague. Something like this perhaps? Or it could be +// something like a json-schema that contains enough info for the frontend to +// be able to present a form and explanations +/** @public */ +export type AnalyzeLocationEntityField = { + /** + * e.g. "spec.owner"? The frontend needs to know how to "inject" the field into the + * entity again if the user wants to change it + */ + field: string; + + /** The outcome of the analysis for this particular field */ + state: + | 'analysisSuggestedValue' + | 'analysisSuggestedNoValue' + | 'needsUserInput'; + + // If the analysis did suggest a value, this is where it would be. Not sure if we want + // to limit this to strings or if we want it to be any JsonValue + value: string | null; + /** + * A text to show to the user to inform about the choices made. Like, it could say + * "Found a CODEOWNERS file that covers this target, so we suggest leaving this + * field empty; which would currently make it owned by X" where X is taken from the + * codeowners file. + */ + description: string; +}; diff --git a/plugins/catalog-common/src/ingestion/RecursivePartial.test.ts b/plugins/catalog-common/src/ingestion/RecursivePartial.test.ts new file mode 100644 index 0000000000..ab8d50534e --- /dev/null +++ b/plugins/catalog-common/src/ingestion/RecursivePartial.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RecursivePartial } from './RecursivePartial'; + +describe('RecursivePartial', () => { + it('is recursive', () => { + type X = { + required: { + required: string; + }; + }; + const x: RecursivePartial = { + required: {}, + }; + expect(x).toEqual({ required: {} }); + }); +}); diff --git a/plugins/catalog-common/src/ingestion/RecursivePartial.ts b/plugins/catalog-common/src/ingestion/RecursivePartial.ts new file mode 100644 index 0000000000..c452836f34 --- /dev/null +++ b/plugins/catalog-common/src/ingestion/RecursivePartial.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Makes all keys of an entire hierarchy optional. + * @ignore + */ +export type RecursivePartial = { + [P in keyof T]?: T[P] extends (infer U)[] + ? RecursivePartial[] + : T[P] extends object + ? RecursivePartial + : T[P]; +}; diff --git a/plugins/catalog-common/src/ingestion/index.ts b/plugins/catalog-common/src/ingestion/index.ts new file mode 100644 index 0000000000..aced124bb4 --- /dev/null +++ b/plugins/catalog-common/src/ingestion/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { + AnalyzeLocationResponse, + AnalyzeLocationRequest, + AnalyzeLocationExistingEntity, + AnalyzeLocationGenerateEntity, + AnalyzeLocationEntityField, +} from './LocationAnalyzer'; diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 5355d3acd9..abb114ad2f 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -40,6 +40,7 @@ "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/integration-react": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-react": "workspace:^", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index 30b9f47d99..d507ac3bef 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -297,7 +297,7 @@ describe('CatalogImportClient', () => { it('should find locations from github', async () => { (new Octokit().search.code as any as jest.Mock).mockResolvedValueOnce({ data: { - total_count: 2, + total_count: 3, items: [ { path: 'simple/path/catalog-info.yaml' }, { path: 'co/mple/x/path/catalog-info.yaml' }, @@ -305,24 +305,72 @@ describe('CatalogImportClient', () => { ], }, }); - - catalogApi.addLocation.mockImplementation(async ({ type, target }) => ({ - location: { - id: 'id-0', - type: type ?? 'url', - target, - }, - entities: [ - { - apiVersion: '1', - kind: 'k', - metadata: { - name: 'e', - namespace: 'n', + server.use( + rest.post(`${mockBaseUrl}/analyze-location`, (req, res, ctx) => { + expect(req.body).toEqual({ + location: { + target: 'https://github.com/backstage/backstage', + type: 'url', }, - }, - ], - })); + }); + + return res( + ctx.json({ + generateEntities: [], + existingEntityFiles: [ + { + isRegistered: false, + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/main/simple/path/catalog-info.yaml', + }, + entity: { + apiVersion: '1', + kind: 'k', + metadata: { + name: 'e', + namespace: 'n', + }, + }, + }, + { + isRegistered: false, + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/main/co/mple/x/path/catalog-info.yaml', + }, + entity: { + apiVersion: '1', + kind: 'k', + metadata: { + name: 'e', + namespace: 'n', + }, + }, + }, + { + isRegistered: false, + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/main/catalog-info.yaml', + }, + entity: { + apiVersion: '1', + kind: 'k', + metadata: { + name: 'e', + namespace: 'n', + }, + }, + }, + ], + }), + ); + }), + ); await expect( catalogImportClient.analyzeUrl( @@ -332,16 +380,19 @@ describe('CatalogImportClient', () => { locations: [ { entities: [{ kind: 'k', name: 'e', namespace: 'n' }], + exists: false, target: 'https://github.com/backstage/backstage/blob/main/simple/path/catalog-info.yaml', }, { entities: [{ kind: 'k', name: 'e', namespace: 'n' }], + exists: false, target: 'https://github.com/backstage/backstage/blob/main/co/mple/x/path/catalog-info.yaml', }, { entities: [{ kind: 'k', name: 'e', namespace: 'n' }], + exists: false, target: 'https://github.com/backstage/backstage/blob/main/catalog-info.yaml', }, @@ -426,31 +477,57 @@ describe('CatalogImportClient', () => { }), ); - catalogApi.addLocation.mockImplementation(async ({ type, target }) => ({ - location: { - id: 'id-0', - type: type ?? 'url', - target, - }, - entities: [ - { - apiVersion: '1', - kind: 'Location', - metadata: { - name: 'my-entity', - namespace: 'my-namespace', + server.use( + rest.post(`${mockBaseUrl}/analyze-location`, (req, res, ctx) => { + expect(req.body).toEqual({ + location: { + target: 'https://github.com/acme-corp/our-awesome-api', + type: 'url', }, - }, - { - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'my-entity', - namespace: 'my-namespace', - }, - }, - ], - })); + catalogFilename: 'anvil.yaml', + }); + + return res( + ctx.json({ + generateEntities: [], + existingEntityFiles: [ + { + isRegistered: false, + location: { + type: 'url', + target: + 'https://github.com/acme-corp/our-awesome-api/blob/main/anvil.yaml', + }, + entity: { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'my-entity', + namespace: 'my-namespace', + }, + }, + }, + { + isRegistered: false, + location: { + type: 'url', + target: + 'https://github.com/acme-corp/our-awesome-api/blob/main/anvil.yaml', + }, + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'my-entity', + namespace: 'my-namespace', + }, + }, + }, + ], + }), + ); + }), + ); await expect( catalogImportClient.analyzeUrl(repositoryUrl), @@ -470,6 +547,7 @@ describe('CatalogImportClient', () => { }, ], target: `${repositoryUrl}/blob/main/${entityFilename}`, + exists: false, }, ], type: 'locations', diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index c12e391ec9..0663122253 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -15,7 +15,6 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi, DiscoveryApi, @@ -28,11 +27,11 @@ import { import { ScmAuthApi } from '@backstage/integration-react'; import { Octokit } from '@octokit/rest'; import { Base64 } from 'js-base64'; -import { PartialEntity } from '../types'; import { AnalyzeResult, CatalogImportApi } from './CatalogImportApi'; import { getGithubIntegrationConfig } from './GitHub'; -import { trimEnd } from 'lodash'; import { getBranchName, getCatalogFilename } from '../components/helpers'; +import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-common'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * The default implementation of the {@link CatalogImportApi}. @@ -105,16 +104,40 @@ export class CatalogImportClient implements CatalogImportApi { ); } - // TODO: this could be part of the analyze-location endpoint - const locations = await this.checkGitHubForExistingCatalogInfo({ - ...ghConfig, - url, + const analyzation = await this.analyzeLocation({ + repo: url, }); - if (locations.length > 0) { + if (analyzation.existingEntityFiles.length > 0) { + const locations = analyzation.existingEntityFiles.reduce< + Record< + string, + { + target: string; + exists?: boolean; + entities: CompoundEntityRef[]; + } + > + >((state, curr) => { + state[curr.location.target] = { + target: curr.location.target, + exists: curr.isRegistered, + entities: [ + ...(curr.location.target in state + ? state[curr.location.target].entities + : []), + { + name: curr.entity.metadata.name, + namespace: curr.entity.metadata.namespace ?? 'default', + kind: curr.entity.kind, + }, + ], + }; + return state; + }, {}); return { type: 'locations', - locations, + locations: Object.values(locations), }; } @@ -122,9 +145,7 @@ export class CatalogImportClient implements CatalogImportApi { type: 'repository', integrationType: 'github', url: url, - generatedEntities: await this.generateEntityDefinitions({ - repo: url, - }), + generatedEntities: analyzation.generateEntities.map(x => x.entity), }; } @@ -174,9 +195,9 @@ the component will become available.\n\nFor more information, read an \ } // TODO: this could be part of the catalog api - private async generateEntityDefinitions(options: { + private async analyzeLocation(options: { repo: string; - }): Promise { + }): Promise { const { token } = await this.identityApi.getCredentials(); const response = await fetch( `${await this.discoveryApi.getBaseUrl('catalog')}/analyze-location`, @@ -188,6 +209,13 @@ the component will become available.\n\nFor more information, read an \ method: 'POST', body: JSON.stringify({ location: { type: 'url', target: options.repo }, + ...(this.configApi.getOptionalString( + 'catalog.import.entityFilename', + ) && { + catalogFilename: this.configApi.getOptionalString( + 'catalog.import.entityFilename', + ), + }), }), }, ).catch(e => { @@ -200,69 +228,7 @@ the component will become available.\n\nFor more information, read an \ } const payload = await response.json(); - return payload.generateEntities.map((x: any) => x.entity); - } - - // TODO: this response should better be part of the analyze-locations response and scm-independent / implemented per scm - private async checkGitHubForExistingCatalogInfo(options: { - url: string; - owner: string; - repo: string; - githubIntegrationConfig: GitHubIntegrationConfig; - }): Promise< - Array<{ - target: string; - entities: CompoundEntityRef[]; - }> - > { - const { url, owner, repo, githubIntegrationConfig } = options; - - const { token } = await this.scmAuthApi.getCredentials({ url }); - const octo = new Octokit({ - auth: token, - baseUrl: githubIntegrationConfig.apiBaseUrl, - }); - const catalogFilename = getCatalogFilename(this.configApi); - const query = `repo:${owner}/${repo}+filename:${catalogFilename}`; - - const searchResult = await octo.search.code({ q: query }).catch(e => { - throw new Error( - formatHttpErrorMessage( - "Couldn't search repository for metadata file.", - e, - ), - ); - }); - const exists = searchResult.data.total_count > 0; - if (exists) { - const repoInformation = await octo.repos.get({ owner, repo }).catch(e => { - throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e)); - }); - const defaultBranch = repoInformation.data.default_branch; - - return await Promise.all( - searchResult.data.items - .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`) - .map(async target => { - const result = await this.catalogApi.addLocation({ - type: 'url', - target, - dryRun: true, - }); - return { - target, - exists: result.exists, - entities: result.entities.map(e => ({ - kind: e.kind, - namespace: e.metadata.namespace ?? 'default', - name: e.metadata.name, - })), - }; - }), - ); - } - - return []; + return payload; } // TODO: extract this function and implement for non-github diff --git a/plugins/catalog-node/api-report.md b/plugins/catalog-node/api-report.md index 525516be54..09a05ec7d9 100644 --- a/plugins/catalog-node/api-report.md +++ b/plugins/catalog-node/api-report.md @@ -10,6 +10,7 @@ import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { JsonValue } from '@backstage/types'; +import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -31,7 +32,7 @@ export const catalogProcessingExtensionPoint: ExtensionPoint; preProcessEntity?( entity: Entity, - location: LocationSpec, + location: LocationSpec_2, emit: CatalogProcessorEmit, - originLocation: LocationSpec, + originLocation: LocationSpec_2, cache: CatalogProcessorCache, ): Promise; validateEntityKind?(entity: Entity): Promise; postProcessEntity?( entity: Entity, - location: LocationSpec, + location: LocationSpec_2, emit: CatalogProcessorEmit, cache: CatalogProcessorCache, ): Promise; @@ -66,26 +67,26 @@ export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void; export type CatalogProcessorEntityResult = { type: 'entity'; entity: Entity; - location: LocationSpec; + location: LocationSpec_2; }; // @public (undocumented) export type CatalogProcessorErrorResult = { type: 'error'; error: Error; - location: LocationSpec; + location: LocationSpec_2; }; // @public (undocumented) export type CatalogProcessorLocationResult = { type: 'location'; - location: LocationSpec; + location: LocationSpec_2; }; // @public export type CatalogProcessorParser = (options: { data: Buffer; - location: LocationSpec; + location: LocationSpec_2; }) => AsyncIterable; // @public (undocumented) @@ -153,30 +154,26 @@ export type EntityRelationSpec = { target: CompoundEntityRef; }; -// @public -export type LocationSpec = { - type: string; - target: string; - presence?: 'optional' | 'required'; -}; +// @public @deprecated +export type LocationSpec = LocationSpec_2; // @public export const processingResult: Readonly<{ readonly notFoundError: ( - atLocation: LocationSpec, + atLocation: LocationSpec_2, message: string, ) => CatalogProcessorResult; readonly inputError: ( - atLocation: LocationSpec, + atLocation: LocationSpec_2, message: string, ) => CatalogProcessorResult; readonly generalError: ( - atLocation: LocationSpec, + atLocation: LocationSpec_2, message: string, ) => CatalogProcessorResult; - readonly location: (newLocation: LocationSpec) => CatalogProcessorResult; + readonly location: (newLocation: LocationSpec_2) => CatalogProcessorResult; readonly entity: ( - atLocation: LocationSpec, + atLocation: LocationSpec_2, newEntity: Entity, ) => CatalogProcessorResult; readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; diff --git a/plugins/catalog-node/package.json b/plugins/catalog-node/package.json index 20ffc22216..f008e38e6b 100644 --- a/plugins/catalog-node/package.json +++ b/plugins/catalog-node/package.json @@ -28,6 +28,7 @@ "@backstage/catalog-client": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/errors": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/types": "workspace:^" }, "devDependencies": { diff --git a/plugins/catalog-node/src/api/common.ts b/plugins/catalog-node/src/api/common.ts index f3b4a387ba..c91aeca2d9 100644 --- a/plugins/catalog-node/src/api/common.ts +++ b/plugins/catalog-node/src/api/common.ts @@ -15,6 +15,7 @@ */ import { CompoundEntityRef } from '@backstage/catalog-model'; +import { LocationSpec as NonDeprecatedLocationSpec } from '@backstage/plugin-catalog-common'; /** * Holds the entity location information. @@ -26,12 +27,9 @@ import { CompoundEntityRef } from '@backstage/catalog-model'; * default value: 'required'. * * @public + * @deprecated use the same type from `@backstage/plugin-catalog-common` instead */ -export type LocationSpec = { - type: string; - target: string; - presence?: 'optional' | 'required'; -}; +export type LocationSpec = NonDeprecatedLocationSpec; /** * Holds the relation data for entities. diff --git a/plugins/catalog-node/src/api/processingResult.ts b/plugins/catalog-node/src/api/processingResult.ts index 84f1f4b70d..d8b6f3ae02 100644 --- a/plugins/catalog-node/src/api/processingResult.ts +++ b/plugins/catalog-node/src/api/processingResult.ts @@ -17,7 +17,8 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Entity } from '@backstage/catalog-model'; import { CatalogProcessorResult } from './processor'; -import { EntityRelationSpec, LocationSpec } from './common'; +import { EntityRelationSpec } from './common'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; /** * Factory functions for the standard processing result types. diff --git a/plugins/catalog-node/src/api/processor.ts b/plugins/catalog-node/src/api/processor.ts index 44e8b298b0..3ad080f014 100644 --- a/plugins/catalog-node/src/api/processor.ts +++ b/plugins/catalog-node/src/api/processor.ts @@ -16,7 +16,8 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; -import { EntityRelationSpec, LocationSpec } from './common'; +import { EntityRelationSpec } from './common'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; /** * @public diff --git a/yarn.lock b/yarn.lock index aa664b6d40..8be2e5c8a6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4576,6 +4576,7 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-client": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" @@ -4585,7 +4586,9 @@ __metadata: "@backstage/plugin-catalog-node": "workspace:^" "@backstage/types": "workspace:^" "@octokit/graphql": ^5.0.0 + "@octokit/rest": ^19.0.3 "@types/lodash": ^4.14.151 + git-url-parse: ^13.0.0 lodash: ^4.17.21 msw: ^0.47.0 node-fetch: ^2.6.7 @@ -4750,6 +4753,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-common@workspace:plugins/catalog-common" dependencies: + "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-search-common": "workspace:^" @@ -4831,6 +4835,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/integration-react": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@material-ui/core": ^4.12.2 @@ -4867,6 +4872,7 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/errors": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/types": "workspace:^" languageName: unknown linkType: soft