From f48950e34b14606b12964ed24fbdedde5d6b1ffd Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:00:53 -0400 Subject: [PATCH] feat: github discovery provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .changeset/odd-tomatoes-juggle.md | 9 + docs/integrations/github/discovery.md | 114 ++++++++ .../api-report.md | 21 ++ .../src/index.ts | 12 +- .../GithubDiscoveryProcessor.test.ts | 2 +- .../GithubDiscoveryProcessor.ts | 2 +- .../GithubMultiOrgReaderProcessor.ts | 2 +- .../GithubOrgReaderProcessor.test.ts | 0 .../GithubOrgReaderProcessor.ts | 2 +- .../providers/GitHubEntityProvider.test.ts | 185 ++++++++++++ .../src/providers/GitHubEntityProvider.ts | 271 ++++++++++++++++++ .../GitHubEntityProviderConfig.test.ts | 115 ++++++++ .../providers/GitHubEntityProviderConfig.ts | 90 ++++++ .../GitHubOrgEntityProvider.test.ts | 0 .../GitHubOrgEntityProvider.ts | 2 +- 15 files changed, 817 insertions(+), 10 deletions(-) create mode 100644 .changeset/odd-tomatoes-juggle.md rename plugins/catalog-backend-module-github/src/{ => processors}/GithubDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubDiscoveryProcessor.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubMultiOrgReaderProcessor.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubOrgReaderProcessor.test.ts (100%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubOrgReaderProcessor.ts (99%) create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts rename plugins/catalog-backend-module-github/src/{ => providers}/GitHubOrgEntityProvider.test.ts (100%) rename plugins/catalog-backend-module-github/src/{ => providers}/GitHubOrgEntityProvider.ts (99%) diff --git a/.changeset/odd-tomatoes-juggle.md b/.changeset/odd-tomatoes-juggle.md new file mode 100644 index 0000000000..336af17eb4 --- /dev/null +++ b/.changeset/odd-tomatoes-juggle.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Github Entity Provider functionality for adding entities to the catalog. + +This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities. + +More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page. diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index a0c3fed153..82abc25eca 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -6,6 +6,120 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from repositories in a GitHub organization --- +## GitHub Provider + +The GitHub integration has a discovery provider for discovering catalog +entities within a GitHub organization. The provider will crawl the GitHub +organization and register entities matching the configured path. This can be +useful as an alternative to static locations or manually adding things to the +catalog. This is the prefered method for ingesting entities into the catalog. + +## Installation + +You will have to add the provider in the catalog initialization code of your +backend. They are not installed by default, therefore you have to add a +dependency on `@backstage/plugin-catalog-backend-module-github` to your backend +package + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github +``` + +And then add the entity provider to your catalog builder: + +```diff + // In packages/backend/src/plugins/catalog.ts ++ import { GitHubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ builder.addEntityProvider( ++ GitHubEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ }), ++ ); + + // [...] + } +``` + +## Configuration + +To use the discovery provider, you'll need a GitHub integration +[set up](locations.md) with either a [Personal Access Token](../../getting-started/configuration.md#setting-up-a-github-integration) or [GitHub Apps](./github-apps.md). + +Then you can add a github config to the catalog providers configuration: + +```yaml +catalog: + providers: + github: + # the provider ID can be any camelCase string + providerId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + customProviderId: + organization: 'new-org' # string + catalogPath: '/custom/path/catalog-info.yaml' # string + filters: # optional filters + branch: 'develop' # optional string + repository: '.*' # optional Regex +``` + +This provider supports multiple organizations via unique provider IDs + +> **Note:** It is possible but certainly not recommended to skip the provider ID level. +> If you do so, `default` will be used as provider ID. + +- **catalogPath** _(optional)_: + Default: `/catalog-info.yaml`. + Path where to look for `catalog-info.yaml` files. + When started with `/`, it is an absolute path from the repo root. +- **filters** _(optional)_: + - **branch** _(optional)_: + String used to filter results based on the branch name. + - **repository** _(optional)_: + Regular expression used to filter results based on the repository name. +- **organization**: + Name of your organization account/workspace. + If you want to add multiple organizations, you need to add one provider config each. + +## GitHub API Rate Limits + +GitHub [rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) API requests to 5,000 per hour (or more for Enterprise +accounts). The default Backstage catalog backend refreshes data every 100 +seconds, which issues an API request for each discovered location. + +This means if you have more than ~140 catalog entities, you may get throttled by +rate limiting. You can change the refresh frequency of the catalog in your `packages/backend/src/plugins/catalog.ts` file: + +```typescript +schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 35 }, + timeout: { minutes: 30 }, +}), +``` + +More information about scheduling can be found on the [TaskScheduleDefinition](https://backstage.io/docs/reference/backend-tasks.taskscheduledefinition) page. + +Alternatively, or additionally, you can configure [github-apps](github-apps.md) authentication +which carries a much higher rate limit at GitHub. + +This is true for any method of adding GitHub entities to the catalog, but +especially easy to hit with automatic discovery. + +## GitHub Processor (To Be Deprecated) + The GitHub integration has a special discovery processor for discovering catalog entities within a GitHub organization. The processor will crawl the GitHub organization and register entities matching the configured path. This can be diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index e5fa2ea46a..62e64122bb 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -40,6 +40,27 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ): Promise; } +// @public +export class GitHubEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: GitHubEntityProviderOptions, + ): GitHubEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(): Promise; +} + +// @public +export interface GitHubEntityProviderOptions { + logger: Logger; + schedule: TaskRunner; +} + // @public export type GithubMultiOrgConfig = Array<{ name: string; diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 2f666e6547..81dc1727c8 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -20,9 +20,11 @@ * @packageDocumentation */ -export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; -export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; -export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; -export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider'; -export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor'; +export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor'; +export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; +export { GitHubEntityProvider } from './providers/GitHubEntityProvider'; +export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider'; +export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider'; +export type { GitHubEntityProviderOptions } from './providers/GitHubEntityProvider'; export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index b18e3baf75..aa6dc9229e 100644 --- a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -22,7 +22,7 @@ import { } from '@backstage/integration'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; -import { getOrganizationRepositories } from './lib'; +import { getOrganizationRepositories } from '../lib'; jest.mock('./lib'); const mockGetOrganizationRepositories = diff --git a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts index 78a1bca169..97c77bf033 100644 --- a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts @@ -29,7 +29,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationRepositories } from './lib'; +import { getOrganizationRepositories } from '../lib'; /** * Extracts repositories out of a GitHub org. diff --git a/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts index 321e9cc8f8..c5315ad166 100644 --- a/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts @@ -37,7 +37,7 @@ import { getOrganizationUsers, GithubMultiOrgConfig, readGithubMultiOrgConfig, -} from './lib'; +} from '../lib'; /** * Extracts teams and users out of a multiple GitHub orgs namespaced per org. diff --git a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.test.ts similarity index 100% rename from plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.test.ts diff --git a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts index ca1dd96b3c..c6f2f5d0e2 100644 --- a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts @@ -36,7 +36,7 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './lib'; +} from '../lib'; type GraphQL = typeof graphql; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts new file mode 100644 index 0000000000..f933053f4a --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -0,0 +1,185 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { GitHubEntityProvider } from './GitHubEntityProvider'; +import * as helpers from '../lib/github'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +describe('GitHubEntityProvider', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual('github-provider:default'); + }); + + it('multiple provider configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org1', + }, + anotherProvider: { + organization: 'test-org2', + }, + }, + }, + }, + }); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'github-provider:myProvider', + ); + expect(providers[1].getProviderName()).toEqual( + 'github-provider:anotherProvider', + ); + }); + + it.only('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-21936a3d1e926b8bb3b00ac4398dc9a8dbb90b45', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts new file mode 100644 index 0000000000..5f228dff28 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -0,0 +1,271 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, + GitHubIntegrationConfig, + GitHubIntegration, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-backend'; + +import { graphql } from '@octokit/graphql'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; +import { + readProviderConfigs, + GitHubEntityProviderConfig, +} from './GitHubEntityProviderConfig'; +import { getOrganizationRepositories, Repository } from '../lib/github'; + +/** + * Options for {@link GitHubEntityProvider}. + * + * @public + */ +export interface GitHubEntityProviderOptions { + /** + * A Scheduled Task Runner + * + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule: TaskRunner; + + /** + * The logger to use. + */ + logger: Logger; +} + +/** + * Discovers catalog files located in [GitHub](https://github.com). + * The provider will search your GitHub account and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * + * @public + */ +export class GitHubEntityProvider implements EntityProvider { + private readonly config: GitHubEntityProviderConfig; + private readonly logger: Logger; + private readonly integration: GitHubIntegrationConfig; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + private readonly githubCredentialsProvider: GithubCredentialsProvider; + + static fromConfig( + config: Config, + options: GitHubEntityProviderOptions, + ): GitHubEntityProvider[] { + const integrations = ScmIntegrations.fromConfig(config); + const integration = integrations.github.byHost('github.com'); + + if (!integration) { + throw new Error( + `There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`, + ); + } + + return readProviderConfigs(config).map( + providerConfig => + new GitHubEntityProvider( + providerConfig, + integration, + options.logger, + options.schedule, + ), + ); + } + + private constructor( + config: GitHubEntityProviderConfig, + integration: GitHubIntegration, + logger: Logger, + schedule: TaskRunner, + ) { + this.config = config; + this.integration = integration.config; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(schedule); + this.githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integration.config); + this.scheduleFn = this.createScheduleFn(schedule); + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `github-provider:${this.config.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + return await this.scheduleFn(); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + taskId, + taskInstanceId: uuid.v4(), + }); + try { + await this.refresh(); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + async refresh() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const targets = await this.findCatalogFiles(); + const matchingTargets = this.matchesFilters(targets); + const entities = matchingTargets + .map(repository => this.createLocationUrl(repository)) + .map(GitHubEntityProvider.toLocationSpec) + .map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }); + + await this.connection.applyMutation({ + type: 'full', + entities, + }); + + this.logger.info( + `Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`, + ); + } + + // go to the server and get all of the repositories + private async findCatalogFiles(): Promise { + const organization = this.config.organization; + const host = this.integration.host; + const orgUrl = `https://${host}/${organization}`; + + const { headers } = await this.githubCredentialsProvider.getCredentials({ + url: orgUrl, + }); + + const client = graphql.defaults({ + baseUrl: this.integration.apiBaseUrl, + headers, + }); + + const { repositories } = await getOrganizationRepositories( + client, + organization, + ); + + return repositories; + } + + private matchesFilters(repositories: Repository[]) { + const repositoryFilter = this.config.filters?.repository; + + const matchingRepositories = repositories.filter(r => { + return ( + !r.isArchived && + repositoryFilter?.test(r.name) && + r.defaultBranchRef?.name + ); + }); + return matchingRepositories; + } + + private createLocationUrl(repository: Repository): string { + const branch = + this.config.filters?.branch || repository.defaultBranchRef?.name || '-'; + const catalogFile = this.config.catalogPath.substring( + this.config.catalogPath.lastIndexOf('/') + 1, + ); + return `${repository.url}/blob/${branch}/${catalogFile}`; + } + + private static toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'optional', + }; + } +} + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + org: string; + repoSearchPath: RegExp; + catalogPath: string; + branch: string; + host: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + // can also be + // /backstage + if (path.length > 2 && path[0].length && path[1].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + branch: decodeURIComponent(path[3]), + host: url.host, + }; + } else if (path.length === 1 && path[0].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp('*'), + catalogPath: '/catalog-info.yaml', + branch: '-', + host: url.host, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts new file mode 100644 index 0000000000..ba72dbf950 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './GitHubEntityProviderConfig'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('default'); + expect(providerConfigs[0].organization).toEqual('test-org'); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + providerOrganizationOnly: { + organization: 'test-org1', + }, + providerCustomCatalogPath: { + organization: 'test-org2', + catalogPath: 'custom/path/catalog-info.yaml', + }, + providerWithRepositoryFilter: { + organization: 'test-org3', + filters: { + repository: 'repository.*filter', + }, + }, + providerWithBranchFilter: { + organization: 'test-org4', + filters: { + branch: 'branch-name', + }, + }, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(4); + expect(providerConfigs[0]).toEqual({ + id: 'providerOrganizationOnly', + organization: 'test-org1', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + }, + }); + expect(providerConfigs[1]).toEqual({ + id: 'providerCustomCatalogPath', + organization: 'test-org2', + catalogPath: 'custom/path/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + }, + }); + expect(providerConfigs[2]).toEqual({ + id: 'providerWithRepositoryFilter', + organization: 'test-org3', // organization + catalogPath: '/catalog-info.yaml', // file + filters: { + repository: /^repository.*filter$/, // repo + branch: undefined, // branch + }, + }); + expect(providerConfigs[3]).toEqual({ + id: 'providerWithBranchFilter', + organization: 'test-org4', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: 'branch-name', + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts new file mode 100644 index 0000000000..b5929c5300 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; +const DEFAULT_PROVIDER_ID = 'default'; + +export type GitHubEntityProviderConfig = { + id: string; + catalogPath: string; + organization: string; + filters?: { + repository?: RegExp; + branch?: string; + }; +}; + +export function readProviderConfigs( + config: Config, +): GitHubEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig('catalog.providers.github'); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('organization')) { + // simple/single config variant + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + const providerConfig = providersConfig.getConfig(id); + + return readProviderConfig(id, providerConfig); + }); +} + +function readProviderConfig( + id: string, + config: Config, +): GitHubEntityProviderConfig { + const organization = config.getString('organization'); + const catalogPath = + config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; + const repositoryPattern = config.getOptionalString('filters.repository'); + const branchPattern = config.getOptionalString('filters.branch'); + + return { + id, + catalogPath, + organization, + filters: { + repository: repositoryPattern + ? compileRegExp(repositoryPattern) + : undefined, + branch: branchPattern || undefined, + }, + }; +} +/** + * Compiles a RegExp while enforcing the pattern to contain + * the start-of-line and end-of-line anchors. + * + * @param pattern + */ +function compileRegExp(pattern: string): RegExp { + let fullLinePattern = pattern; + if (!fullLinePattern.startsWith('^')) { + fullLinePattern = `^${fullLinePattern}`; + } + if (!fullLinePattern.endsWith('$')) { + fullLinePattern = `${fullLinePattern}$`; + } + + return new RegExp(fullLinePattern); +} diff --git a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts similarity index 100% rename from plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts diff --git a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts index e6cb413a6f..bb994c193c 100644 --- a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts @@ -42,7 +42,7 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './lib'; +} from '../lib'; /** * Options for {@link GitHubOrgEntityProvider}.