From f48950e34b14606b12964ed24fbdedde5d6b1ffd Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:00:53 -0400 Subject: [PATCH 1/8] feat: github discovery provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .changeset/odd-tomatoes-juggle.md | 9 + docs/integrations/github/discovery.md | 114 ++++++++ .../api-report.md | 21 ++ .../src/index.ts | 12 +- .../GithubDiscoveryProcessor.test.ts | 2 +- .../GithubDiscoveryProcessor.ts | 2 +- .../GithubMultiOrgReaderProcessor.ts | 2 +- .../GithubOrgReaderProcessor.test.ts | 0 .../GithubOrgReaderProcessor.ts | 2 +- .../providers/GitHubEntityProvider.test.ts | 185 ++++++++++++ .../src/providers/GitHubEntityProvider.ts | 271 ++++++++++++++++++ .../GitHubEntityProviderConfig.test.ts | 115 ++++++++ .../providers/GitHubEntityProviderConfig.ts | 90 ++++++ .../GitHubOrgEntityProvider.test.ts | 0 .../GitHubOrgEntityProvider.ts | 2 +- 15 files changed, 817 insertions(+), 10 deletions(-) create mode 100644 .changeset/odd-tomatoes-juggle.md rename plugins/catalog-backend-module-github/src/{ => processors}/GithubDiscoveryProcessor.test.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubDiscoveryProcessor.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubMultiOrgReaderProcessor.ts (99%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubOrgReaderProcessor.test.ts (100%) rename plugins/catalog-backend-module-github/src/{ => processors}/GithubOrgReaderProcessor.ts (99%) create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts create mode 100644 plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts rename plugins/catalog-backend-module-github/src/{ => providers}/GitHubOrgEntityProvider.test.ts (100%) rename plugins/catalog-backend-module-github/src/{ => providers}/GitHubOrgEntityProvider.ts (99%) diff --git a/.changeset/odd-tomatoes-juggle.md b/.changeset/odd-tomatoes-juggle.md new file mode 100644 index 0000000000..336af17eb4 --- /dev/null +++ b/.changeset/odd-tomatoes-juggle.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Github Entity Provider functionality for adding entities to the catalog. + +This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities. + +More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page. diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index a0c3fed153..82abc25eca 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -6,6 +6,120 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from repositories in a GitHub organization --- +## GitHub Provider + +The GitHub integration has a discovery provider for discovering catalog +entities within a GitHub organization. The provider will crawl the GitHub +organization and register entities matching the configured path. This can be +useful as an alternative to static locations or manually adding things to the +catalog. This is the prefered method for ingesting entities into the catalog. + +## Installation + +You will have to add the provider in the catalog initialization code of your +backend. They are not installed by default, therefore you have to add a +dependency on `@backstage/plugin-catalog-backend-module-github` to your backend +package + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github +``` + +And then add the entity provider to your catalog builder: + +```diff + // In packages/backend/src/plugins/catalog.ts ++ import { GitHubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ builder.addEntityProvider( ++ GitHubEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ }), ++ ); + + // [...] + } +``` + +## Configuration + +To use the discovery provider, you'll need a GitHub integration +[set up](locations.md) with either a [Personal Access Token](../../getting-started/configuration.md#setting-up-a-github-integration) or [GitHub Apps](./github-apps.md). + +Then you can add a github config to the catalog providers configuration: + +```yaml +catalog: + providers: + github: + # the provider ID can be any camelCase string + providerId: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + customProviderId: + organization: 'new-org' # string + catalogPath: '/custom/path/catalog-info.yaml' # string + filters: # optional filters + branch: 'develop' # optional string + repository: '.*' # optional Regex +``` + +This provider supports multiple organizations via unique provider IDs + +> **Note:** It is possible but certainly not recommended to skip the provider ID level. +> If you do so, `default` will be used as provider ID. + +- **catalogPath** _(optional)_: + Default: `/catalog-info.yaml`. + Path where to look for `catalog-info.yaml` files. + When started with `/`, it is an absolute path from the repo root. +- **filters** _(optional)_: + - **branch** _(optional)_: + String used to filter results based on the branch name. + - **repository** _(optional)_: + Regular expression used to filter results based on the repository name. +- **organization**: + Name of your organization account/workspace. + If you want to add multiple organizations, you need to add one provider config each. + +## GitHub API Rate Limits + +GitHub [rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) API requests to 5,000 per hour (or more for Enterprise +accounts). The default Backstage catalog backend refreshes data every 100 +seconds, which issues an API request for each discovered location. + +This means if you have more than ~140 catalog entities, you may get throttled by +rate limiting. You can change the refresh frequency of the catalog in your `packages/backend/src/plugins/catalog.ts` file: + +```typescript +schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 35 }, + timeout: { minutes: 30 }, +}), +``` + +More information about scheduling can be found on the [TaskScheduleDefinition](https://backstage.io/docs/reference/backend-tasks.taskscheduledefinition) page. + +Alternatively, or additionally, you can configure [github-apps](github-apps.md) authentication +which carries a much higher rate limit at GitHub. + +This is true for any method of adding GitHub entities to the catalog, but +especially easy to hit with automatic discovery. + +## GitHub Processor (To Be Deprecated) + The GitHub integration has a special discovery processor for discovering catalog entities within a GitHub organization. The processor will crawl the GitHub organization and register entities matching the configured path. This can be diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index e5fa2ea46a..62e64122bb 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -40,6 +40,27 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ): Promise; } +// @public +export class GitHubEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: GitHubEntityProviderOptions, + ): GitHubEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(): Promise; +} + +// @public +export interface GitHubEntityProviderOptions { + logger: Logger; + schedule: TaskRunner; +} + // @public export type GithubMultiOrgConfig = Array<{ name: string; diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 2f666e6547..81dc1727c8 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -20,9 +20,11 @@ * @packageDocumentation */ -export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; -export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; -export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; -export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider'; -export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor'; +export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor'; +export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; +export { GitHubEntityProvider } from './providers/GitHubEntityProvider'; +export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider'; +export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider'; +export type { GitHubEntityProviderOptions } from './providers/GitHubEntityProvider'; export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index b18e3baf75..aa6dc9229e 100644 --- a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -22,7 +22,7 @@ import { } from '@backstage/integration'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; -import { getOrganizationRepositories } from './lib'; +import { getOrganizationRepositories } from '../lib'; jest.mock('./lib'); const mockGetOrganizationRepositories = diff --git a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts index 78a1bca169..97c77bf033 100644 --- a/plugins/catalog-backend-module-github/src/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts @@ -29,7 +29,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationRepositories } from './lib'; +import { getOrganizationRepositories } from '../lib'; /** * Extracts repositories out of a GitHub org. diff --git a/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts index 321e9cc8f8..c5315ad166 100644 --- a/plugins/catalog-backend-module-github/src/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts @@ -37,7 +37,7 @@ import { getOrganizationUsers, GithubMultiOrgConfig, readGithubMultiOrgConfig, -} from './lib'; +} from '../lib'; /** * Extracts teams and users out of a multiple GitHub orgs namespaced per org. diff --git a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.test.ts similarity index 100% rename from plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.test.ts diff --git a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts index ca1dd96b3c..c6f2f5d0e2 100644 --- a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts @@ -36,7 +36,7 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './lib'; +} from '../lib'; type GraphQL = typeof graphql; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts new file mode 100644 index 0000000000..f933053f4a --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -0,0 +1,185 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { GitHubEntityProvider } from './GitHubEntityProvider'; +import * as helpers from '../lib/github'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +describe('GitHubEntityProvider', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({}); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(0); + }); + + it('single simple provider config', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(1); + expect(providers[0].getProviderName()).toEqual('github-provider:default'); + }); + + it('multiple provider configs', () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org1', + }, + anotherProvider: { + organization: 'test-org2', + }, + }, + }, + }, + }); + const providers = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + }); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toEqual( + 'github-provider:myProvider', + ); + expect(providers[1].getProviderName()).toEqual( + 'github-provider:anotherProvider', + ); + }); + + it.only('apply full update on scheduled execution', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'custom/path/catalog-custom.yaml', + filters: { + branch: 'main', + repository: 'test-.*', + }, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/test-repo/blob/main/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-21936a3d1e926b8bb3b00ac4398dc9a8dbb90b45', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toBeCalledTimes(1); + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts new file mode 100644 index 0000000000..5f228dff28 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -0,0 +1,271 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, + GitHubIntegrationConfig, + GitHubIntegration, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-backend'; + +import { graphql } from '@octokit/graphql'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; +import { + readProviderConfigs, + GitHubEntityProviderConfig, +} from './GitHubEntityProviderConfig'; +import { getOrganizationRepositories, Repository } from '../lib/github'; + +/** + * Options for {@link GitHubEntityProvider}. + * + * @public + */ +export interface GitHubEntityProviderOptions { + /** + * A Scheduled Task Runner + * + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule: TaskRunner; + + /** + * The logger to use. + */ + logger: Logger; +} + +/** + * Discovers catalog files located in [GitHub](https://github.com). + * The provider will search your GitHub account and register catalog files matching the configured path + * as Location entity and via following processing steps add all contained catalog entities. + * This can be useful as an alternative to static locations or manually adding things to the catalog. + * + * @public + */ +export class GitHubEntityProvider implements EntityProvider { + private readonly config: GitHubEntityProviderConfig; + private readonly logger: Logger; + private readonly integration: GitHubIntegrationConfig; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + private readonly githubCredentialsProvider: GithubCredentialsProvider; + + static fromConfig( + config: Config, + options: GitHubEntityProviderOptions, + ): GitHubEntityProvider[] { + const integrations = ScmIntegrations.fromConfig(config); + const integration = integrations.github.byHost('github.com'); + + if (!integration) { + throw new Error( + `There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`, + ); + } + + return readProviderConfigs(config).map( + providerConfig => + new GitHubEntityProvider( + providerConfig, + integration, + options.logger, + options.schedule, + ), + ); + } + + private constructor( + config: GitHubEntityProviderConfig, + integration: GitHubIntegration, + logger: Logger, + schedule: TaskRunner, + ) { + this.config = config; + this.integration = integration.config; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(schedule); + this.githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integration.config); + this.scheduleFn = this.createScheduleFn(schedule); + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `github-provider:${this.config.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + return await this.scheduleFn(); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + taskId, + taskInstanceId: uuid.v4(), + }); + try { + await this.refresh(); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + async refresh() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const targets = await this.findCatalogFiles(); + const matchingTargets = this.matchesFilters(targets); + const entities = matchingTargets + .map(repository => this.createLocationUrl(repository)) + .map(GitHubEntityProvider.toLocationSpec) + .map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }); + + await this.connection.applyMutation({ + type: 'full', + entities, + }); + + this.logger.info( + `Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`, + ); + } + + // go to the server and get all of the repositories + private async findCatalogFiles(): Promise { + const organization = this.config.organization; + const host = this.integration.host; + const orgUrl = `https://${host}/${organization}`; + + const { headers } = await this.githubCredentialsProvider.getCredentials({ + url: orgUrl, + }); + + const client = graphql.defaults({ + baseUrl: this.integration.apiBaseUrl, + headers, + }); + + const { repositories } = await getOrganizationRepositories( + client, + organization, + ); + + return repositories; + } + + private matchesFilters(repositories: Repository[]) { + const repositoryFilter = this.config.filters?.repository; + + const matchingRepositories = repositories.filter(r => { + return ( + !r.isArchived && + repositoryFilter?.test(r.name) && + r.defaultBranchRef?.name + ); + }); + return matchingRepositories; + } + + private createLocationUrl(repository: Repository): string { + const branch = + this.config.filters?.branch || repository.defaultBranchRef?.name || '-'; + const catalogFile = this.config.catalogPath.substring( + this.config.catalogPath.lastIndexOf('/') + 1, + ); + return `${repository.url}/blob/${branch}/${catalogFile}`; + } + + private static toLocationSpec(target: string): LocationSpec { + return { + type: 'url', + target: target, + presence: 'optional', + }; + } +} + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + org: string; + repoSearchPath: RegExp; + catalogPath: string; + branch: string; + host: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + // can also be + // /backstage + if (path.length > 2 && path[0].length && path[1].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, + branch: decodeURIComponent(path[3]), + host: url.host, + }; + } else if (path.length === 1 && path[0].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp('*'), + catalogPath: '/catalog-info.yaml', + branch: '-', + host: url.host, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts new file mode 100644 index 0000000000..ba72dbf950 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './GitHubEntityProviderConfig'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('default'); + expect(providerConfigs[0].organization).toEqual('test-org'); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + providerOrganizationOnly: { + organization: 'test-org1', + }, + providerCustomCatalogPath: { + organization: 'test-org2', + catalogPath: 'custom/path/catalog-info.yaml', + }, + providerWithRepositoryFilter: { + organization: 'test-org3', + filters: { + repository: 'repository.*filter', + }, + }, + providerWithBranchFilter: { + organization: 'test-org4', + filters: { + branch: 'branch-name', + }, + }, + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(4); + expect(providerConfigs[0]).toEqual({ + id: 'providerOrganizationOnly', + organization: 'test-org1', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + }, + }); + expect(providerConfigs[1]).toEqual({ + id: 'providerCustomCatalogPath', + organization: 'test-org2', + catalogPath: 'custom/path/catalog-info.yaml', + filters: { + repository: undefined, + branch: undefined, + }, + }); + expect(providerConfigs[2]).toEqual({ + id: 'providerWithRepositoryFilter', + organization: 'test-org3', // organization + catalogPath: '/catalog-info.yaml', // file + filters: { + repository: /^repository.*filter$/, // repo + branch: undefined, // branch + }, + }); + expect(providerConfigs[3]).toEqual({ + id: 'providerWithBranchFilter', + organization: 'test-org4', + catalogPath: '/catalog-info.yaml', + filters: { + repository: undefined, + branch: 'branch-name', + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts new file mode 100644 index 0000000000..b5929c5300 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const DEFAULT_CATALOG_PATH = '/catalog-info.yaml'; +const DEFAULT_PROVIDER_ID = 'default'; + +export type GitHubEntityProviderConfig = { + id: string; + catalogPath: string; + organization: string; + filters?: { + repository?: RegExp; + branch?: string; + }; +}; + +export function readProviderConfigs( + config: Config, +): GitHubEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig('catalog.providers.github'); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('organization')) { + // simple/single config variant + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + const providerConfig = providersConfig.getConfig(id); + + return readProviderConfig(id, providerConfig); + }); +} + +function readProviderConfig( + id: string, + config: Config, +): GitHubEntityProviderConfig { + const organization = config.getString('organization'); + const catalogPath = + config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; + const repositoryPattern = config.getOptionalString('filters.repository'); + const branchPattern = config.getOptionalString('filters.branch'); + + return { + id, + catalogPath, + organization, + filters: { + repository: repositoryPattern + ? compileRegExp(repositoryPattern) + : undefined, + branch: branchPattern || undefined, + }, + }; +} +/** + * Compiles a RegExp while enforcing the pattern to contain + * the start-of-line and end-of-line anchors. + * + * @param pattern + */ +function compileRegExp(pattern: string): RegExp { + let fullLinePattern = pattern; + if (!fullLinePattern.startsWith('^')) { + fullLinePattern = `^${fullLinePattern}`; + } + if (!fullLinePattern.endsWith('$')) { + fullLinePattern = `${fullLinePattern}$`; + } + + return new RegExp(fullLinePattern); +} diff --git a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts similarity index 100% rename from plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.test.ts diff --git a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts similarity index 99% rename from plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts index e6cb413a6f..bb994c193c 100644 --- a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts @@ -42,7 +42,7 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './lib'; +} from '../lib'; /** * Options for {@link GitHubOrgEntityProvider}. From 7a6007d808990814688b5f58759b34c70d77a06a Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:27:38 -0400 Subject: [PATCH 2/8] feat: github discovery provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .../src/processors/GithubDiscoveryProcessor.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index aa6dc9229e..7ce6f3f9cf 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -24,7 +24,7 @@ import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from '../lib'; -jest.mock('./lib'); +jest.mock('../lib'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories From 9328f4b349332d9e580657c3475ea7699872af8c Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:34:54 -0400 Subject: [PATCH 3/8] feat: github discovery provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .../src/providers/GitHubEntityProvider.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index f933053f4a..0bda258b6e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -100,7 +100,7 @@ describe('GitHubEntityProvider', () => { ); }); - it.only('apply full update on scheduled execution', async () => { + it('apply full update on scheduled execution', async () => { const config = new ConfigReader({ catalog: { providers: { From ae09b7855161b0a70474e6134ef81ff9e9a9b141 Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Mon, 25 Jul 2022 14:37:22 -0400 Subject: [PATCH 4/8] chore: requested update to docs Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- docs/integrations/github/discovery.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 82abc25eca..dba0907492 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -19,7 +19,7 @@ catalog. This is the prefered method for ingesting entities into the catalog. You will have to add the provider in the catalog initialization code of your backend. They are not installed by default, therefore you have to add a dependency on `@backstage/plugin-catalog-backend-module-github` to your backend -package +package. ```bash # From your Backstage root directory @@ -76,31 +76,34 @@ catalog: repository: '.*' # optional Regex ``` -This provider supports multiple organizations via unique provider IDs +This provider supports multiple organizations via unique provider IDs. > **Note:** It is possible but certainly not recommended to skip the provider ID level. > If you do so, `default` will be used as provider ID. -- **catalogPath** _(optional)_: +``` +catalogPath (optional): Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. When started with `/`, it is an absolute path from the repo root. -- **filters** _(optional)_: - - **branch** _(optional)_: + +filters (optional): + - branch (optional): String used to filter results based on the branch name. - - **repository** _(optional)_: + - repository (optional): Regular expression used to filter results based on the repository name. -- **organization**: + +organization: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. +``` ## GitHub API Rate Limits GitHub [rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) API requests to 5,000 per hour (or more for Enterprise -accounts). The default Backstage catalog backend refreshes data every 100 -seconds, which issues an API request for each discovered location. +accounts). The snippet below refreshes the Backstage catalog data every 35 minutes, which issues an API request for each discovered location. -This means if you have more than ~140 catalog entities, you may get throttled by +If your requests are too frequent then you may get throttled by rate limiting. You can change the refresh frequency of the catalog in your `packages/backend/src/plugins/catalog.ts` file: ```typescript From 1c7e940b39053300f9dc689e9d24c603e1a7c8b3 Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Tue, 26 Jul 2022 08:31:51 -0400 Subject: [PATCH 5/8] chore: requested update to docs Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- docs/integrations/github/discovery.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index dba0907492..3b49644613 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -81,22 +81,18 @@ This provider supports multiple organizations via unique provider IDs. > **Note:** It is possible but certainly not recommended to skip the provider ID level. > If you do so, `default` will be used as provider ID. -``` -catalogPath (optional): +- **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. When started with `/`, it is an absolute path from the repo root. - -filters (optional): - - branch (optional): +- **filters** _(optional)_: + - **branch** _(optional)_: String used to filter results based on the branch name. - - repository (optional): + - **repository** _(optional)_: Regular expression used to filter results based on the repository name. - -organization: +- **organization**: Name of your organization account/workspace. If you want to add multiple organizations, you need to add one provider config each. -``` ## GitHub API Rate Limits From 7a4b052d8733492eb622ab108c7eddc3077a3401 Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Thu, 28 Jul 2022 11:05:48 -0400 Subject: [PATCH 6/8] chore: removed unnecessary type declaration from entity provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .../api-report.md | 13 +++----- .../src/index.ts | 1 - .../src/providers/GitHubEntityProvider.ts | 31 +++++-------------- 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 62e64122bb..3e4c2cb06b 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -47,18 +47,15 @@ export class GitHubEntityProvider implements EntityProvider { // (undocumented) static fromConfig( config: Config, - options: GitHubEntityProviderOptions, + options: { + logger: Logger; + schedule: TaskRunner; + }, ): GitHubEntityProvider[]; // (undocumented) getProviderName(): string; // (undocumented) - refresh(): Promise; -} - -// @public -export interface GitHubEntityProviderOptions { - logger: Logger; - schedule: TaskRunner; + refresh(logger: Logger): Promise; } // @public diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 81dc1727c8..22a1167d48 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -26,5 +26,4 @@ export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor' export { GitHubEntityProvider } from './providers/GitHubEntityProvider'; export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider'; export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider'; -export type { GitHubEntityProviderOptions } from './providers/GitHubEntityProvider'; export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 5f228dff28..ea2d9c8820 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -39,26 +39,6 @@ import { } from './GitHubEntityProviderConfig'; import { getOrganizationRepositories, Repository } from '../lib/github'; -/** - * Options for {@link GitHubEntityProvider}. - * - * @public - */ -export interface GitHubEntityProviderOptions { - /** - * A Scheduled Task Runner - * - * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} - * to enable automatic scheduling of tasks. - */ - schedule: TaskRunner; - - /** - * The logger to use. - */ - logger: Logger; -} - /** * Discovers catalog files located in [GitHub](https://github.com). * The provider will search your GitHub account and register catalog files matching the configured path @@ -77,7 +57,10 @@ export class GitHubEntityProvider implements EntityProvider { static fromConfig( config: Config, - options: GitHubEntityProviderOptions, + options: { + logger: Logger; + schedule: TaskRunner; + }, ): GitHubEntityProvider[] { const integrations = ScmIntegrations.fromConfig(config); const integration = integrations.github.byHost('github.com'); @@ -138,7 +121,7 @@ export class GitHubEntityProvider implements EntityProvider { taskInstanceId: uuid.v4(), }); try { - await this.refresh(); + await this.refresh(logger); } catch (error) { logger.error(error); } @@ -147,7 +130,7 @@ export class GitHubEntityProvider implements EntityProvider { }; } - async refresh() { + async refresh(logger: Logger) { if (!this.connection) { throw new Error('Not initialized'); } @@ -169,7 +152,7 @@ export class GitHubEntityProvider implements EntityProvider { entities, }); - this.logger.info( + logger.info( `Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`, ); } From ddae16639ad6bab188a6d9cbb72e947a01d04b7e Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Thu, 28 Jul 2022 11:15:42 -0400 Subject: [PATCH 7/8] chore: removed unnecessary type declaration from entity provider Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .../src/providers/GitHubEntityProvider.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index ea2d9c8820..4a82b37909 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -117,6 +117,7 @@ export class GitHubEntityProvider implements EntityProvider { id: taskId, fn: async () => { const logger = this.logger.child({ + class: GitHubEntityProvider.prototype.constructor.name, taskId, taskInstanceId: uuid.v4(), }); From 149dc12367848eb3e53e92a026fb94581dc8cd3b Mon Sep 17 00:00:00 2001 From: brentg-telus <50498366+brentg-telus@users.noreply.github.com> Date: Thu, 28 Jul 2022 11:43:23 -0400 Subject: [PATCH 8/8] chore: added filtering exception for optional repository filter Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com> --- .../src/providers/GitHubEntityProvider.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 4a82b37909..43c01e8f7c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -185,11 +185,9 @@ export class GitHubEntityProvider implements EntityProvider { const repositoryFilter = this.config.filters?.repository; const matchingRepositories = repositories.filter(r => { - return ( - !r.isArchived && - repositoryFilter?.test(r.name) && - r.defaultBranchRef?.name - ); + return !r.isArchived && repositoryFilter + ? repositoryFilter?.test(r.name) + : true && r.defaultBranchRef?.name; }); return matchingRepositories; }