From a7de43f6484c804483cd7f5a46954c38f89edca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 3 May 2022 13:52:27 +0200 Subject: [PATCH] update docs for github org ingestion, and add scheduling to the provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/hungry-chefs-return.md | 24 ++++ docs/integrations/github/org.md | 121 ++++++++++++++--- .../api-report.md | 22 ++-- .../package.json | 2 + .../src/GitHubOrgEntityProvider.ts | 123 +++++++++++++++--- .../src/GithubOrgReaderProcessor.ts | 5 + .../src/index.ts | 1 + 7 files changed, 253 insertions(+), 45 deletions(-) create mode 100644 .changeset/hungry-chefs-return.md diff --git a/.changeset/hungry-chefs-return.md b/.changeset/hungry-chefs-return.md new file mode 100644 index 0000000000..d369b95edb --- /dev/null +++ b/.changeset/hungry-chefs-return.md @@ -0,0 +1,24 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +`GitHubOrgEntityProvider.fromConfig` now supports a `schedule` option like other +entity providers, that makes it more convenient to leverage using the common +task scheduler. + +If you want to use this in your own project, it is used something like the following: + +```ts +// In packages/backend/src/plugins/catalog.ts +builder.addEntityProvider( + GitHubOrgEntityProvider.fromConfig(env.config, { + id: 'production', + orgUrl: 'https://github.com/backstage', + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { cron: '*/30 * * * *' }, + timeout: { minutes: 10 }, + }), + logger: env.logger, + }), +); +``` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 0460b1adbc..5482733a58 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -19,22 +19,72 @@ entities that mirror your org setup. ## Installation -See the [discovery](discovery.md) article for installation instructions. +This guide will use the Entity Provider method. If you for some reason prefer +the Processor method (not recommended), it is described separately below. + +The provider is not installed by default, therefore you have to add a dependency +to `@backstage/plugin-catalog-backend-module-github` to your backend package. + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github +``` + +> Note: When configuring to use a Provider instead of a Processor you do not +> need to add a _location_ pointing to your GitHub server/organization + +Update the catalog plugin initialization in your backend to add the provider and +schedule it: + +```diff + // packages/backend/src/plugins/catalog.ts ++import { GitHubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + ++ // The org URL below needs to match a configured integrations.github entry ++ // specified in your app-config. ++ builder.addEntityProvider( ++ GitHubOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ orgUrl: 'https://github.com/backstage', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 60 }, ++ timeout: { minutes: 15 }, ++ }), ++ }), ++ ); +``` ## Configuration -The following configuration enables an import of the teams and users under the -org `https://github.com/my-org-name` on public GitHub. +As mentioned above, you also must have some configuration in your app-config +that describes the targets that you want to import. This lets the entity +provider know what authorization to use, and what the API endpoints are. You may +or may not have such an entry already added since before: ```yaml -catalog: - locations: - - type: github-org - target: https://github.com/my-org-name - rules: - - allow: [User, Group] +integrations: + github: + # example for public github + - host: github.com + token: ${GITHUB_TOKEN} + # example for a private GitHub Enterprise instance + - host: ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + token: ${GHE_TOKEN} ``` +These examples use `${}` placeholders to reference environment variables. This +is often suitable for production setups, but also means that you will have to +supply those variables to the backend as it starts up. If you want, for local +development in particular, you can experiment first by putting the actual tokens +in a mirrored config directly in your `app-config.local.yaml` as well. + If Backstage is configured to use GitHub Apps authentication you must grant `Read-Only` access for `Members` under `Organization` in order to ingest users correctly. You can modify the app's permissions under the organization settings, @@ -47,11 +97,50 @@ that must be approved first before the changes are applied.** ![email](../../assets/integrations/github/email.png) -Locations point out the specific org(s) you want to import. The `type` of these -locations must be `github-org`, and the `target` must point to the exact URL of -some organization. You can have several such location entries if needed. +## Using a Processor instead of a Provider -The authorization for loading org information comes from a configured -[GitHub integration](locations.md#configuration). When using a personal access -token, the token needs to have at least the scopes `read:org`, `read:user`, and -`user:email` in the given `target`. +An alternative to using the Provider for ingesting organizational entities is to +use a Processor. This is the old way that's based on registering locations with +the proper type and target, triggering the processor to run. + +The drawback of this method is that it will leave orphaned Group/User entities +whenever they are deleted on your GitHub server, and you cannot control the +frequency with which they are refreshed, separately from other processors. + +### Processor Installation + +The `GithubOrgReaderProcessor` is not registered by default, so you have to +install and register it in the catalog plugin: + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github +``` + +```typescript +// packages/backend/src/plugins/catalog.ts +import { GithubOrgReaderProcessor } from '@backstage/plugin-catalog-backend-module-github'; +// ... +builder.addProcessor( + GithubOrgReaderProcessor.fromConfig(env.config, { logger: env.logger }), +); +``` + +### Processor Configuration + +The integration section of your app-config needs to be set up in the same way as +for the Entity Provider - see above. + +In addition to that, you typically want to add a few static locations to your +app-config, which reference your organizations to import. The following +configuration enables an import of the teams and users under the org +`https://github.com/my-org-name` on public GitHub. + +```yaml +catalog: + locations: + - type: github-org + target: https://github.com/my-org-name + rules: + - allow: [User, Group] +``` diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index a1b3d34058..e5fa2ea46a 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -13,6 +13,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { TaskRunner } from '@backstage/backend-tasks'; // @public export class GithubDiscoveryProcessor implements CatalogProcessor { @@ -72,7 +73,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public export class GitHubOrgEntityProvider implements EntityProvider { constructor(options: { id: string; @@ -86,17 +87,20 @@ export class GitHubOrgEntityProvider implements EntityProvider { // (undocumented) static fromConfig( config: Config, - options: { - id: string; - orgUrl: string; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, + options: GitHubOrgEntityProviderOptions, ): GitHubOrgEntityProvider; // (undocumented) getProviderName(): string; - // (undocumented) - read(): Promise; + read(options?: { logger?: Logger }): Promise; +} + +// @public +export interface GitHubOrgEntityProviderOptions { + githubCredentialsProvider?: GithubCredentialsProvider; + id: string; + logger: Logger; + orgUrl: string; + schedule?: 'manual' | TaskRunner; } // @public diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index 6069834cbc..0d143f755b 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.13.3-next.0", + "@backstage/backend-tasks": "^0.3.1-next.0", "@backstage/catalog-model": "^1.0.1", "@backstage/config": "^1.0.0", "@backstage/errors": "^1.0.0", @@ -44,6 +45,7 @@ "lodash": "^4.17.21", "msw": "^0.35.0", "node-fetch": "^2.6.7", + "uuid": "^8.0.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts index 5bf501aff0..e6cb413a6f 100644 --- a/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -33,6 +34,7 @@ import { } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; +import * as uuid from 'uuid'; import { Logger } from 'winston'; import { assignGroupsToUsers, @@ -42,21 +44,64 @@ import { parseGitHubOrgUrl, } from './lib'; -// TODO: Consider supporting an (optional) webhook that reacts on org changes -/** @public */ -export class GitHubOrgEntityProvider implements EntityProvider { - private connection?: EntityProviderConnection; - private githubCredentialsProvider: GithubCredentialsProvider; +/** + * Options for {@link GitHubOrgEntityProvider}. + * + * @public + */ +export interface GitHubOrgEntityProviderOptions { + /** + * A unique, stable identifier for this provider. + * + * @example "production" + */ + id: string; - static fromConfig( - config: Config, - options: { - id: string; - orgUrl: string; - logger: Logger; - githubCredentialsProvider?: GithubCredentialsProvider; - }, - ) { + /** + * The target that this provider should consume. + * + * @example "https://github.com/backstage" + */ + orgUrl: string; + + /** + * The refresh schedule to use. + * + * @defaultValue "manual" + * @remarks + * + * If you pass in 'manual', you are responsible for calling the `read` method + * manually at some interval. + * + * But more commonly you will pass in the result of + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule?: 'manual' | TaskRunner; + + /** + * The logger to use. + */ + logger: Logger; + + /** + * Optionally supply a custom credentials provider, replacing the default one. + */ + githubCredentialsProvider?: GithubCredentialsProvider; +} + +// TODO: Consider supporting an (optional) webhook that reacts on org changes +/** + * Ingests org data (users and groups) from GitHub. + * + * @public + */ +export class GitHubOrgEntityProvider implements EntityProvider { + private readonly credentialsProvider: GithubCredentialsProvider; + private connection?: EntityProviderConnection; + private scheduleFn?: () => Promise; + + static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -70,7 +115,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { target: options.orgUrl, }); - return new GitHubOrgEntityProvider({ + const provider = new GitHubOrgEntityProvider({ id: options.id, orgUrl: options.orgUrl, logger, @@ -79,6 +124,10 @@ export class GitHubOrgEntityProvider implements EntityProvider { options.githubCredentialsProvider || DefaultGithubCredentialsProvider.fromIntegrations(integrations), }); + + provider.schedule(options.schedule); + + return provider; } constructor( @@ -90,28 +139,36 @@ export class GitHubOrgEntityProvider implements EntityProvider { githubCredentialsProvider?: GithubCredentialsProvider; }, ) { - this.githubCredentialsProvider = + this.credentialsProvider = options.githubCredentialsProvider || - SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig); + SingleInstanceGithubCredentialsProvider.create(this.options.gitHubConfig); } + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ getProviderName() { return `GitHubOrgEntityProvider:${this.options.id}`; } + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; + await this.scheduleFn?.(); } - async read() { + /** + * Runs one single complete ingestion. This is only necessary if you use + * manual scheduling. + */ + async read(options?: { logger?: Logger }) { if (!this.connection) { throw new Error('Not initialized'); } - const { markReadComplete } = trackProgress(this.options.logger); + const logger = options?.logger ?? this.options.logger; + const { markReadComplete } = trackProgress(logger); const { headers, type: tokenType } = - await this.githubCredentialsProvider.getCredentials({ + await this.credentialsProvider.getCredentials({ url: this.options.orgUrl, }); const client = graphql.defaults({ @@ -144,6 +201,32 @@ export class GitHubOrgEntityProvider implements EntityProvider { markCommitComplete(); } + + private schedule(schedule: GitHubOrgEntityProviderOptions['schedule']) { + if (!schedule || schedule === 'manual') { + return; + } + + this.scheduleFn = async () => { + const id = `${this.getProviderName()}:refresh`; + await schedule.run({ + id, + fn: async () => { + const logger = this.options.logger.child({ + class: GitHubOrgEntityProvider.prototype.constructor.name, + taskId: id, + taskInstanceId: uuid.v4(), + }); + + try { + await this.read({ logger }); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } } // Helps wrap the timing and logging behaviors diff --git a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts index 84c12e82ba..ca1dd96b3c 100644 --- a/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts @@ -42,6 +42,11 @@ type GraphQL = typeof graphql; /** * Extracts teams and users out of a GitHub org. + * + * @remarks + * + * Consider using {@link GitHubOrgEntityProvider} instead. + * * @public */ export class GithubOrgReaderProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 1394fcd21b..2f666e6547 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -23,5 +23,6 @@ export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export type { GithubMultiOrgConfig } from './lib';