Merge pull request #11249 from backstage/freben/github-org

update docs for github org ingestion, and add scheduling to the provider

plugins/catalog-backend-module-github/api-report.md

@@ -13,6 +13,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
 import { ScmIntegrationRegistry } from '@backstage/integration';
+import { TaskRunner } from '@backstage/backend-tasks';
 
 // @public
 export class GithubDiscoveryProcessor implements CatalogProcessor {
@@ -72,7 +73,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
-// @public (undocumented)
+// @public
 export class GitHubOrgEntityProvider implements EntityProvider {
   constructor(options: {
     id: string;
@@ -86,17 +87,20 @@ export class GitHubOrgEntityProvider implements EntityProvider {
   // (undocumented)
   static fromConfig(
     config: Config,
-    options: {
-      id: string;
-      orgUrl: string;
-      logger: Logger;
-      githubCredentialsProvider?: GithubCredentialsProvider;
-    },
+    options: GitHubOrgEntityProviderOptions,
   ): GitHubOrgEntityProvider;
   // (undocumented)
   getProviderName(): string;
-  // (undocumented)
-  read(): Promise<void>;
+  read(options?: { logger?: Logger }): Promise<void>;
+}
+
+// @public
+export interface GitHubOrgEntityProviderOptions {
+  githubCredentialsProvider?: GithubCredentialsProvider;
+  id: string;
+  logger: Logger;
+  orgUrl: string;
+  schedule?: 'manual' | TaskRunner;
 }
 
 // @public

plugins/catalog-backend-module-github/package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.13.3-next.0",
+    "@backstage/backend-tasks": "^0.3.1-next.0",
     "@backstage/catalog-model": "^1.0.1",
     "@backstage/config": "^1.0.0",
     "@backstage/errors": "^1.0.0",
@@ -44,6 +45,7 @@
     "lodash": "^4.17.21",
     "msw": "^0.35.0",
     "node-fetch": "^2.6.7",
+    "uuid": "^8.0.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {

plugins/catalog-backend-module-github/src/GitHubOrgEntityProvider.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { TaskRunner } from '@backstage/backend-tasks';
 import {
   ANNOTATION_LOCATION,
   ANNOTATION_ORIGIN_LOCATION,
@@ -33,6 +34,7 @@ import {
 } from '@backstage/plugin-catalog-backend';
 import { graphql } from '@octokit/graphql';
 import { merge } from 'lodash';
+import * as uuid from 'uuid';
 import { Logger } from 'winston';
 import {
   assignGroupsToUsers,
@@ -42,21 +44,64 @@ import {
   parseGitHubOrgUrl,
 } from './lib';
 
-// TODO: Consider supporting an (optional) webhook that reacts on org changes
-/** @public */
-export class GitHubOrgEntityProvider implements EntityProvider {
-  private connection?: EntityProviderConnection;
-  private githubCredentialsProvider: GithubCredentialsProvider;
+/**
+ * Options for {@link GitHubOrgEntityProvider}.
+ *
+ * @public
+ */
+export interface GitHubOrgEntityProviderOptions {
+  /**
+   * A unique, stable identifier for this provider.
+   *
+   * @example "production"
+   */
+  id: string;
 
-  static fromConfig(
-    config: Config,
-    options: {
-      id: string;
-      orgUrl: string;
-      logger: Logger;
-      githubCredentialsProvider?: GithubCredentialsProvider;
-    },
-  ) {
+  /**
+   * The target that this provider should consume.
+   *
+   * @example "https://github.com/backstage"
+   */
+  orgUrl: string;
+
+  /**
+   * The refresh schedule to use.
+   *
+   * @defaultValue "manual"
+   * @remarks
+   *
+   * If you pass in 'manual', you are responsible for calling the `read` method
+   * manually at some interval.
+   *
+   * But more commonly you will pass in the result of
+   * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+   * to enable automatic scheduling of tasks.
+   */
+  schedule?: 'manual' | TaskRunner;
+
+  /**
+   * The logger to use.
+   */
+  logger: Logger;
+
+  /**
+   * Optionally supply a custom credentials provider, replacing the default one.
+   */
+  githubCredentialsProvider?: GithubCredentialsProvider;
+}
+
+// TODO: Consider supporting an (optional) webhook that reacts on org changes
+/**
+ * Ingests org data (users and groups) from GitHub.
+ *
+ * @public
+ */
+export class GitHubOrgEntityProvider implements EntityProvider {
+  private readonly credentialsProvider: GithubCredentialsProvider;
+  private connection?: EntityProviderConnection;
+  private scheduleFn?: () => Promise<void>;
+
+  static fromConfig(config: Config, options: GitHubOrgEntityProviderOptions) {
     const integrations = ScmIntegrations.fromConfig(config);
     const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config;
 
@@ -70,7 +115,7 @@ export class GitHubOrgEntityProvider implements EntityProvider {
       target: options.orgUrl,
     });
 
-    return new GitHubOrgEntityProvider({
+    const provider = new GitHubOrgEntityProvider({
       id: options.id,
       orgUrl: options.orgUrl,
       logger,
@@ -79,6 +124,10 @@ export class GitHubOrgEntityProvider implements EntityProvider {
         options.githubCredentialsProvider ||
         DefaultGithubCredentialsProvider.fromIntegrations(integrations),
     });
+
+    provider.schedule(options.schedule);
+
+    return provider;
   }
 
   constructor(
@@ -90,28 +139,36 @@ export class GitHubOrgEntityProvider implements EntityProvider {
       githubCredentialsProvider?: GithubCredentialsProvider;
     },
   ) {
-    this.githubCredentialsProvider =
+    this.credentialsProvider =
       options.githubCredentialsProvider ||
-      SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig);
+      SingleInstanceGithubCredentialsProvider.create(this.options.gitHubConfig);
   }
 
+  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
   getProviderName() {
     return `GitHubOrgEntityProvider:${this.options.id}`;
   }
 
+  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
   async connect(connection: EntityProviderConnection) {
     this.connection = connection;
+    await this.scheduleFn?.();
   }
 
-  async read() {
+  /**
+   * Runs one single complete ingestion. This is only necessary if you use
+   * manual scheduling.
+   */
+  async read(options?: { logger?: Logger }) {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
 
-    const { markReadComplete } = trackProgress(this.options.logger);
+    const logger = options?.logger ?? this.options.logger;
+    const { markReadComplete } = trackProgress(logger);
 
     const { headers, type: tokenType } =
-      await this.githubCredentialsProvider.getCredentials({
+      await this.credentialsProvider.getCredentials({
         url: this.options.orgUrl,
       });
     const client = graphql.defaults({
@@ -144,6 +201,32 @@ export class GitHubOrgEntityProvider implements EntityProvider {
 
     markCommitComplete();
   }
+
+  private schedule(schedule: GitHubOrgEntityProviderOptions['schedule']) {
+    if (!schedule || schedule === 'manual') {
+      return;
+    }
+
+    this.scheduleFn = async () => {
+      const id = `${this.getProviderName()}:refresh`;
+      await schedule.run({
+        id,
+        fn: async () => {
+          const logger = this.options.logger.child({
+            class: GitHubOrgEntityProvider.prototype.constructor.name,
+            taskId: id,
+            taskInstanceId: uuid.v4(),
+          });
+
+          try {
+            await this.read({ logger });
+          } catch (error) {
+            logger.error(error);
+          }
+        },
+      });
+    };
+  }
 }
 
 // Helps wrap the timing and logging behaviors

plugins/catalog-backend-module-github/src/GithubOrgReaderProcessor.ts

@@ -42,6 +42,11 @@ type GraphQL = typeof graphql;
 
 /**
  * Extracts teams and users out of a GitHub org.
+ *
+ * @remarks
+ *
+ * Consider using {@link GitHubOrgEntityProvider} instead.
+ *
  * @public
  */
 export class GithubOrgReaderProcessor implements CatalogProcessor {

plugins/catalog-backend-module-github/src/index.ts

@@ -23,5 +23,6 @@
 export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
 export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
 export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';
+export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export type { GithubMultiOrgConfig } from './lib';