feat: github discovery provider

Signed-off-by: brentg-telus <50498366+brentg-telus@users.noreply.github.com>

.changeset/odd-tomatoes-juggle.md

@@ -0,0 +1,9 @@
+---
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+Github Entity Provider functionality for adding entities to the catalog.
+
+This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities.
+
+More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page.

docs/integrations/github/discovery.md

@@ -6,6 +6,120 @@ sidebar_label: Discovery
 description: Automatically discovering catalog entities from repositories in a GitHub organization
 ---
 
+## GitHub Provider
+
+The GitHub integration has a discovery provider for discovering catalog
+entities within a GitHub organization. The provider will crawl the GitHub
+organization and register entities matching the configured path. This can be
+useful as an alternative to static locations or manually adding things to the
+catalog. This is the prefered method for ingesting entities into the catalog.
+
+## Installation
+
+You will have to add the provider in the catalog initialization code of your
+backend. They are not installed by default, therefore you have to add a
+dependency on `@backstage/plugin-catalog-backend-module-github` to your backend
+package
+
+```bash
+# From your Backstage root directory
+yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github
+```
+
+And then add the entity provider to your catalog builder:
+
+```diff
+  // In packages/backend/src/plugins/catalog.ts
++ import { GitHubEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
+
+  export default async function createPlugin(
+    env: PluginEnvironment,
+  ): Promise<Router> {
+    const builder = await CatalogBuilder.create(env);
++   builder.addEntityProvider(
++     GitHubEntityProvider.fromConfig(env.config, {
++       logger: env.logger,
++       schedule: env.scheduler.createScheduledTaskRunner({
++         frequency: { minutes: 30 },
++         timeout: { minutes: 3 },
++       }),
++     }),
++   );
+
+    // [...]
+  }
+```
+
+## Configuration
+
+To use the discovery provider, you'll need a GitHub integration
+[set up](locations.md) with either a [Personal Access Token](../../getting-started/configuration.md#setting-up-a-github-integration) or [GitHub Apps](./github-apps.md).
+
+Then you can add a github config to the catalog providers configuration:
+
+```yaml
+catalog:
+  providers:
+    github:
+      # the provider ID can be any camelCase string
+      providerId:
+        organization: 'backstage' # string
+        catalogPath: '/catalog-info.yaml' # string
+        filters:
+          branch: 'main' # string
+          repository: '.*' # Regex
+      customProviderId:
+        organization: 'new-org' # string
+        catalogPath: '/custom/path/catalog-info.yaml' # string
+        filters: # optional filters
+          branch: 'develop' # optional string
+          repository: '.*' # optional Regex
+```
+
+This provider supports multiple organizations via unique provider IDs
+
+> **Note:** It is possible but certainly not recommended to skip the provider ID level.
+> If you do so, `default` will be used as provider ID.
+
+- **catalogPath** _(optional)_:
+  Default: `/catalog-info.yaml`.
+  Path where to look for `catalog-info.yaml` files.
+  When started with `/`, it is an absolute path from the repo root.
+- **filters** _(optional)_:
+  - **branch** _(optional)_:
+    String used to filter results based on the branch name.
+  - **repository** _(optional)_:
+    Regular expression used to filter results based on the repository name.
+- **organization**:
+  Name of your organization account/workspace.
+  If you want to add multiple organizations, you need to add one provider config each.
+
+## GitHub API Rate Limits
+
+GitHub [rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) API requests to 5,000 per hour (or more for Enterprise
+accounts). The default Backstage catalog backend refreshes data every 100
+seconds, which issues an API request for each discovered location.
+
+This means if you have more than ~140 catalog entities, you may get throttled by
+rate limiting. You can change the refresh frequency of the catalog in your `packages/backend/src/plugins/catalog.ts` file:
+
+```typescript
+schedule: env.scheduler.createScheduledTaskRunner({
+  frequency: { minutes: 35 },
+  timeout: { minutes: 30 },
+}),
+```
+
+More information about scheduling can be found on the [TaskScheduleDefinition](https://backstage.io/docs/reference/backend-tasks.taskscheduledefinition) page.
+
+Alternatively, or additionally, you can configure [github-apps](github-apps.md) authentication
+which carries a much higher rate limit at GitHub.
+
+This is true for any method of adding GitHub entities to the catalog, but
+especially easy to hit with automatic discovery.
+
+## GitHub Processor (To Be Deprecated)
+
 The GitHub integration has a special discovery processor for discovering catalog
 entities within a GitHub organization. The processor will crawl the GitHub
 organization and register entities matching the configured path. This can be

plugins/catalog-backend-module-github/api-report.md

@@ -40,6 +40,27 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// @public
+export class GitHubEntityProvider implements EntityProvider {
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: GitHubEntityProviderOptions,
+  ): GitHubEntityProvider[];
+  // (undocumented)
+  getProviderName(): string;
+  // (undocumented)
+  refresh(): Promise<void>;
+}
+
+// @public
+export interface GitHubEntityProviderOptions {
+  logger: Logger;
+  schedule: TaskRunner;
+}
+
 // @public
 export type GithubMultiOrgConfig = Array<{
   name: string;

plugins/catalog-backend-module-github/src/index.ts

@@ -20,9 +20,11 @@
  * @packageDocumentation
  */
 
-export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
-export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
-export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';
-export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider';
-export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
+export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
+export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
+export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
+export { GitHubEntityProvider } from './providers/GitHubEntityProvider';
+export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider';
+export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider';
+export type { GitHubEntityProviderOptions } from './providers/GitHubEntityProvider';
 export type { GithubMultiOrgConfig } from './lib';

plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts

@@ -22,7 +22,7 @@ import {
 } from '@backstage/integration';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
-import { getOrganizationRepositories } from './lib';
+import { getOrganizationRepositories } from '../lib';
 
 jest.mock('./lib');
 const mockGetOrganizationRepositories =

plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts

@@ -29,7 +29,7 @@ import {
 } from '@backstage/plugin-catalog-backend';
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
-import { getOrganizationRepositories } from './lib';
+import { getOrganizationRepositories } from '../lib';
 
 /**
  * Extracts repositories out of a GitHub org.

plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts

@@ -37,7 +37,7 @@ import {
   getOrganizationUsers,
   GithubMultiOrgConfig,
   readGithubMultiOrgConfig,
-} from './lib';
+} from '../lib';
 
 /**
  * Extracts teams and users out of a multiple GitHub orgs namespaced per org.

plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts

@@ -36,7 +36,7 @@ import {
   getOrganizationTeams,
   getOrganizationUsers,
   parseGitHubOrgUrl,
-} from './lib';
+} from '../lib';
 
 type GraphQL = typeof graphql;
 

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts

@@ -0,0 +1,185 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
+import { ConfigReader } from '@backstage/config';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { GitHubEntityProvider } from './GitHubEntityProvider';
+import * as helpers from '../lib/github';
+
+class PersistingTaskRunner implements TaskRunner {
+  private tasks: TaskInvocationDefinition[] = [];
+
+  getTasks() {
+    return this.tasks;
+  }
+
+  run(task: TaskInvocationDefinition): Promise<void> {
+    this.tasks.push(task);
+    return Promise.resolve(undefined);
+  }
+}
+
+const logger = getVoidLogger();
+
+describe('GitHubEntityProvider', () => {
+  afterEach(() => jest.resetAllMocks());
+
+  it('no provider config', () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({});
+    const providers = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    });
+
+    expect(providers).toHaveLength(0);
+  });
+
+  it('single simple provider config', () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+    const providers = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    });
+
+    expect(providers).toHaveLength(1);
+    expect(providers[0].getProviderName()).toEqual('github-provider:default');
+  });
+
+  it('multiple provider configs', () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            myProvider: {
+              organization: 'test-org1',
+            },
+            anotherProvider: {
+              organization: 'test-org2',
+            },
+          },
+        },
+      },
+    });
+    const providers = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    });
+
+    expect(providers).toHaveLength(2);
+    expect(providers[0].getProviderName()).toEqual(
+      'github-provider:myProvider',
+    );
+    expect(providers[1].getProviderName()).toEqual(
+      'github-provider:anotherProvider',
+    );
+  });
+
+  it.only('apply full update on scheduled execution', async () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            myProvider: {
+              organization: 'test-org',
+              catalogPath: 'custom/path/catalog-custom.yaml',
+              filters: {
+                branch: 'main',
+                repository: 'test-.*',
+              },
+            },
+          },
+        },
+      },
+    });
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+    };
+
+    const provider = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const mockGetOrganizationRepositories = jest.spyOn(
+      helpers,
+      'getOrganizationRepositories',
+    );
+
+    mockGetOrganizationRepositories.mockReturnValue(
+      Promise.resolve({
+        repositories: [
+          {
+            name: 'test-repo',
+            url: 'https://github.com/test-org/test-repo',
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+          },
+        ],
+      }),
+    );
+
+    await provider.connect(entityProviderConnection);
+
+    const taskDef = schedule.getTasks()[0];
+    expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
+    await (taskDef.fn as () => Promise<void>)();
+
+    const url = `https://github.com/test-org/test-repo/blob/main/catalog-custom.yaml`;
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-21936a3d1e926b8bb3b00ac4398dc9a8dbb90b45',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:myProvider',
+      },
+    ];
+
+    expect(entityProviderConnection.applyMutation).toBeCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toBeCalledWith({
+      type: 'full',
+      entities: expectedEntities,
+    });
+  });
+});

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts

@@ -0,0 +1,271 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TaskRunner } from '@backstage/backend-tasks';
+import { Config } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  ScmIntegrations,
+  GitHubIntegrationConfig,
+  GitHubIntegration,
+  SingleInstanceGithubCredentialsProvider,
+} from '@backstage/integration';
+import {
+  EntityProvider,
+  EntityProviderConnection,
+  LocationSpec,
+  locationSpecToLocationEntity,
+} from '@backstage/plugin-catalog-backend';
+
+import { graphql } from '@octokit/graphql';
+import * as uuid from 'uuid';
+import { Logger } from 'winston';
+import {
+  readProviderConfigs,
+  GitHubEntityProviderConfig,
+} from './GitHubEntityProviderConfig';
+import { getOrganizationRepositories, Repository } from '../lib/github';
+
+/**
+ * Options for {@link GitHubEntityProvider}.
+ *
+ * @public
+ */
+export interface GitHubEntityProviderOptions {
+  /**
+   * A Scheduled Task Runner
+   *
+   * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+   * to enable automatic scheduling of tasks.
+   */
+  schedule: TaskRunner;
+
+  /**
+   * The logger to use.
+   */
+  logger: Logger;
+}
+
+/**
+ * Discovers catalog files located in [GitHub](https://github.com).
+ * The provider will search your GitHub account and register catalog files matching the configured path
+ * as Location entity and via following processing steps add all contained catalog entities.
+ * This can be useful as an alternative to static locations or manually adding things to the catalog.
+ *
+ * @public
+ */
+export class GitHubEntityProvider implements EntityProvider {
+  private readonly config: GitHubEntityProviderConfig;
+  private readonly logger: Logger;
+  private readonly integration: GitHubIntegrationConfig;
+  private readonly scheduleFn: () => Promise<void>;
+  private connection?: EntityProviderConnection;
+  private readonly githubCredentialsProvider: GithubCredentialsProvider;
+
+  static fromConfig(
+    config: Config,
+    options: GitHubEntityProviderOptions,
+  ): GitHubEntityProvider[] {
+    const integrations = ScmIntegrations.fromConfig(config);
+    const integration = integrations.github.byHost('github.com');
+
+    if (!integration) {
+      throw new Error(
+        `There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`,
+      );
+    }
+
+    return readProviderConfigs(config).map(
+      providerConfig =>
+        new GitHubEntityProvider(
+          providerConfig,
+          integration,
+          options.logger,
+          options.schedule,
+        ),
+    );
+  }
+
+  private constructor(
+    config: GitHubEntityProviderConfig,
+    integration: GitHubIntegration,
+    logger: Logger,
+    schedule: TaskRunner,
+  ) {
+    this.config = config;
+    this.integration = integration.config;
+    this.logger = logger.child({
+      target: this.getProviderName(),
+    });
+    this.scheduleFn = this.createScheduleFn(schedule);
+    this.githubCredentialsProvider =
+      SingleInstanceGithubCredentialsProvider.create(integration.config);
+    this.scheduleFn = this.createScheduleFn(schedule);
+  }
+
+  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
+  getProviderName(): string {
+    return `github-provider:${this.config.id}`;
+  }
+
+  /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
+  async connect(connection: EntityProviderConnection): Promise<void> {
+    this.connection = connection;
+    return await this.scheduleFn();
+  }
+
+  private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
+    return async () => {
+      const taskId = `${this.getProviderName()}:refresh`;
+      return schedule.run({
+        id: taskId,
+        fn: async () => {
+          const logger = this.logger.child({
+            taskId,
+            taskInstanceId: uuid.v4(),
+          });
+          try {
+            await this.refresh();
+          } catch (error) {
+            logger.error(error);
+          }
+        },
+      });
+    };
+  }
+
+  async refresh() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const targets = await this.findCatalogFiles();
+    const matchingTargets = this.matchesFilters(targets);
+    const entities = matchingTargets
+      .map(repository => this.createLocationUrl(repository))
+      .map(GitHubEntityProvider.toLocationSpec)
+      .map(location => {
+        return {
+          locationKey: this.getProviderName(),
+          entity: locationSpecToLocationEntity({ location }),
+        };
+      });
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities,
+    });
+
+    this.logger.info(
+      `Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`,
+    );
+  }
+
+  // go to the server and get all of the repositories
+  private async findCatalogFiles(): Promise<Repository[]> {
+    const organization = this.config.organization;
+    const host = this.integration.host;
+    const orgUrl = `https://${host}/${organization}`;
+
+    const { headers } = await this.githubCredentialsProvider.getCredentials({
+      url: orgUrl,
+    });
+
+    const client = graphql.defaults({
+      baseUrl: this.integration.apiBaseUrl,
+      headers,
+    });
+
+    const { repositories } = await getOrganizationRepositories(
+      client,
+      organization,
+    );
+
+    return repositories;
+  }
+
+  private matchesFilters(repositories: Repository[]) {
+    const repositoryFilter = this.config.filters?.repository;
+
+    const matchingRepositories = repositories.filter(r => {
+      return (
+        !r.isArchived &&
+        repositoryFilter?.test(r.name) &&
+        r.defaultBranchRef?.name
+      );
+    });
+    return matchingRepositories;
+  }
+
+  private createLocationUrl(repository: Repository): string {
+    const branch =
+      this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
+    const catalogFile = this.config.catalogPath.substring(
+      this.config.catalogPath.lastIndexOf('/') + 1,
+    );
+    return `${repository.url}/blob/${branch}/${catalogFile}`;
+  }
+
+  private static toLocationSpec(target: string): LocationSpec {
+    return {
+      type: 'url',
+      target: target,
+      presence: 'optional',
+    };
+  }
+}
+
+/*
+ * Helpers
+ */
+
+export function parseUrl(urlString: string): {
+  org: string;
+  repoSearchPath: RegExp;
+  catalogPath: string;
+  branch: string;
+  host: string;
+} {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // /backstage/techdocs-*/blob/master/catalog-info.yaml
+  // can also be
+  // /backstage
+  if (path.length > 2 && path[0].length && path[1].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
+      branch: decodeURIComponent(path[3]),
+      host: url.host,
+    };
+  } else if (path.length === 1 && path[0].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      repoSearchPath: escapeRegExp('*'),
+      catalogPath: '/catalog-info.yaml',
+      branch: '-',
+      host: url.host,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
+export function escapeRegExp(str: string): RegExp {
+  return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
+}

plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readProviderConfigs } from './GitHubEntityProviderConfig';
+
+describe('readProviderConfigs', () => {
+  afterEach(() => jest.resetAllMocks());
+
+  it('no provider config', () => {
+    const config = new ConfigReader({});
+    const providerConfigs = readProviderConfigs(config);
+
+    expect(providerConfigs).toHaveLength(0);
+  });
+
+  it('single simple provider config', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+    const providerConfigs = readProviderConfigs(config);
+
+    expect(providerConfigs).toHaveLength(1);
+    expect(providerConfigs[0].id).toEqual('default');
+    expect(providerConfigs[0].organization).toEqual('test-org');
+  });
+
+  it('multiple provider configs', () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            providerOrganizationOnly: {
+              organization: 'test-org1',
+            },
+            providerCustomCatalogPath: {
+              organization: 'test-org2',
+              catalogPath: 'custom/path/catalog-info.yaml',
+            },
+            providerWithRepositoryFilter: {
+              organization: 'test-org3',
+              filters: {
+                repository: 'repository.*filter',
+              },
+            },
+            providerWithBranchFilter: {
+              organization: 'test-org4',
+              filters: {
+                branch: 'branch-name',
+              },
+            },
+          },
+        },
+      },
+    });
+    const providerConfigs = readProviderConfigs(config);
+
+    expect(providerConfigs).toHaveLength(4);
+    expect(providerConfigs[0]).toEqual({
+      id: 'providerOrganizationOnly',
+      organization: 'test-org1',
+      catalogPath: '/catalog-info.yaml',
+      filters: {
+        repository: undefined,
+        branch: undefined,
+      },
+    });
+    expect(providerConfigs[1]).toEqual({
+      id: 'providerCustomCatalogPath',
+      organization: 'test-org2',
+      catalogPath: 'custom/path/catalog-info.yaml',
+      filters: {
+        repository: undefined,
+        branch: undefined,
+      },
+    });
+    expect(providerConfigs[2]).toEqual({
+      id: 'providerWithRepositoryFilter',
+      organization: 'test-org3', // organization
+      catalogPath: '/catalog-info.yaml', // file
+      filters: {
+        repository: /^repository.*filter$/, // repo
+        branch: undefined, // branch
+      },
+    });
+    expect(providerConfigs[3]).toEqual({
+      id: 'providerWithBranchFilter',
+      organization: 'test-org4',
+      catalogPath: '/catalog-info.yaml',
+      filters: {
+        repository: undefined,
+        branch: 'branch-name',
+      },
+    });
+  });
+});

plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
+const DEFAULT_PROVIDER_ID = 'default';
+
+export type GitHubEntityProviderConfig = {
+  id: string;
+  catalogPath: string;
+  organization: string;
+  filters?: {
+    repository?: RegExp;
+    branch?: string;
+  };
+};
+
+export function readProviderConfigs(
+  config: Config,
+): GitHubEntityProviderConfig[] {
+  const providersConfig = config.getOptionalConfig('catalog.providers.github');
+  if (!providersConfig) {
+    return [];
+  }
+
+  if (providersConfig.has('organization')) {
+    // simple/single config variant
+    return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
+  }
+
+  return providersConfig.keys().map(id => {
+    const providerConfig = providersConfig.getConfig(id);
+
+    return readProviderConfig(id, providerConfig);
+  });
+}
+
+function readProviderConfig(
+  id: string,
+  config: Config,
+): GitHubEntityProviderConfig {
+  const organization = config.getString('organization');
+  const catalogPath =
+    config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
+  const repositoryPattern = config.getOptionalString('filters.repository');
+  const branchPattern = config.getOptionalString('filters.branch');
+
+  return {
+    id,
+    catalogPath,
+    organization,
+    filters: {
+      repository: repositoryPattern
+        ? compileRegExp(repositoryPattern)
+        : undefined,
+      branch: branchPattern || undefined,
+    },
+  };
+}
+/**
+ * Compiles a RegExp while enforcing the pattern to contain
+ * the start-of-line and end-of-line anchors.
+ *
+ * @param pattern
+ */
+function compileRegExp(pattern: string): RegExp {
+  let fullLinePattern = pattern;
+  if (!fullLinePattern.startsWith('^')) {
+    fullLinePattern = `^${fullLinePattern}`;
+  }
+  if (!fullLinePattern.endsWith('$')) {
+    fullLinePattern = `${fullLinePattern}$`;
+  }
+
+  return new RegExp(fullLinePattern);
+}

plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts

@@ -42,7 +42,7 @@ import {
   getOrganizationTeams,
   getOrganizationUsers,
   parseGitHubOrgUrl,
-} from './lib';
+} from '../lib';
 
 /**
  * Options for {@link GitHubOrgEntityProvider}.