Merge pull request #7627 from SDA-SE/feat/githuborgentityprovider

Add `GitHubOrgEntityProvider`

.changeset/sharp-crabs-stare.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add a `GitHubOrgEntityProvider` that can be used instead of the `GithubOrgReaderProcessor`.

plugins/catalog-backend/api-report.md

@@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model';
 import { EntityPolicy } from '@backstage/catalog-model';
 import { EntityRelationSpec } from '@backstage/catalog-model';
 import express from 'express';
+import { GitHubIntegrationConfig } from '@backstage/integration';
 import { IndexableDocument } from '@backstage/search-common';
 import { JsonObject } from '@backstage/config';
 import { JsonValue } from '@backstage/config';
@@ -1020,6 +1021,33 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// Warning: (ae-missing-release-tag) "GitHubOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class GitHubOrgEntityProvider implements EntityProvider {
+  constructor(options: {
+    id: string;
+    orgUrl: string;
+    gitHubConfig: GitHubIntegrationConfig;
+    logger: Logger_2;
+  });
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      id: string;
+      orgUrl: string;
+      logger: Logger_2;
+    },
+  ): GitHubOrgEntityProvider;
+  // (undocumented)
+  getProviderName(): string;
+  // (undocumented)
+  read(): Promise<void>;
+}
+
 // Warning: (ae-missing-release-tag) "GithubOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public

plugins/catalog-backend/src/ingestion/index.ts

@@ -14,14 +14,15 @@
  * limitations under the License.
  */
 
-export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules';
 export { DefaultCatalogRulesEnforcer } from './CatalogRules';
+export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules';
+export * from './processors';
+export * from './providers';
 export type {
-  AnalyzeLocationRequest,
-  AnalyzeLocationResponse,
-  LocationAnalyzer,
   AnalyzeLocationEntityField,
   AnalyzeLocationExistingEntity,
   AnalyzeLocationGenerateEntity,
+  AnalyzeLocationRequest,
+  AnalyzeLocationResponse,
+  LocationAnalyzer,
 } from './types';
-export * from './processors';

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts

@@ -13,27 +13,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-jest.mock('@octokit/graphql');
 import { getVoidLogger } from '@backstage/backend-common';
 import { LocationSpec } from '@backstage/catalog-model';
-import {
-  ScmIntegrations,
-  GithubCredentialsProvider,
-} from '@backstage/integration';
-import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor';
-import { graphql } from '@octokit/graphql';
 import { ConfigReader } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
+
+jest.mock('@octokit/graphql');
 
 describe('GithubOrgReaderProcessor', () => {
-  describe('parseUrl', () => {
-    it('only supports clean org urls, and decodes them', () => {
-      expect(() => parseUrl('https://github.com')).toThrow();
-      expect(() => parseUrl('https://github.com/org/foo')).toThrow();
-      expect(() => parseUrl('https://github.com/org/foo/teams')).toThrow();
-      expect(parseUrl('https://github.com/foo%32')).toEqual({ org: 'foo2' });
-    });
-  });
-
   describe('implementation', () => {
     const logger = getVoidLogger();
     const integrations = ScmIntegrations.fromConfig(

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts

@@ -23,10 +23,14 @@ import {
 } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
-import { getOrganizationTeams, getOrganizationUsers } from './github';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  parseGitHubOrgUrl,
+} from './github';
 import * as results from './results';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
-import { buildOrgHierarchy } from './util/org';
+import { assignGroupsToUsers, buildOrgHierarchy } from './util/org';
 
 type GraphQL = typeof graphql;
 
@@ -61,7 +65,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     }
 
     const { client, tokenType } = await this.createClient(location.target);
-    const { org } = parseUrl(location.target);
+    const { org } = parseGitHubOrgUrl(location.target);
 
     // Read out all of the raw data
     const startTimestamp = Date.now();
@@ -78,16 +82,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
       `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
     );
 
-    // Fill out the hierarchy
-    const usersByName = new Map(users.map(u => [u.metadata.name, u]));
-    for (const [groupName, userNames] of groupMemberUsers.entries()) {
-      for (const userName of userNames) {
-        const user = usersByName.get(userName);
-        if (user && !user.spec.memberOf.includes(groupName)) {
-          user.spec.memberOf.push(groupName);
-        }
-      }
-    }
+    assignGroupsToUsers(users, groupMemberUsers);
     buildOrgHierarchy(groups);
 
     // Done!
@@ -126,18 +121,3 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     return { client, tokenType };
   }
 }
-
-/*
- * Helpers
- */
-
-export function parseUrl(urlString: string): { org: string } {
-  const path = new URL(urlString).pathname.substr(1).split('/');
-
-  // /backstage
-  if (path.length === 1 && path[0].length) {
-    return { org: decodeURIComponent(path[0]) };
-  }
-
-  throw new Error(`Expected a URL pointing to /<org>`);
-}

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -17,7 +17,8 @@
 export { readGithubConfig, readGithubMultiOrgConfig } from './config';
 export type { GithubMultiOrgConfig, ProviderConfig } from './config';
 export {
+  getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
-  getOrganizationRepositories,
 } from './github';
+export { parseGitHubOrgUrl } from './util';

plugins/catalog-backend/src/ingestion/processors/github/util.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { parseGitHubOrgUrl } from './util';
+
+describe('parseGitHubOrgUrl', () => {
+  it('only supports clean org urls, and decodes them', () => {
+    expect(() => parseGitHubOrgUrl('https://github.com')).toThrow();
+    expect(() => parseGitHubOrgUrl('https://github.com/org/foo')).toThrow();
+    expect(() =>
+      parseGitHubOrgUrl('https://github.com/org/foo/teams'),
+    ).toThrow();
+    expect(parseGitHubOrgUrl('https://github.com/foo%32')).toEqual({
+      org: 'foo2',
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/github/util.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function parseGitHubOrgUrl(urlString: string): { org: string } {
+  const path = new URL(urlString).pathname.substr(1).split('/');
+
+  // /backstage
+  if (path.length === 1 && path[0].length) {
+    return { org: decodeURIComponent(path[0]) };
+  }
+
+  throw new Error(`Expected a URL pointing to /<org>`);
+}

plugins/catalog-backend/src/ingestion/processors/util/org.test.ts

@@ -15,7 +15,16 @@
  */
 
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
-import { buildMemberOf, buildOrgHierarchy } from './org';
+import { assignGroupsToUsers, buildMemberOf, buildOrgHierarchy } from './org';
+
+function u(name: string, memberOf: string[] = []): UserEntity {
+  return {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: { name },
+    spec: { memberOf },
+  };
+}
 
 function g(
   name: string,
@@ -56,22 +65,33 @@ describe('buildOrgHierarchy', () => {
   });
 });
 
+describe('assignGroupsToUsers', () => {
+  it('should assign groups to users', () => {
+    const users: UserEntity[] = [u('u1'), u('u2')];
+    const groupMemberUsers = new Map<string, string[]>([
+      ['g1', ['u1', 'u2']],
+      ['g2', ['u2']],
+      ['g3', ['u3']],
+    ]);
+
+    assignGroupsToUsers(users, groupMemberUsers);
+
+    expect(users[0].spec.memberOf).toEqual(['g1']);
+    expect(users[1].spec.memberOf).toEqual(['g1', 'g2']);
+  });
+});
+
 describe('buildMemberOf', () => {
   it('fills indirect member of groups', () => {
     const a = g('a', undefined, []);
     const b = g('b', 'a', []);
     const c = g('c', 'b', []);
-    const u: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: { name: 'n' },
-      spec: { profile: {}, memberOf: ['c'] },
-    };
+    const user = u('n', ['c']);
 
     const groups = [a, b, c];
     buildOrgHierarchy(groups);
-    buildMemberOf(groups, [u]);
-    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+    buildMemberOf(groups, [user]);
+    expect(user.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
   });
 
   it('takes group spec.members into account', () => {
@@ -79,16 +99,11 @@ describe('buildMemberOf', () => {
     const b = g('b', 'a', []);
     const c = g('c', 'b', []);
     c.spec.members = ['n'];
-    const u: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: { name: 'n' },
-      spec: { profile: {}, memberOf: [] },
-    };
+    const user = u('n');
 
     const groups = [a, b, c];
     buildOrgHierarchy(groups);
-    buildMemberOf(groups, [u]);
-    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+    buildMemberOf(groups, [user]);
+    expect(user.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
   });
 });

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -49,6 +49,22 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
   }
 }
 
+// Ensure that users have their direct group memberships.
+export function assignGroupsToUsers(
+  users: UserEntity[],
+  groupMemberUsers: Map<string, string[]>,
+) {
+  const usersByName = new Map(users.map(u => [u.metadata.name, u]));
+  for (const [groupName, userNames] of groupMemberUsers.entries()) {
+    for (const userName of userNames) {
+      const user = usersByName.get(userName);
+      if (user && !user.spec.memberOf.includes(groupName)) {
+        user.spec.memberOf.push(groupName);
+      }
+    }
+  }
+}
+
 // Ensure that users have their transitive group memberships. Requires that
 // the groups were previously processed with buildOrgHierarchy()
 export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) {

plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import {
+  GithubCredentialsProvider,
+  GitHubIntegrationConfig,
+} from '@backstage/integration';
+import { GitHubOrgEntityProvider } from '.';
+import { EntityProviderConnection } from '../../providers';
+import { withLocations } from './GitHubOrgEntityProvider';
+import { graphql } from '@octokit/graphql';
+
+jest.mock('@octokit/graphql');
+
+describe('GitHubOrgEntityProvider', () => {
+  describe('read', () => {
+    afterEach(() => jest.resetAllMocks());
+
+    it('should read org data and apply mutation', async () => {
+      const mockClient = jest.fn();
+
+      mockClient
+        .mockResolvedValueOnce({
+          organization: {
+            membersWithRole: {
+              pageInfo: { hasNextPage: false },
+              nodes: [
+                {
+                  login: 'a',
+                  name: 'b',
+                  bio: 'c',
+                  email: 'd',
+                  avatarUrl: 'e',
+                },
+              ],
+            },
+          },
+        })
+        .mockResolvedValueOnce({
+          organization: {
+            teams: {
+              pageInfo: { hasNextPage: false },
+              nodes: [
+                {
+                  slug: 'team',
+                  combinedSlug: 'blah/team',
+                  name: 'Team',
+                  description: 'The one and only team',
+                  avatarUrl: 'http://example.com/team.jpeg',
+                  parentTeam: {
+                    slug: 'parent',
+                    combinedSlug: '',
+                    members: { pageInfo: { hasNextPage: false }, nodes: [] },
+                  },
+                  members: {
+                    pageInfo: { hasNextPage: false },
+                    nodes: [{ login: 'a' }],
+                  },
+                },
+              ],
+            },
+          },
+        });
+
+      (graphql.defaults as jest.Mock).mockReturnValue(mockClient);
+
+      const entityProviderConnection: EntityProviderConnection = {
+        applyMutation: jest.fn(),
+      };
+
+      const logger = getVoidLogger();
+      const gitHubConfig: GitHubIntegrationConfig = {
+        host: 'https://github.com',
+      };
+
+      const mockGetCredentials = jest.fn().mockReturnValue({
+        headers: { token: 'blah' },
+        type: 'app',
+      });
+
+      jest.spyOn(GithubCredentialsProvider, 'create').mockReturnValue({
+        getCredentials: mockGetCredentials,
+      } as any);
+
+      const entityProvider = new GitHubOrgEntityProvider({
+        id: 'my-id',
+        orgUrl: 'https://github.com/backstage',
+        gitHubConfig,
+        logger,
+      });
+
+      entityProvider.connect(entityProviderConnection);
+
+      await entityProvider.read();
+
+      expect(entityProviderConnection.applyMutation).toBeCalledWith({
+        entities: [
+          {
+            entity: {
+              apiVersion: 'backstage.io/v1alpha1',
+              kind: 'User',
+              metadata: {
+                annotations: {
+                  'backstage.io/managed-by-location':
+                    'url:https://https://github.com/a',
+                  'backstage.io/managed-by-origin-location':
+                    'url:https://https://github.com/a',
+                  'github.com/user-login': 'a',
+                },
+                description: 'c',
+                name: 'a',
+              },
+              spec: {
+                memberOf: ['team'],
+                profile: {
+                  displayName: 'b',
+                  email: 'd',
+                  picture: 'e',
+                },
+              },
+            },
+            locationKey: 'github-org-provider:my-id',
+          },
+          {
+            entity: {
+              apiVersion: 'backstage.io/v1alpha1',
+              kind: 'Group',
+              metadata: {
+                annotations: {
+                  'backstage.io/managed-by-location':
+                    'url:https://https://github.com/orgs/backstage/teams/team',
+                  'backstage.io/managed-by-origin-location':
+                    'url:https://https://github.com/orgs/backstage/teams/team',
+                  'github.com/team-slug': 'blah/team',
+                },
+                name: 'team',
+                description: 'The one and only team',
+              },
+              spec: {
+                children: [],
+                parent: 'parent',
+                profile: {
+                  displayName: 'Team',
+                  picture: 'http://example.com/team.jpeg',
+                },
+                type: 'team',
+              },
+            },
+            locationKey: 'github-org-provider:my-id',
+          },
+        ],
+        type: 'full',
+      });
+    });
+  });
+
+  describe('withLocations', () => {
+    it('should set location for user', () => {
+      const entity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+        },
+        spec: {
+          memberOf: [],
+        },
+      };
+
+      expect(withLocations('https://github.com', 'backstage', entity)).toEqual({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+          },
+        },
+        spec: {
+          memberOf: [],
+        },
+      });
+    });
+
+    it('should set location for group', () => {
+      const entity: GroupEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'mygroup',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+        },
+      };
+
+      expect(withLocations('https://github.com', 'backstage', entity)).toEqual({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'mygroup',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/orgs/backstage/teams/mygroup',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/orgs/backstage/teams/mygroup',
+          },
+        },
+        spec: {
+          type: 'team',
+          children: [],
+        },
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts

@@ -0,0 +1,180 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  Entity,
+  LOCATION_ANNOTATION,
+  ORIGIN_LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  GitHubIntegrationConfig,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { merge } from 'lodash';
+import { Logger } from 'winston';
+import {
+  EntityProvider,
+  EntityProviderConnection,
+} from '../../providers/types';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  parseGitHubOrgUrl,
+} from '../processors/github';
+import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org';
+
+// TODO: Consider supporting an (optional) webhook that reacts on org changes
+
+export class GitHubOrgEntityProvider implements EntityProvider {
+  private connection?: EntityProviderConnection;
+  private readonly credentialsProvider: GithubCredentialsProvider;
+
+  static fromConfig(
+    config: Config,
+    options: { id: string; orgUrl: string; logger: Logger },
+  ) {
+    const integrations = ScmIntegrations.fromConfig(config);
+    const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config;
+
+    if (!gitHubConfig) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`,
+      );
+    }
+
+    const logger = options.logger.child({
+      target: options.orgUrl,
+    });
+
+    return new GitHubOrgEntityProvider({
+      id: options.id,
+      orgUrl: options.orgUrl,
+      logger,
+      gitHubConfig,
+    });
+  }
+
+  constructor(
+    private options: {
+      id: string;
+      orgUrl: string;
+      gitHubConfig: GitHubIntegrationConfig;
+      logger: Logger;
+    },
+  ) {
+    this.credentialsProvider = GithubCredentialsProvider.create(
+      options.gitHubConfig,
+    );
+  }
+
+  getProviderName() {
+    return `GitHubOrgEntityProvider:${this.options.id}`;
+  }
+
+  async connect(connection: EntityProviderConnection) {
+    this.connection = connection;
+  }
+
+  async read() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const { markReadComplete } = trackProgress(this.options.logger);
+
+    const { headers, type: tokenType } =
+      await this.credentialsProvider.getCredentials({
+        url: this.options.orgUrl,
+      });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const { org } = parseGitHubOrgUrl(this.options.orgUrl);
+    const { users } = await getOrganizationUsers(client, org, tokenType);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(
+      client,
+      org,
+    );
+    assignGroupsToUsers(users, groupMemberUsers);
+    buildOrgHierarchy(groups);
+
+    const { markCommitComplete } = markReadComplete({ users, groups });
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: [...users, ...groups].map(entity => ({
+        locationKey: `github-org-provider:${this.options.id}`,
+        entity: withLocations(
+          `https://${this.options.gitHubConfig.host}`,
+          org,
+          entity,
+        ),
+      })),
+    });
+
+    markCommitComplete();
+  }
+}
+
+// Helps wrap the timing and logging behaviors
+function trackProgress(logger: Logger) {
+  let timestamp = Date.now();
+  let summary: string;
+
+  logger.info('Reading GitHub users and groups');
+
+  function markReadComplete(read: { users: unknown[]; groups: unknown[] }) {
+    summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`;
+    const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    timestamp = Date.now();
+    logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
+    return { markCommitComplete };
+  }
+
+  function markCommitComplete() {
+    const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    logger.info(`Committed ${summary} in ${commitDuration} seconds.`);
+  }
+
+  return { markReadComplete };
+}
+
+// Makes sure that emitted entities have a proper location
+export function withLocations(
+  baseUrl: string,
+  org: string,
+  entity: Entity,
+): Entity {
+  const location =
+    entity.kind === 'Group'
+      ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}`
+      : `url:${baseUrl}/${entity.metadata.name}`;
+  return merge(
+    {
+      metadata: {
+        annotations: {
+          [LOCATION_ANNOTATION]: location,
+          [ORIGIN_LOCATION_ANNOTATION]: location,
+        },
+      },
+    },
+    entity,
+  ) as Entity;
+}

plugins/catalog-backend/src/ingestion/providers/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';