Merge pull request #4284 from nirga/org-repo

Add support for github-discovery locations

.changeset/moody-mice-cheer.md

@@ -0,0 +1,15 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Added an option to scan GitHub for repositories using a new location type `github-discovery`.
+Example:
+
+```yaml
+type: 'github-discovery',
+target:
+   'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml'
+```
+
+You can use wildcards (`*`) as well. This will add `location` entities for each matching repository.
+Currently though, you must specify the exact path of the `catalog.yaml` file in the repository.

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
+import { getOrganizationRepositories } from './github';
+import { ConfigReader } from '@backstage/config';
+
+jest.mock('./github');
+const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction<
+  typeof getOrganizationRepositories
+>;
+
+describe('GithubDiscoveryProcessor', () => {
+  describe('parseUrl', () => {
+    it('parses well formed URLs', () => {
+      expect(
+        parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'),
+      ).toEqual({
+        org: 'foo',
+        repoSearchPath: /^proj$/,
+        catalogPath: '/blob/master/catalog.yaml',
+      });
+      expect(
+        parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'),
+      ).toEqual({
+        org: 'foo',
+        repoSearchPath: /^proj.*$/,
+        catalogPath: '/blob/master/catalog.yaml',
+      });
+    });
+
+    it('throws on incorrectly formed URLs', () => {
+      expect(() => parseUrl('https://github.com')).toThrow();
+      expect(() => parseUrl('https://github.com//')).toThrow();
+      expect(() => parseUrl('https://github.com/foo')).toThrow();
+      expect(() => parseUrl('https://github.com//foo')).toThrow();
+      expect(() => parseUrl('https://github.com/org/teams')).toThrow();
+      expect(() => parseUrl('https://github.com/org//teams')).toThrow();
+    });
+  });
+
+  describe('reject unrelated entries', () => {
+    it('rejects unknown types', async () => {
+      const processor = GithubDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            github: [{ host: 'github.com', token: 'blob' }],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'not-github-discovery',
+        target: 'https://github.com',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).resolves.toBeFalsy();
+    });
+
+    it('rejects unknown targets', async () => {
+      const processor = GithubDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            github: [
+              { host: 'github.com', token: 'blob' },
+              { host: 'ghe.example.net', token: 'blob' },
+            ],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target: 'https://not.github.com/apa',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).rejects.toThrow(
+        /There is no GitHub integration that matches https:\/\/not.github.com\/apa/,
+      );
+    });
+  });
+
+  describe('handles repositories', () => {
+    const processor = GithubDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          github: [{ host: 'github.com', token: 'blob' }],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
+    beforeEach(() => {
+      mockGetOrganizationRepositories.mockClear();
+    });
+
+    it('output all repositories', async () => {
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target: 'https://github.com/backstage/*/blob/master/catalog.yaml',
+      };
+      mockGetOrganizationRepositories.mockResolvedValueOnce({
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          { name: 'demo', url: 'https://github.com/backstage/demo' },
+        ],
+      });
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/backstage/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target: 'https://github.com/backstage/demo/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+    });
+
+    it('output repositories with wildcards', async () => {
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target:
+          'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml',
+      };
+      mockGetOrganizationRepositories.mockResolvedValueOnce({
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'techdocs-cli',
+            url: 'https://github.com/backstage/techdocs-cli',
+          },
+          {
+            name: 'techdocs-container',
+            url: 'https://github.com/backstage/techdocs-container',
+          },
+        ],
+      });
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+    });
+    it('filter unrelated repositories', async () => {
+      const location: LocationSpec = {
+        type: 'github-discovery',
+        target: 'https://github.com/backstage/test/blob/master/catalog.yaml',
+      };
+      mockGetOrganizationRepositories.mockResolvedValueOnce({
+        repositories: [
+          { name: 'abstest', url: 'https://github.com/backstage/abctest' },
+          {
+            name: 'test',
+            url: 'https://github.com/backstage/test',
+          },
+          {
+            name: 'testxyz',
+            url: 'https://github.com/backstage/testxyz',
+          },
+        ],
+      });
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target: 'https://github.com/backstage/test/blob/master/catalog.yaml',
+        },
+        optional: false,
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { Logger } from 'winston';
+import { getOrganizationRepositories } from './github';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+/**
+ * Extracts repositories out of a GitHub org.
+ */
+export class GithubDiscoveryProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new GithubDiscoveryProcessor({
+      ...options,
+      integrations,
+    });
+  }
+
+  constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-discovery') {
+      return false;
+    }
+
+    const gitHubConfig = this.integrations.github.byUrl(location.target)
+      ?.config;
+    if (!gitHubConfig) {
+      throw new Error(
+        `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`,
+      );
+    }
+    const { headers } = await GithubCredentialsProvider.create(
+      gitHubConfig,
+    ).getCredentials({ url: location.target });
+    const { org, repoSearchPath, catalogPath } = parseUrl(location.target);
+
+    const client = graphql.defaults({
+      baseUrl: gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info(`Reading GitHub repositories from ${location.target}`);
+
+    const { repositories } = await getOrganizationRepositories(client, org);
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${repositories.length} GitHub repositories in ${duration} seconds`,
+    );
+
+    for (const repository of repositories) {
+      if (!repoSearchPath.test(repository.name)) {
+        continue;
+      }
+      emit(
+        results.location(
+          {
+            type: 'url',
+            target: `${repository.url}${catalogPath}`,
+          },
+          false,
+        ),
+      );
+    }
+
+    return true;
+  }
+}
+
+/*
+ * Helpers
+ */
+
+export function parseUrl(
+  urlString: string,
+): { org: string; repoSearchPath: RegExp; catalogPath: string } {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  // /backstage/techdocs-*/blob/master/catalog-info.yaml
+  if (path.length > 2 && path[0].length && path[1].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
+      catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}
+
+export function escapeRegExp(str: string): RegExp {
+  return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
+}

plugins/catalog-backend/src/ingestion/processors/github/github.test.ts

@@ -22,6 +22,7 @@ import {
   getOrganizationTeams,
   getOrganizationUsers,
   getTeamMembers,
+  getOrganizationRepositories,
   QueryResponse,
 } from './github';
 
@@ -151,4 +152,48 @@ describe('github', () => {
       await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output);
     });
   });
+
+  describe('getOrganizationRepositories', () => {
+    it('read repositories', async () => {
+      const input: QueryResponse = {
+        organization: {
+          repositories: {
+            nodes: [
+              {
+                name: 'backstage',
+                url: 'https://github.com/backstage/backstage',
+              },
+              {
+                name: 'demo',
+                url: 'https://github.com/backstage/demo',
+              },
+            ],
+            pageInfo: {
+              hasNextPage: false,
+            },
+          },
+        },
+      };
+
+      const output = {
+        repositories: [
+          { name: 'backstage', url: 'https://github.com/backstage/backstage' },
+          {
+            name: 'demo',
+            url: 'https://github.com/backstage/demo',
+          },
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('repositories', (_req, res, ctx) =>
+          res(ctx.data(input)),
+        ),
+      );
+
+      await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual(
+        output,
+      );
+    });
+  });
 });

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -27,6 +27,7 @@ export type Organization = {
   membersWithRole?: Connection<User>;
   team?: Team;
   teams?: Connection<Team>;
+  repositories?: Connection<Repository>;
 };
 
 export type PageInfo = {
@@ -52,6 +53,11 @@ export type Team = {
   members: Connection<User>;
 };
 
+export type Repository = {
+  name: string;
+  url: string;
+};
+
 export type Connection<T> = {
   pageInfo: PageInfo;
   nodes: T[];
@@ -216,6 +222,39 @@ export async function getOrganizationTeams(
   return { groups, groupMemberUsers };
 }
 
+export async function getOrganizationRepositories(
+  client: typeof graphql,
+  org: string,
+): Promise<{ repositories: Repository[] }> {
+  const query = `
+  query repositories($org: String!, $cursor: String) {
+    organization(login: $org) {
+      name
+      repositories(first: 100, after: $cursor) {
+        nodes {
+          name
+          url
+        }
+        pageInfo {
+          hasNextPage
+          endCursor
+        }
+      }
+    }
+  }
+  `;
+
+  const repositories = await queryWithPaging(
+    client,
+    query,
+    r => r.organization?.repositories,
+    x => x,
+    { org },
+  );
+
+  return { repositories };
+}
+
 /**
  * Gets all the users out of a GitHub organization.
  *

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -16,4 +16,8 @@
 
 export { readGithubConfig } from './config';
 export type { ProviderConfig } from './config';
-export { getOrganizationTeams, getOrganizationUsers } from './github';
+export {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  getOrganizationRepositories,
+} from './github';

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -21,6 +21,7 @@ export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAcco
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
+export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor';
 export { LocationEntityProcessor } from './LocationEntityProcessor';

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -43,6 +43,7 @@ import {
   CatalogProcessorParser,
   CodeOwnersProcessor,
   FileReaderProcessor,
+  GithubDiscoveryProcessor,
   GithubOrgReaderProcessor,
   HigherOrderOperation,
   HigherOrderOperations,
@@ -303,6 +304,7 @@ export class CatalogBuilder {
     if (!this.processorsReplace) {
       processors.push(
         new FileReaderProcessor(),
+        GithubDiscoveryProcessor.fromConfig(config, { logger }),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
         LdapOrgReaderProcessor.fromConfig(config, { logger }),
         MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),