ability to discover catalog-infos from default branch

Previously the github-discovery supported only repos from a specific
branch. This change cause the discovery to work in two modes. It continues
to work as it did with a wildcard for repositories on a specified branch.

The new mode added causes the discovery processor to use the default branch
and looks for a file called catalog-info.yaml.

Signed-off-by: Brian Fletcher <brian@roadie.io>

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts

@@ -45,12 +45,15 @@ describe('GithubDiscoveryProcessor', () => {
         repoSearchPath: /^proj.*$/,
         catalogPath: '/blob/master/catalog.yaml',
       });
+      expect(parseUrl('https://github.com/foo')).toEqual({
+        org: 'foo',
+        host: 'github.com',
+      });
     });
 
     it('throws on incorrectly formed URLs', () => {
       expect(() => parseUrl('https://github.com')).toThrow();
       expect(() => parseUrl('https://github.com//')).toThrow();
-      expect(() => parseUrl('https://github.com/foo')).toThrow();
       expect(() => parseUrl('https://github.com//foo')).toThrow();
       expect(() => parseUrl('https://github.com/org/teams')).toThrow();
       expect(() => parseUrl('https://github.com/org//teams')).toThrow();

plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts

@@ -28,7 +28,15 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types';
 
 /**
  * Extracts repositories out of a GitHub org.
- */
+ *
+ * It can be configured in two modes. The first will create locations for all
+ * catalog-info.yaml files on the default branch. The second will create locations
+ * for all projects which have a catalog-info.yaml on the master branch.
+ *
+ *    target: "https://github.com/backstage"
+ *    or
+ *    target: https://github.com/backstage/*\/blob/master/catalog-info.yaml
+ **/
 export class GithubDiscoveryProcessor implements CatalogProcessor {
   private readonly integrations: ScmIntegrations;
   private readonly logger: Logger;
@@ -87,9 +95,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
     this.logger.info(`Reading GitHub repositories from ${location.target}`);
 
     const { repositories } = await getOrganizationRepositories(client, org);
-    const matching = repositories.filter(
-      r => !r.isArchived && repoSearchPath.test(r.name),
-    );
+    const matching = repoSearchPath
+      ? repositories.filter(r => !r.isArchived && repoSearchPath.test(r.name))
+      : repositories;
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
     this.logger.debug(
@@ -97,11 +105,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
     );
 
     for (const repository of matching) {
+      const path = catalogPath
+        ? catalogPath
+        : `/blob/${repository.defaultBranchRef.name}/catalog-info.yaml`;
       emit(
         results.location(
           {
             type: 'url',
-            target: `${repository.url}${catalogPath}`,
+            target: `${repository.url}${path}`,
           },
           // Not all locations may actually exist, since the user defined them as a wildcard pattern.
           // Thus, we emit them as optional and let the downstream processor find them while not outputting
@@ -121,14 +132,16 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
 
 export function parseUrl(urlString: string): {
   org: string;
-  repoSearchPath: RegExp;
-  catalogPath: string;
+  repoSearchPath?: RegExp;
+  catalogPath?: string;
   host: string;
 } {
   const url = new URL(urlString);
   const path = url.pathname.substr(1).split('/');
 
   // /backstage/techdocs-*/blob/master/catalog-info.yaml
+  // can also be
+  // /backstage
   if (path.length > 2 && path[0].length && path[1].length) {
     return {
       org: decodeURIComponent(path[0]),
@@ -136,6 +149,11 @@ export function parseUrl(urlString: string): {
       catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`,
       host: url.host,
     };
+  } else if (path.length === 1 && path[0].length) {
+    return {
+      org: decodeURIComponent(path[0]),
+      host: url.host,
+    };
   }
 
   throw new Error(`Failed to parse ${urlString}`);

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -60,6 +60,9 @@ export type Repository = {
   name: string;
   url: string;
   isArchived: boolean;
+  defaultBranchRef: {
+    name: string;
+  };
 };
 
 export type Connection<T> = {
@@ -252,6 +255,9 @@ export async function getOrganizationRepositories(
             name
             url
             isArchived
+            defaultBranchRef {
+              name
+            }
           }
           pageInfo {
             hasNextPage