Implement filtering of github entities based on presence

This implements the config from the previous commit. The getRepositories
graphql query is updated to include information about the specified
catalog info file. When validation is turned on, if the file is not
present or empty, the location will not be emitted.

Signed-off-by: Nikolas Skoufis <nskoufis@seek.com.au>

plugins/catalog-backend-module-github/src/lib/github.test.ts

@@ -213,6 +213,7 @@ describe('github', () => {
                 defaultBranchRef: {
                   name: 'main',
                 },
+                catalogInfoFile: null,
               },
               {
                 name: 'demo',
@@ -222,6 +223,11 @@ describe('github', () => {
                 defaultBranchRef: {
                   name: 'main',
                 },
+                catalogInfoFile: {
+                  __typename: 'Blob',
+                  id: 'acb123',
+                  text: 'some yaml',
+                },
               },
             ],
             pageInfo: {
@@ -243,6 +249,7 @@ describe('github', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: null,
           },
           {
             name: 'demo',
@@ -252,6 +259,11 @@ describe('github', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: {
+              __typename: 'Blob',
+              id: 'acb123',
+              text: 'some yaml',
+            },
           },
         ],
       };
@@ -262,9 +274,9 @@ describe('github', () => {
         ),
       );
 
-      await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual(
-        output,
-      );
+      await expect(
+        getOrganizationRepositories(graphql, 'a', 'catalog-info.yaml'),
+      ).resolves.toEqual(output);
     });
   });
 });

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -65,6 +65,11 @@ export type Repository = {
   defaultBranchRef: {
     name: string;
   } | null;
+  catalogInfoFile: {
+    __typename: string;
+    id: string;
+    text: string;
+  } | null;
 };
 
 type RepositoryTopics = {
@@ -266,14 +271,22 @@ export async function getOrganizationTeams(
 export async function getOrganizationRepositories(
   client: typeof graphql,
   org: string,
+  catalogPath: string,
 ): Promise<{ repositories: Repository[] }> {
+  const catalogPathRef = `HEAD:${catalogPath}`;
   const query = `
-    query repositories($org: String!, $cursor: String) {
+    query repositories($org: String!, $catalogPathRef: String!, $cursor: String) {
       repositoryOwner(login: $org) {
         login
         repositories(first: 100, after: $cursor) {
           nodes {
             name
+            catalogInfoFile: object(expression: $catalogPathRef) {
+              __typename
+              ... on Blob {
+                id
+              }
+            }
             url
             isArchived
             repositoryTopics(first: 100) {
@@ -302,7 +315,7 @@ export async function getOrganizationRepositories(
     query,
     r => r.repositoryOwner?.repositories,
     x => x,
-    { org },
+    { org, catalogPathRef },
   );
 
   return { repositories };

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts

@@ -170,6 +170,11 @@ describe('GitHubEntityProvider', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: {
+              __typename: 'Blob',
+              id: 'abc123',
+              text: 'some yaml',
+            },
           },
         ],
       }),
@@ -267,6 +272,11 @@ describe('GitHubEntityProvider', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: {
+              __typename: 'Blob',
+              id: 'abc123',
+              text: 'some yaml',
+            },
           },
         ],
       }),
@@ -341,6 +351,11 @@ describe('GitHubEntityProvider', () => {
             defaultBranchRef: {
               name: 'main',
             },
+            catalogInfoFile: {
+              __typename: 'Blob',
+              id: 'abc123',
+              text: 'some yaml',
+            },
           },
         ],
       }),
@@ -381,6 +396,110 @@ describe('GitHubEntityProvider', () => {
       entities: expectedEntities,
     });
   });
+
+  it('should filter out invalid locations when validateLocationsExist is set to true', async () => {
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            myProvider: {
+              organization: 'test-org',
+              catalogPath: 'catalog-custom.yaml',
+              filters: {
+                branch: 'main',
+              },
+              validateLocationsExist: true,
+            },
+          },
+        },
+      },
+    });
+    const schedule = new PersistingTaskRunner();
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+
+    const provider = GitHubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const mockGetOrganizationRepositories = jest.spyOn(
+      helpers,
+      'getOrganizationRepositories',
+    );
+
+    mockGetOrganizationRepositories.mockReturnValue(
+      Promise.resolve({
+        repositories: [
+          {
+            name: 'test-repo',
+            url: 'https://github.com/test-org/test-repo',
+            repositoryTopics: {
+              nodes: [],
+            },
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+            catalogInfoFile: null,
+          },
+          {
+            name: 'another-repo',
+            url: 'https://github.com/test-org/another-repo',
+            repositoryTopics: {
+              nodes: [],
+            },
+            isArchived: false,
+            defaultBranchRef: {
+              name: 'main',
+            },
+            catalogInfoFile: {
+              __typename: 'Blob',
+              id: 'abc123',
+              text: 'some yaml',
+            },
+          },
+        ],
+      }),
+    );
+
+    await provider.connect(entityProviderConnection);
+
+    const taskDef = schedule.getTasks()[0];
+    expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
+    await (taskDef.fn as () => Promise<void>)();
+
+    const url = `https://github.com/test-org/another-repo/blob/main/catalog-custom.yaml`;
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-934f500db2ba2e8ea3524567926f45a73bb0b532',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:myProvider',
+      },
+    ];
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: expectedEntities,
+    });
+  });
 });
 
 it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
@@ -437,6 +556,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit
           defaultBranchRef: {
             name: 'main',
           },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
         },
         {
           name: 'test-repo-2',
@@ -455,6 +579,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit
           defaultBranchRef: {
             name: 'main',
           },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
         },
         {
           name: 'test-repo-3',
@@ -470,6 +599,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit
           defaultBranchRef: {
             name: 'main',
           },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
         },
       ],
     }),

plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts

@@ -163,6 +163,7 @@ export class GitHubEntityProvider implements EntityProvider {
   private async findCatalogFiles(): Promise<Repository[]> {
     const organization = this.config.organization;
     const host = this.integration.host;
+    const catalogPath = this.config.catalogPath;
     const orgUrl = `https://${host}/${organization}`;
 
     const { headers } = await this.githubCredentialsProvider.getCredentials({
@@ -177,8 +178,18 @@ export class GitHubEntityProvider implements EntityProvider {
     const { repositories } = await getOrganizationRepositories(
       client,
       organization,
+      catalogPath,
     );
 
+    if (this.config.validateLocationsExist) {
+      return repositories.filter(repository => {
+        return (
+          repository.catalogInfoFile?.__typename === 'Blob' &&
+          repository.catalogInfoFile.text !== ''
+        );
+      });
+    }
+
     return repositories;
   }