Merge pull request #5173 from kuangp/feat/githubMultiOrgReaderProcessor

feat(GithubMultiOrgReaderProcessor): implement processor to handle multi-github org ingestion

packages/integration/api-report.md

@@ -5,6 +5,7 @@
 ```ts
 
 import { Config } from '@backstage/config';
+import { RestEndpointMethodTypes } from '@octokit/rest';
 
 // @public (undocumented)
 export class AzureIntegration implements ScmIntegration {
@@ -106,6 +107,15 @@ export function getGitLabFileFetchUrl(url: string, config: GitLabIntegrationConf
 // @public
 export function getGitLabRequestOptions(config: GitLabIntegrationConfig): RequestInit;
 
+// @public (undocumented)
+export class GithubAppCredentialsMux {
+    constructor(config: GitHubIntegrationConfig);
+    // (undocumented)
+    getAllInstallations(): Promise<RestEndpointMethodTypes['apps']['listInstallations']['response']['data']>;
+    // (undocumented)
+    getAppToken(owner: string, repo?: string): Promise<string | undefined>;
+}
+
 // @public (undocumented)
 export class GithubCredentialsProvider {
     // (undocumented)

packages/integration/src/github/GithubCredentialsProvider.test.ts

@@ -15,6 +15,7 @@
  */
 
 const octokit = {
+  paginate: async (fn: any) => (await fn()).data,
   apps: {
     listInstallations: jest.fn(),
     createInstallationAccessToken: jest.fn(),
@@ -53,7 +54,7 @@ describe('GithubCredentialsProvider tests', () => {
     jest.resetAllMocks();
   });
   it('create repository specific tokens', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -72,7 +73,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -84,12 +84,8 @@ describe('GithubCredentialsProvider tests', () => {
     const { token, headers, type } = await github.getCredentials({
       url: 'https://github.com/backstage/foobar',
     });
-    const { token: accessToken2 } = await github.getCredentials({
-      url: 'https://github.com/backstage/foobar',
-    });
     expect(type).toEqual('app');
     expect(token).toEqual('secret_token');
-    expect(token).toEqual(accessToken2);
     expect(headers).toEqual({ Authorization: 'Bearer secret_token' });
 
     // fallback to the configured token if no application is matching
@@ -107,7 +103,7 @@ describe('GithubCredentialsProvider tests', () => {
   });
 
   it('creates tokens for an organization', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -121,7 +117,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -133,17 +128,13 @@ describe('GithubCredentialsProvider tests', () => {
     const { token, headers } = await github.getCredentials({
       url: 'https://github.com/backstage',
     });
-    const { token: accessToken2 } = await github.getCredentials({
-      url: 'https://github.com/backstage',
-    });
 
     expect(headers).toEqual({ Authorization: 'Bearer secret_token' });
     expect(token).toEqual('secret_token');
-    expect(token).toEqual(accessToken2);
   });
 
   it('should fail to issue tokens for an organization when the app is installed for a single repo', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -157,7 +148,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({
       data: {
@@ -176,7 +166,7 @@ describe('GithubCredentialsProvider tests', () => {
   });
 
   it('should throw if the app is suspended', async () => {
-    octokit.apps.listInstallations.mockResolvedValueOnce({
+    octokit.apps.listInstallations.mockResolvedValue({
       headers: {
         etag: '123',
       },
@@ -193,7 +183,6 @@ describe('GithubCredentialsProvider tests', () => {
         },
       ],
     } as RestEndpointMethodTypes['apps']['listInstallations']['response']);
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
 
     await expect(
       github.getCredentials({
@@ -229,7 +218,7 @@ describe('GithubCredentialsProvider tests', () => {
     ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' }));
   });
 
-  it('should return the configured token if listing installations throws', async () => {
+  it('should return the configured token if there are no installations', async () => {
     const githubProvider = GithubCredentialsProvider.create({
       host: 'github.com',
       apps: [
@@ -243,7 +232,9 @@ describe('GithubCredentialsProvider tests', () => {
       ],
       token: 'hardcoded_token',
     });
-    octokit.apps.listInstallations.mockRejectedValue({ status: 304 });
+    octokit.apps.listInstallations.mockResolvedValue(({
+      data: [],
+    } as unknown) as RestEndpointMethodTypes['apps']['listInstallations']['response']);
 
     await expect(
       githubProvider.getCredentials({

packages/integration/src/github/GithubCredentialsProvider.ts

@@ -66,7 +66,6 @@ const HEADERS = {
 class GithubAppManager {
   private readonly appClient: Octokit;
   private readonly baseAuthConfig: { appId: number; privateKey: string };
-  private installations?: RestEndpointMethodTypes['apps']['listInstallations']['response'];
   private readonly cache = new Cache();
 
   constructor(config: GithubAppConfig, baseUrl?: string) {
@@ -121,22 +120,15 @@ class GithubAppManager {
     });
   }
 
+  getInstallations(): Promise<
+    RestEndpointMethodTypes['apps']['listInstallations']['response']['data']
+  > {
+    return this.appClient.paginate(this.appClient.apps.listInstallations);
+  }
+
   private async getInstallationData(owner: string): Promise<InstallationData> {
-    // List all installations using the last used etag.
-    // Return cached InstallationData if error with status 304 is thrown.
-    try {
-      this.installations = await this.appClient.apps.listInstallations({
-        headers: {
-          'If-None-Match': this.installations?.headers.etag,
-          Accept: HEADERS.Accept,
-        },
-      });
-    } catch (error) {
-      if (error.status !== 304) {
-        throw error;
-      }
-    }
-    const installation = this.installations?.data.find(
+    const allInstallations = await this.getInstallations();
+    const installation = allInstallations.find(
       inst => inst.account?.login === owner,
     );
     if (installation) {
@@ -163,6 +155,20 @@ export class GithubAppCredentialsMux {
       config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
   }
 
+  async getAllInstallations(): Promise<
+    RestEndpointMethodTypes['apps']['listInstallations']['response']['data']
+  > {
+    if (!this.apps.length) {
+      return [];
+    }
+
+    const installs = await Promise.all(
+      this.apps.map(app => app.getInstallations()),
+    );
+
+    return installs.flat();
+  }
+
   async getAppToken(owner: string, repo?: string): Promise<string | undefined> {
     if (this.apps.length === 0) {
       return undefined;

packages/integration/src/github/index.ts

@@ -20,6 +20,9 @@ export {
 } from './config';
 export type { GitHubIntegrationConfig } from './config';
 export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
-export { GithubCredentialsProvider } from './GithubCredentialsProvider';
+export {
+  GithubAppCredentialsMux,
+  GithubCredentialsProvider,
+} from './GithubCredentialsProvider';
 export type { GithubCredentialType } from './GithubCredentialsProvider';
 export { GitHubIntegration } from './GitHubIntegration';