diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 160a1a533b..59e136c138 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -16,7 +16,7 @@ import { getGitHubFileFetchUrl, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, GithubCredentialsProvider, GitHubIntegration, ScmIntegrations, @@ -59,7 +59,7 @@ export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); return integrations.github.list().map(integration => { const reader = new GithubUrlReader(integration, { treeResponseFactory, diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts new file mode 100644 index 0000000000..f537fee783 --- /dev/null +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts @@ -0,0 +1,347 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScmIntegrations } from '../ScmIntegrations'; + +const octokit = { + paginate: async (fn: any) => (await fn()).data, + apps: { + listInstallations: jest.fn(), + createInstallationAccessToken: jest.fn(), + }, +}; + +jest.doMock('@octokit/rest', () => { + class Octokit { + constructor() { + return octokit; + } + } + return { Octokit }; +}); + +import { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; +import { RestEndpointMethodTypes } from '@octokit/rest'; +import { DateTime } from 'luxon'; +import { ConfigReader } from '@backstage/config'; + +let integrations: ScmIntegrations; + +describe('DefaultGithubCredentialsProvider tests', () => { + beforeEach(() => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], + }, + }), + ); + jest.resetAllMocks(); + }); + + it('create repository specific tokens', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'selected', + account: null, + }, + { + id: 2, + repository_selection: 'selected', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + const { token, headers, type } = await github.getCredentials({ + url: 'https://github.com/backstage/foobar', + }); + expect(type).toEqual('app'); + expect(token).toEqual('secret_token'); + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + + // fallback to the configured token if no application is matching + await expect( + github.getCredentials({ + url: 'https://github.com/404/foobar', + }), + ).resolves.toEqual({ + headers: { + Authorization: 'Bearer hardcoded_token', + }, + token: 'hardcoded_token', + type: 'token', + }); + }); + + it('creates tokens for an organization', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'all', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + expect(token).toEqual('secret_token'); + }); + + it('should not fail to issue tokens for an organization when the app is installed for a single repo', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'selected', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + const expectedToken = 'secret_token'; + expect(headers).toEqual({ Authorization: `Bearer ${expectedToken}` }); + expect(token).toEqual('secret_token'); + }); + + it('should throw if the app is suspended', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + suspended_by: { + login: 'admin', + }, + repository_selection: 'all', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + github.getCredentials({ + url: 'https://github.com/backstage', + }), + ).rejects.toThrow('The GitHub application for backstage is suspended'); + }); + + it('should return the default token when the call to github return a status that is not recognized', async () => { + octokit.apps.listInstallations.mockRejectedValue({ + status: 404, + message: 'NotFound', + }); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + await expect( + github.getCredentials({ + url: 'https://github.com/backstage', + }), + ).rejects.toEqual({ status: 404, message: 'NotFound' }); + }); + + it('should return the default token if no app is configured', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [], + token: 'fallback_token', + }, + ], + }, + }), + ); + + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/404/foobar', + }), + ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' })); + }); + + it('should return the configured token if there are no installations', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], + }, + }), + ); + + octokit.apps.listInstallations.mockResolvedValue({ + data: [], + } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/backstage', + }), + ).resolves.toEqual(expect.objectContaining({ token: 'hardcoded_token' })); + }); + + it('should return undefined if no token or apps are configured', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }), + ); + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/backstage', + }), + ).resolves.toEqual({ headers: undefined, token: undefined, type: 'token' }); + }); + + it('should to create a token for the organization ignoring case sensitive', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'all', + account: { + login: 'BACKSTAGE', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + expect(token).toEqual('secret_token'); + }); +}); diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts new file mode 100644 index 0000000000..37aa23698e --- /dev/null +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import parseGitUrl from 'git-url-parse'; +import { + GithubCredentials, + GithubCredentialsProvider, + GithubCredentialType, +} from './types'; +import { ScmIntegrations } from '../ScmIntegrations'; +import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; + +type MuxCollection = { + [url: string]: { + githubAppCredentialsMux: GithubAppCredentialsMux; + token?: string; + }; +}; + +/** + * Handles the creation and caching of credentials for GitHub integrations. + * + * @public + * @remarks + * + * TODO: Possibly move this to a backend only package so that it's not used in the frontend by mistake + */ +export class DefaultGithubCredentialsProvider + implements GithubCredentialsProvider +{ + static fromIntegrations(integrations: ScmIntegrations) { + const muxen: MuxCollection = {}; + + integrations.github.list().forEach(integration => { + muxen[integration.config.host] = { + githubAppCredentialsMux: new GithubAppCredentialsMux( + integration.config, + ), + token: integration.config.token, + }; + }); + return new DefaultGithubCredentialsProvider(muxen); + } + + private constructor( + private readonly githubAppCredentialsMuxen: MuxCollection, + ) {} + + /** + * Returns {@link GithubCredentials} for a given URL. + * + * @remarks + * + * Consecutive calls to this method with the same URL will return cached + * credentials. + * + * The shortest lifetime for a token returned is 10 minutes. + * + * @example + * ```ts + * const { token, headers } = await getCredentials({ + * url: 'github.com/backstage/foobar' + * }) + * ``` + * + * @param opts - The organization or repository URL + * @returns A promise of {@link GithubCredentials}. + */ + async getCredentials(opts: { url: string }): Promise { + const parsed = parseGitUrl(opts.url); + + if (!this.githubAppCredentialsMuxen[parsed.resource]) { + throw new Error( + `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, + ); + } + + const githubAppCredentialsMux = + this.githubAppCredentialsMuxen[parsed.resource].githubAppCredentialsMux; + const defaultToken = this.githubAppCredentialsMuxen[parsed.resource].token; + + const owner = parsed.owner || parsed.name; + const repo = parsed.owner ? parsed.name : undefined; + + let type: GithubCredentialType = 'app'; + let token = await githubAppCredentialsMux.getAppToken(owner, repo); + if (!token) { + type = 'token'; + token = defaultToken; + } + + return { + headers: token ? { Authorization: `Bearer ${token}` } : undefined, + token, + type, + }; + } +} diff --git a/packages/integration/src/github/GithubAppCredentialsMux.ts b/packages/integration/src/github/GithubAppCredentialsMux.ts new file mode 100644 index 0000000000..c6d055a213 --- /dev/null +++ b/packages/integration/src/github/GithubAppCredentialsMux.ts @@ -0,0 +1,214 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GithubAppConfig, GitHubIntegrationConfig } from './config'; +import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; +import { createAppAuth } from '@octokit/auth-app'; +import { DateTime } from 'luxon'; + +/** + * This accept header is required when calling App APIs in GitHub Enterprise. + * It has no effect on calls to github.com and can probably be removed entirely + * once GitHub Apps is out of preview. + */ +const HEADERS = { + Accept: 'application/vnd.github.machine-man-preview+json', +}; + +type InstallationData = { + installationId: number; + suspended: boolean; +}; + +class Cache { + private readonly tokenCache = new Map< + string, + { token: string; expiresAt: DateTime } + >(); + + async getOrCreateToken( + key: string, + supplier: () => Promise<{ token: string; expiresAt: DateTime }>, + ): Promise<{ accessToken: string }> { + const item = this.tokenCache.get(key); + if (item && this.isNotExpired(item.expiresAt)) { + return { accessToken: item.token }; + } + + const result = await supplier(); + this.tokenCache.set(key, result); + return { accessToken: result.token }; + } + + // consider timestamps older than 50 minutes to be expired. + private isNotExpired = (date: DateTime) => + date.diff(DateTime.local(), 'minutes').minutes > 50; +} + +/** + * GithubAppManager issues and caches tokens for a specific GitHub App. + */ +class GithubAppManager { + private readonly appClient: Octokit; + private readonly baseUrl?: string; + private readonly baseAuthConfig: { appId: number; privateKey: string }; + private readonly cache = new Cache(); + private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations + + constructor(config: GithubAppConfig, baseUrl?: string) { + this.allowedInstallationOwners = config.allowedInstallationOwners; + this.baseUrl = baseUrl; + this.baseAuthConfig = { + appId: config.appId, + privateKey: config.privateKey.replace(/\\n/gm, '\n'), + }; + this.appClient = new Octokit({ + baseUrl, + headers: HEADERS, + authStrategy: createAppAuth, + auth: this.baseAuthConfig, + }); + } + + async getInstallationCredentials( + owner: string, + repo?: string, + ): Promise<{ accessToken: string }> { + const { installationId, suspended } = await this.getInstallationData(owner); + if (this.allowedInstallationOwners) { + if (!this.allowedInstallationOwners?.includes(owner)) { + throw new Error( + `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, + ); + } + } + if (suspended) { + throw new Error(`The GitHub application for ${owner} is suspended`); + } + + const cacheKey = repo ? `${owner}/${repo}` : owner; + + // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. + return this.cache.getOrCreateToken(cacheKey, async () => { + const result = await this.appClient.apps.createInstallationAccessToken({ + installation_id: installationId, + headers: HEADERS, + }); + if (repo && result.data.repository_selection === 'selected') { + const installationClient = new Octokit({ + baseUrl: this.baseUrl, + auth: result.data.token, + }); + const repos = await installationClient.paginate( + installationClient.apps.listReposAccessibleToInstallation, + ); + const hasRepo = repos.some(repository => { + return repository.name === repo; + }); + if (!hasRepo) { + throw new Error( + `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, + ); + } + } + return { + token: result.data.token, + expiresAt: DateTime.fromISO(result.data.expires_at), + }; + }); + } + + getInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + return this.appClient.paginate(this.appClient.apps.listInstallations); + } + + private async getInstallationData(owner: string): Promise { + const allInstallations = await this.getInstallations(); + const installation = allInstallations.find( + inst => + inst.account?.login?.toLocaleLowerCase('en-US') === + owner.toLocaleLowerCase('en-US'), + ); + if (installation) { + return { + installationId: installation.id, + suspended: Boolean(installation.suspended_by), + }; + } + const notFoundError = new Error( + `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, + ); + notFoundError.name = 'NotFoundError'; + throw notFoundError; + } +} + +/** + * Corresponds to a Github installation which internally could hold several GitHub Apps. + * + * @public + */ +export class GithubAppCredentialsMux { + private readonly apps: GithubAppManager[]; + + constructor(config: GitHubIntegrationConfig) { + this.apps = + config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; + } + + async getAllInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + if (!this.apps.length) { + return []; + } + + const installs = await Promise.all( + this.apps.map(app => app.getInstallations()), + ); + + return installs.flat(); + } + + async getAppToken(owner: string, repo?: string): Promise { + if (this.apps.length === 0) { + return undefined; + } + + const results = await Promise.all( + this.apps.map(app => + app.getInstallationCredentials(owner, repo).then( + credentials => ({ credentials, error: undefined }), + error => ({ credentials: undefined, error }), + ), + ), + ); + + const result = results.find(resultItem => resultItem.credentials); + if (result) { + return result.credentials!.accessToken; + } + + const errors = results.map(r => r.error); + const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); + if (notNotFoundError) { + throw notNotFoundError; + } + + return undefined; + } +} diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts index 4278218a2c..e5c6307338 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ScmIntegrations } from '../ScmIntegrations'; +import { GithubCredentialsProvider } from './types'; const octokit = { paginate: async (fn: any) => (await fn()).data, @@ -36,35 +36,25 @@ jest.doMock('@octokit/rest', () => { import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; import { RestEndpointMethodTypes } from '@octokit/rest'; import { DateTime } from 'luxon'; -import { ConfigReader } from '@backstage/config'; - -let integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', - }, - ], - token: 'hardcoded_token', - }, - ], - }, - }), -); - -const github = SingleInstanceGithubCredentialsProvider.create(integrations); describe('SingleInstanceGithubCredentialsProvider tests', () => { + let github: GithubCredentialsProvider; + beforeEach(() => { jest.resetAllMocks(); + github = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }); }); it('create repository specific tokens', async () => { octokit.apps.listInstallations.mockResolvedValue({ @@ -217,22 +207,11 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return the default token if no app is configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [], - token: 'fallback_token', - }, - ], - }, - }), - ); - - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [], + token: 'fallback_token', + }); await expect( githubProvider.getCredentials({ @@ -242,29 +221,19 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return the configured token if there are no installations', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', - }, - ], - token: 'hardcoded_token', - }, - ], + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', }, - }), - ); - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + ], + token: 'hardcoded_token', + }); octokit.apps.listInstallations.mockResolvedValue({ data: [], } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); @@ -277,19 +246,9 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return undefined if no token or apps are configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - }, - ], - }, - }), - ); - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + }); await expect( githubProvider.getCredentials({ diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 5b7aea02ec..5f04a3ed6e 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,211 +15,14 @@ */ import parseGitUrl from 'git-url-parse'; -import { GithubAppConfig, GitHubIntegrationConfig } from './config'; -import { createAppAuth } from '@octokit/auth-app'; -import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import { DateTime } from 'luxon'; + import { GithubCredentials, GithubCredentialsProvider, GithubCredentialType, } from './types'; -import { ScmIntegrations } from '../ScmIntegrations'; - -type InstallationData = { - installationId: number; - suspended: boolean; -}; - -class Cache { - private readonly tokenCache = new Map< - string, - { token: string; expiresAt: DateTime } - >(); - - async getOrCreateToken( - key: string, - supplier: () => Promise<{ token: string; expiresAt: DateTime }>, - ): Promise<{ accessToken: string }> { - const item = this.tokenCache.get(key); - if (item && this.isNotExpired(item.expiresAt)) { - return { accessToken: item.token }; - } - - const result = await supplier(); - this.tokenCache.set(key, result); - return { accessToken: result.token }; - } - - // consider timestamps older than 50 minutes to be expired. - private isNotExpired = (date: DateTime) => - date.diff(DateTime.local(), 'minutes').minutes > 50; -} - -/** - * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entirely - * once GitHub Apps is out of preview. - */ -const HEADERS = { - Accept: 'application/vnd.github.machine-man-preview+json', -}; - -/** - * GithubAppManager issues and caches tokens for a specific GitHub App. - */ -class GithubAppManager { - private readonly appClient: Octokit; - private readonly baseUrl?: string; - private readonly baseAuthConfig: { appId: number; privateKey: string }; - private readonly cache = new Cache(); - private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations - - constructor(config: GithubAppConfig, baseUrl?: string) { - this.allowedInstallationOwners = config.allowedInstallationOwners; - this.baseUrl = baseUrl; - this.baseAuthConfig = { - appId: config.appId, - privateKey: config.privateKey.replace(/\\n/gm, '\n'), - }; - this.appClient = new Octokit({ - baseUrl, - headers: HEADERS, - authStrategy: createAppAuth, - auth: this.baseAuthConfig, - }); - } - - async getInstallationCredentials( - owner: string, - repo?: string, - ): Promise<{ accessToken: string }> { - const { installationId, suspended } = await this.getInstallationData(owner); - if (this.allowedInstallationOwners) { - if (!this.allowedInstallationOwners?.includes(owner)) { - throw new Error( - `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, - ); - } - } - if (suspended) { - throw new Error(`The GitHub application for ${owner} is suspended`); - } - - const cacheKey = repo ? `${owner}/${repo}` : owner; - - // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. - return this.cache.getOrCreateToken(cacheKey, async () => { - const result = await this.appClient.apps.createInstallationAccessToken({ - installation_id: installationId, - headers: HEADERS, - }); - if (repo && result.data.repository_selection === 'selected') { - const installationClient = new Octokit({ - baseUrl: this.baseUrl, - auth: result.data.token, - }); - const repos = await installationClient.paginate( - installationClient.apps.listReposAccessibleToInstallation, - ); - const hasRepo = repos.some(repository => { - return repository.name === repo; - }); - if (!hasRepo) { - throw new Error( - `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, - ); - } - } - return { - token: result.data.token, - expiresAt: DateTime.fromISO(result.data.expires_at), - }; - }); - } - - getInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - return this.appClient.paginate(this.appClient.apps.listInstallations); - } - - private async getInstallationData(owner: string): Promise { - const allInstallations = await this.getInstallations(); - const installation = allInstallations.find( - inst => - inst.account?.login?.toLocaleLowerCase('en-US') === - owner.toLocaleLowerCase('en-US'), - ); - if (installation) { - return { - installationId: installation.id, - suspended: Boolean(installation.suspended_by), - }; - } - const notFoundError = new Error( - `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, - ); - notFoundError.name = 'NotFoundError'; - throw notFoundError; - } -} - -/** - * Corresponds to a Github installation which internally could hold several GitHub Apps. - * - * @public - */ -export class GithubAppCredentialsMux { - private readonly apps: GithubAppManager[]; - - constructor(config: GitHubIntegrationConfig) { - this.apps = - config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; - } - - async getAllInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - if (!this.apps.length) { - return []; - } - - const installs = await Promise.all( - this.apps.map(app => app.getInstallations()), - ); - - return installs.flat(); - } - - async getAppToken(owner: string, repo?: string): Promise { - if (this.apps.length === 0) { - return undefined; - } - - const results = await Promise.all( - this.apps.map(app => - app.getInstallationCredentials(owner, repo).then( - credentials => ({ credentials, error: undefined }), - error => ({ credentials: undefined, error }), - ), - ), - ); - - const result = results.find(resultItem => resultItem.credentials); - if (result) { - return result.credentials!.accessToken; - } - - const errors = results.map(r => r.error); - const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); - if (notNotFoundError) { - throw notNotFoundError; - } - - return undefined; - } -} +import { GitHubIntegrationConfig } from './config'; +import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; /** * Handles the creation and caching of credentials for GitHub integrations. @@ -232,11 +35,19 @@ export class GithubAppCredentialsMux { export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { - static create(integrations: ScmIntegrations) { - return new SingleInstanceGithubCredentialsProvider(integrations); - } + static create: ( + config: GitHubIntegrationConfig, + ) => GithubCredentialsProvider = config => { + return new SingleInstanceGithubCredentialsProvider( + new GithubAppCredentialsMux(config), + config.token, + ); + }; - private constructor(private readonly integrations: ScmIntegrations) {} + private constructor( + private readonly githubAppCredentialsMux: GithubAppCredentialsMux, + private readonly token?: string, + ) {} /** * Returns {@link GithubCredentials} for a given URL. @@ -260,24 +71,15 @@ export class SingleInstanceGithubCredentialsProvider */ async getCredentials(opts: { url: string }): Promise { const parsed = parseGitUrl(opts.url); - const gitHubConfig = this.integrations.github.byUrl(opts.url)?.config; - if (!gitHubConfig) { - throw new Error( - `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, - ); - } - - const githubAppCredentialsMux = new GithubAppCredentialsMux(gitHubConfig); - const defaultToken = gitHubConfig.token; const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; let type: GithubCredentialType = 'app'; - let token = await githubAppCredentialsMux.getAppToken(owner, repo); + let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); if (!token) { type = 'token'; - token = defaultToken; + token = this.token; } return { diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 429a2b1c76..51d7bc9dd2 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -20,10 +20,10 @@ export { } from './config'; export type { GithubAppConfig, GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; -export { - GithubAppCredentialsMux, - SingleInstanceGithubCredentialsProvider, -} from './SingleInstanceGithubCredentialsProvider'; +export { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; +export { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; +export { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; + export type { GithubCredentials, GithubCredentialsProvider, diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 06886ecb4f..d6746f13da 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -21,7 +21,7 @@ import { getOrganizationRepositories } from './github'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; jest.mock('./github'); @@ -78,7 +78,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, @@ -103,7 +103,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, @@ -128,7 +128,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index 6fda605ae3..cd1966441b 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -26,7 +26,7 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import lodash from 'lodash'; import { EntitiesCatalog } from '../../catalog'; @@ -295,7 +295,7 @@ export class CatalogBuilder { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); this.checkDeprecatedReaderProcessors(); diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index 24aefb2d96..5b76866bd2 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -28,7 +28,7 @@ import { import { GithubCredentialsProvider, ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; @@ -294,7 +294,7 @@ export class NextCatalogBuilder { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider: GithubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); return [ new FileReaderProcessor(), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 6975f1aa16..ce1e2a7f49 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -19,7 +19,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { GithubCredentialsProvider, ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { Config } from '@backstage/config'; import { @@ -57,7 +57,7 @@ export const createBuiltinActions = (options: { const { reader, integrations, containerRunner, catalogClient, config } = options; const githubCredentialsProvider: GithubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const actions = [ createFetchPlainAction({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts index 39a1ff6ad8..0d4353c857 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts @@ -17,7 +17,7 @@ import { OctokitProvider } from './OctokitProvider'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; @@ -33,7 +33,7 @@ describe('getOctokit', () => { const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const octokitProvider = new OctokitProvider( integrations, githubCredentialsProvider, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts index 6720de4e95..b725f825fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts @@ -15,11 +15,13 @@ */ jest.mock('@octokit/rest'); - +import { TemplateAction } from '../../types'; import { createGithubActionsDispatchAction } from './githubActionsDispatch'; + import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; @@ -36,12 +38,8 @@ describe('github:actions:dispatch', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); - const action = createGithubActionsDispatchAction({ - integrations, - githubCredentialsProvider, - }); + let githubCredentialsProvider: GithubCredentialsProvider; + let action: TemplateAction; const mockContext = { input: { @@ -60,6 +58,12 @@ describe('github:actions:dispatch', () => { beforeEach(() => { jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createGithubActionsDispatchAction({ + integrations, + githubCredentialsProvider, + }); }); it('should call the githubApis for creating WorkflowDispatch', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts index 0c2daeb42f..fe2535ca42 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts @@ -19,11 +19,13 @@ jest.mock('@octokit/rest'); import { createGithubWebhookAction } from './githubWebhook'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; +import { TemplateAction } from '../..'; describe('github:repository:webhook:create', () => { const config = new ConfigReader({ @@ -36,13 +38,19 @@ describe('github:repository:webhook:create', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + let githubCredentialsProvider: GithubCredentialsProvider; const defaultWebhookSecret = 'aafdfdivierernfdk23f'; - const action = createGithubWebhookAction({ - integrations, - defaultWebhookSecret, - githubCredentialsProvider, + let action: TemplateAction; + + beforeEach(() => { + jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createGithubWebhookAction({ + integrations, + defaultWebhookSecret, + githubCredentialsProvider, + }); }); const mockContext = { @@ -59,10 +67,6 @@ describe('github:repository:webhook:create', () => { const { mockGithubClient } = require('@octokit/rest'); - beforeEach(() => { - jest.resetAllMocks(); - }); - it('should call the githubApi for creating repository Webhook', async () => { const repoUrl = 'github.com?repo=repo&owner=owner'; const webhookUrl = 'https://example.com/payload'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 74d31ecc7a..7724dec85e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -14,13 +14,16 @@ * limitations under the License. */ +import { TemplateAction } from '../../types'; + jest.mock('../helpers'); jest.mock('@octokit/rest'); import { createPublishGithubAction } from './github'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; @@ -42,13 +45,9 @@ describe('publish:github', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); - const action = createPublishGithubAction({ - integrations, - config, - githubCredentialsProvider, - }); + let githubCredentialsProvider: GithubCredentialsProvider; + let action: TemplateAction; + const mockContext = { input: { repoUrl: 'github.com?repo=repo&owner=owner', @@ -67,6 +66,13 @@ describe('publish:github', () => { beforeEach(() => { jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createPublishGithubAction({ + integrations, + config, + githubCredentialsProvider, + }); }); it('should call the githubApis with the correct values for createInOrg', async () => {