From 04ad29ca4e0f274bdb8cfc004077ebf39e88e377 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 12 Jan 2021 16:42:39 +0100 Subject: [PATCH] Make GithubUrlReader use GithubCredentialsProvider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw --- .../src/reading/GithubUrlReader.ts | 34 +++++-- packages/integration/package.json | 3 +- ...ubApps.ts => GithubCredentialsProvider.ts} | 88 ++++++++++--------- packages/integration/src/github/config.ts | 9 +- packages/integration/src/github/index.ts | 1 + yarn.lock | 10 +++ 6 files changed, 94 insertions(+), 51 deletions(-) rename packages/integration/src/github/{githubApps.ts => GithubCredentialsProvider.ts} (74%) diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 5ca2a99692..255319dd20 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -18,7 +18,7 @@ import { GitHubIntegrationConfig, readGitHubIntegrationConfigs, getGitHubFileFetchUrl, - getGitHubRequestOptions, + GithubCredentialsProvider, } from '@backstage/integration'; import fetch from 'cross-fetch'; import parseGitUri from 'git-url-parse'; @@ -42,7 +42,11 @@ export class GithubUrlReader implements UrlReader { config.getOptionalConfigArray('integrations.github') ?? [], ); return configs.map(provider => { - const reader = new GithubUrlReader(provider, { treeResponseFactory }); + const credentialsProvider = GithubCredentialsProvider.create(provider); + const reader = new GithubUrlReader(provider, { + treeResponseFactory, + credentialsProvider, + }); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); @@ -50,7 +54,10 @@ export class GithubUrlReader implements UrlReader { constructor( private readonly config: GitHubIntegrationConfig, - private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + credentialsProvider: GithubCredentialsProvider; + }, ) { if (!config.apiBaseUrl && !config.rawBaseUrl) { throw new Error( @@ -61,11 +68,17 @@ export class GithubUrlReader implements UrlReader { async read(url: string): Promise { const ghUrl = getGitHubFileFetchUrl(url, this.config); - const options = getGitHubRequestOptions(this.config); - + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); let response: Response; try { - response = await fetch(ghUrl.toString(), options); + response = await fetch(ghUrl.toString(), { + headers: { + ...headers, + Accept: 'application/vnd.github.v3.raw', + }, + }); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -101,12 +114,19 @@ export class GithubUrlReader implements UrlReader { ); } + const { headers } = await this.deps.credentialsProvider.getCredentials({ + url, + }); // TODO(Rugvip): use API to fetch URL instead const response = await fetch( new URL( `${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`, ).toString(), - getGitHubRequestOptions(this.config), + { + headers: { + ...headers, + }, + }, ); if (!response.ok) { const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; diff --git a/packages/integration/package.json b/packages/integration/package.json index 52dc52a658..58c97ae4e5 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -34,12 +34,13 @@ "git-url-parse": "^11.4.3", "@octokit/rest": "^18.0.12", "@octokit/auth-app": "^2.10.5", - "moment": "^2.29.1" + "luxon": "^1.25.0" }, "devDependencies": { "@backstage/cli": "^0.4.5", "@backstage/test-utils": "^0.1.5", "@types/jest": "^26.0.7", + "@types/luxon": "^1.25.0", "msw": "^0.21.2" }, "files": [ diff --git a/packages/integration/src/github/githubApps.ts b/packages/integration/src/github/GithubCredentialsProvider.ts similarity index 74% rename from packages/integration/src/github/githubApps.ts rename to packages/integration/src/github/GithubCredentialsProvider.ts index 75f83e8662..490e062415 100644 --- a/packages/integration/src/github/githubApps.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.ts @@ -14,11 +14,11 @@ * limitations under the License. */ +import gitUrlParse from 'git-url-parse'; import { GithubAppConfig, GitHubIntegrationConfig } from './config'; import { createAppAuth } from '@octokit/auth-app'; import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import gitUrlParse from 'git-url-parse'; -import moment from 'moment'; +import { DateTime } from 'luxon'; import { InstallationAccessTokenAuthentication } from '@octokit/auth-app/dist-types/types'; type InstallationData = { @@ -30,26 +30,26 @@ type InstallationData = { class Cache { private readonly tokenCache = new Map< string, - { token: string; expiresAt: Date } + { token: string; expiresAt: DateTime } >(); - async getToken( + async getOrCreateToken( key: string, - fn: () => Promise<{ token: string; expiresAt: Date }>, + supplier: () => Promise<{ token: string; expiresAt: DateTime }>, ): Promise<{ accessToken: string }> { const item = this.tokenCache.get(key); if (item && this.isNotExpired(item.expiresAt)) { return { accessToken: item.token }; } - const result = await fn(); + const result = await supplier(); this.tokenCache.set(key, result); return { accessToken: result.token }; } // consider timestamps older than 50 minutes to be expired. - private isNotExpired = (date: Date) => - moment(date).isAfter(moment().subtract(50, 'minutes')); + private isNotExpired = (date: DateTime) => + date.diff(DateTime.local(), 'minutes').minutes > 50; } // GithubAppManager issues tokens for a speicifc GitHub App @@ -85,7 +85,7 @@ class GithubAppManager { // App is installed in the entire org if (repositorySelection === 'all') { - return this.cache.getToken(owner, async () => { + return this.cache.getOrCreateToken(owner, async () => { const auth = createAppAuth({ ...this.baseAuthConfig, installationId, @@ -95,19 +95,19 @@ class GithubAppManager { token, expiresAt, } = result as InstallationAccessTokenAuthentication; - return { token, expiresAt: new Date(expiresAt) }; + return { token, expiresAt: DateTime.fromISO(expiresAt) }; }); } // App is not installed org wide which requires a specific app token. - return this.cache.getToken(`${owner}/${repo}`, async () => { + return this.cache.getOrCreateToken(`${owner}/${repo}`, async () => { const result = await this.appClient.apps.createInstallationAccessToken({ installation_id: installationId, repositories: [repo], }); return { token: result.data.token, - expiresAt: new Date(result.data.expires_at), + expiresAt: DateTime.fromISO(result.data.expires_at), }; }); } @@ -150,18 +150,19 @@ class GithubAppManager { } } -// GithubIntegration corresponds to a Github installation which internally could hold several GitHub Apps. -class GithubIntegration { +// GithubAppCredentialsMux corresponds to a Github installation which internally could hold several GitHub Apps. +export class GithubAppCredentialsMux { private readonly apps: GithubAppManager[]; constructor(config: GitHubIntegrationConfig) { this.apps = config.apps?.map(ac => new GithubAppManager(ac)) ?? []; } - async getCredentialsForAppInstallation( - owner: string, - repo: string, - ): Promise<{ accessToken: string }> { + async getAppToken(owner: string, repo: string): Promise { + if (this.apps.length === 0) { + return undefined; + } + const results = await Promise.all( this.apps.map(app => app.getInstallationCredentials(owner, repo).then( @@ -172,7 +173,7 @@ class GithubIntegration { ); const result = results.find(result => result.credentials); if (result) { - return result.credentials!; + return result.credentials!.accessToken; } const errors = results.map(r => r.error); @@ -180,39 +181,42 @@ class GithubIntegration { if (notNotFoundError) { throw notNotFoundError; } - const notFoundError = new Error( - `No app installation found for ${owner}/${repo}`, - ); - notFoundError.name = 'NotFoundError'; - throw notFoundError; + + return undefined; } } -export class GithubAppAuthProvider { - private readonly integrations: Map; - - constructor(configs: GitHubIntegrationConfig[]) { - this.integrations = new Map( - configs.map(config => [config.host, new GithubIntegration(config)]), +// TODO: Possibly move this to a backend only package so that it's not used in the frontend by mistake +export class GithubCredentialsProvider { + static create(config: GitHubIntegrationConfig): GithubCredentialsProvider { + return new GithubCredentialsProvider( + new GithubAppCredentialsMux(config), + config.token, ); } - // getCredentials('github.com/backstage/somerepo') - async getCredentials(url: string): Promise<{ accessToken: string }> { - const parsed = gitUrlParse(url); + private constructor( + private readonly githubAppCredentialsMux: GithubAppCredentialsMux, + private readonly token?: string, + ) {} - const host = parsed.source; + async getCredentials(opts: { url: string }) { + const parsed = gitUrlParse(opts.url); const owner = parsed.owner; const repo = parsed.name; - const integration = await this.integrations.get(host); - const credentials = await integration?.getCredentialsForAppInstallation( - owner, - repo, - ); - if (!credentials) { - throw new Error(`No app installation found for ${owner}/${repo}`); + let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); + if (!token) { + token = this.token; } - return credentials; + + return { + headers: token + ? { + Authorization: `Bearer ${token}`, + } + : undefined, + token, + }; } } diff --git a/packages/integration/src/github/config.ts b/packages/integration/src/github/config.ts index 527e287419..eced7584a4 100644 --- a/packages/integration/src/github/config.ts +++ b/packages/integration/src/github/config.ts @@ -86,6 +86,13 @@ export function readGitHubIntegrationConfig( let apiBaseUrl = config.getOptionalString('apiBaseUrl'); let rawBaseUrl = config.getOptionalString('rawBaseUrl'); const token = config.getOptionalString('token'); + const apps = config.getOptionalConfigArray('apps')?.map(c => ({ + appId: c.getNumber('appId'), + clientId: c.getString('clientId'), + clientSecret: c.getString('clientSecret'), + webhookSecret: c.getString('webhookSecret'), + privateKey: c.getString('privateKey'), + })); if (!isValidHost(host)) { throw new Error( @@ -105,7 +112,7 @@ export function readGitHubIntegrationConfig( rawBaseUrl = GITHUB_RAW_BASE_URL; } - return { host, apiBaseUrl, rawBaseUrl, token }; + return { host, apiBaseUrl, rawBaseUrl, token, apps }; } /** diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 5f97f6980a..6491e8dcc5 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -20,3 +20,4 @@ export { } from './config'; export type { GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; +export { GithubCredentialsProvider } from './GithubCredentialsProvider'; diff --git a/yarn.lock b/yarn.lock index ea4d11c6d3..6986104789 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6634,6 +6634,11 @@ resolved "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w== +"@types/luxon@^1.25.0": + version "1.25.0" + resolved "https://registry.npmjs.org/@types/luxon/-/luxon-1.25.0.tgz#3d6fe591fac874f48dd225cb5660b2b785a21a05" + integrity sha512-iIJp2CP6C32gVqI08HIYnzqj55tlLnodIBMCcMf28q9ckqMfMzocCmIzd9JWI/ALLPMUiTkCu1JGv3FFtu6t3g== + "@types/markdown-to-jsx@^6.11.0": version "6.11.2" resolved "https://registry.npmjs.org/@types/markdown-to-jsx/-/markdown-to-jsx-6.11.2.tgz#05d1aaffbf15be7be12c70535fa4fed65cc7c64f" @@ -17662,6 +17667,11 @@ lru-queue@0.1: dependencies: es5-ext "~0.10.2" +luxon@^1.25.0: + version "1.25.0" + resolved "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz#d86219e90bc0102c0eb299d65b2f5e95efe1fe72" + integrity sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ== + macos-release@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/macos-release/-/macos-release-2.3.0.tgz#eb1930b036c0800adebccd5f17bc4c12de8bb71f"