From 57261f7e86940184853531167cde943c407d4cf7 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 23 Dec 2021 15:11:38 +0000 Subject: [PATCH 01/14] discovery processor to use gh creds interface Signed-off-by: Brian Fletcher --- packages/integration/api-report.md | 7 +++++- ...SingleInstanceGithubCredentialsProvider.ts | 5 ++-- packages/integration/src/github/index.ts | 1 + packages/integration/src/github/types.ts | 12 ++++++++++ plugins/catalog-backend/api-report.md | 8 ++++++- .../processors/GithubDiscoveryProcessor.ts | 24 ++++++++++++++++--- .../providers/GitHubOrgEntityProvider.ts | 5 +--- 7 files changed, 50 insertions(+), 12 deletions(-) diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 00623dc917..ca24848db8 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -203,6 +203,11 @@ export interface GithubCredentialsProvider { getCredentials(opts: { url: string }): Promise; } +// @public +export type GithubCredentialsProviderFactory = ( + config: GitHubIntegrationConfig, +) => GithubCredentialsProvider; + // @public export type GithubCredentialType = 'app' | 'token'; @@ -427,7 +432,7 @@ export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { // (undocumented) - static create: (config: GitHubIntegrationConfig) => GithubCredentialsProvider; + static create: GithubCredentialsProviderFactory; getCredentials(opts: { url: string }): Promise; } diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 408c5b1348..5b6fa39e75 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -22,6 +22,7 @@ import { DateTime } from 'luxon'; import { GithubCredentials, GithubCredentialsProvider, + GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; @@ -231,9 +232,7 @@ export class GithubAppCredentialsMux { export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { - static create: ( - config: GitHubIntegrationConfig, - ) => GithubCredentialsProvider = config => { + static create: GithubCredentialsProviderFactory = config => { return new SingleInstanceGithubCredentialsProvider( new GithubAppCredentialsMux(config), config.token, diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 429a2b1c76..fa8dfb4b86 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -27,6 +27,7 @@ export { export type { GithubCredentials, GithubCredentialsProvider, + GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; export { GitHubIntegration, replaceGitHubUrlType } from './GitHubIntegration'; diff --git a/packages/integration/src/github/types.ts b/packages/integration/src/github/types.ts index 15f5375570..d8c1f0ee5e 100644 --- a/packages/integration/src/github/types.ts +++ b/packages/integration/src/github/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { GitHubIntegrationConfig } from './config'; + /** * The type of credentials produced by the credential provider. * @@ -41,3 +43,13 @@ export type GithubCredentials = { export interface GithubCredentialsProvider { getCredentials(opts: { url: string }): Promise; } + +/** + * This allows implementations to be provided to create credentials providers. + * + * @public + * + */ +export type GithubCredentialsProviderFactory = ( + config: GitHubIntegrationConfig, +) => GithubCredentialsProvider; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index f773905ca0..13a7ec9a9e 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityRelationSpec } from '@backstage/catalog-model'; import express from 'express'; +import { GithubCredentialsProviderFactory } from '@backstage/integration'; import { GitHubIntegrationConfig } from '@backstage/integration'; import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/types'; @@ -1008,12 +1009,17 @@ function generalError( // // @public export class GithubDiscoveryProcessor implements CatalogProcessor { - constructor(options: { integrations: ScmIntegrations; logger: Logger_2 }); + constructor(options: { + integrations: ScmIntegrations; + logger: Logger_2; + githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; + githubCredentialsProviderFactory?: GithubCredentialsProviderFactory; }, ): GithubDiscoveryProcessor; // (undocumented) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 046455c484..b1a1a948ba 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -19,6 +19,7 @@ import { Config } from '@backstage/config'; import { SingleInstanceGithubCredentialsProvider, ScmIntegrations, + GithubCredentialsProviderFactory, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -43,19 +44,36 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; + private githubCredentialsProviderFactory: GithubCredentialsProviderFactory; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProviderFactory?: GithubCredentialsProviderFactory; + }, + ) { const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProviderFactory = + options.githubCredentialsProviderFactory || + SingleInstanceGithubCredentialsProvider.create; return new GithubDiscoveryProcessor({ ...options, integrations, + githubCredentialsProviderFactory, }); } - constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + }) { this.integrations = options.integrations; this.logger = options.logger; + this.githubCredentialsProviderFactory = + options.githubCredentialsProviderFactory; } async readLocation( @@ -84,7 +102,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { // about how to handle the wild card which is special for this processor. const orgUrl = `https://${host}/${org}`; - const { headers } = await SingleInstanceGithubCredentialsProvider.create( + const { headers } = await this.githubCredentialsProviderFactory( gitHubConfig, ).getCredentials({ url: orgUrl }); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index b8059a783c..7d49a295ef 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -28,10 +28,7 @@ import { import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; -import { - EntityProvider, - EntityProviderConnection, -} from '../../providers/types'; +import { EntityProvider, EntityProviderConnection } from '../../providers'; import { getOrganizationTeams, getOrganizationUsers, From 7af4986af8a42ffba67d14113055929c73a51b61 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 4 Jan 2022 15:30:07 +0000 Subject: [PATCH 02/14] remove factory signature from gh creds provider This change required the SingleInstanceGithubCredentialsProvider to be changed to allow it to look up the credentials from the whole list of integrations. As such all places where it was used I have updated. Signed-off-by: Brian Fletcher --- .../src/reading/GithubUrlReader.ts | 4 +- ...eInstanceGithubCredentialsProvider.test.ts | 110 ++++++++++++------ ...SingleInstanceGithubCredentialsProvider.ts | 29 ++--- packages/integration/src/github/index.ts | 1 - packages/integration/src/github/types.ts | 12 -- .../GithubDiscoveryProcessor.test.ts | 70 ++++++----- .../processors/GithubDiscoveryProcessor.ts | 22 ++-- .../GithubMultiOrgReaderProcessor.ts | 17 ++- .../GithubOrgReaderProcessor.test.ts | 49 ++++---- .../processors/GithubOrgReaderProcessor.ts | 22 +++- .../processors/UrlReaderProcessor.ts | 6 + .../providers/GitHubOrgEntityProvider.test.ts | 11 +- .../providers/GitHubOrgEntityProvider.ts | 18 +-- .../src/legacy/service/CatalogBuilder.ts | 17 ++- .../src/service/NextCatalogBuilder.ts | 18 ++- .../actions/builtin/createBuiltinActions.ts | 12 +- .../builtin/github/OctokitProvider.test.ts | 12 +- .../actions/builtin/github/OctokitProvider.ts | 27 ++--- .../github/githubActionsDispatch.test.ts | 12 +- .../builtin/github/githubActionsDispatch.ts | 13 ++- .../builtin/github/githubWebhook.test.ts | 8 +- .../actions/builtin/github/githubWebhook.ts | 14 ++- .../actions/builtin/publish/github.test.ts | 15 ++- .../actions/builtin/publish/github.ts | 13 ++- .../builtin/publish/githubPullRequest.test.ts | 9 +- .../builtin/publish/githubPullRequest.ts | 25 ++-- 26 files changed, 363 insertions(+), 203 deletions(-) diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 6b18cfbe1a..160a1a533b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -58,9 +58,9 @@ export type GhBlobResponse = export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); + const credentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); return integrations.github.list().map(integration => { - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integration.config); const reader = new GithubUrlReader(integration, { treeResponseFactory, credentialsProvider, diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts index 6bea9647a1..4278218a2c 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { ScmIntegrations } from '../ScmIntegrations'; + const octokit = { paginate: async (fn: any) => (await fn()).data, apps: { @@ -34,22 +36,33 @@ jest.doMock('@octokit/rest', () => { import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; import { RestEndpointMethodTypes } from '@octokit/rest'; import { DateTime } from 'luxon'; +import { ConfigReader } from '@backstage/config'; -const github = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', +let integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], }, - ], - token: 'hardcoded_token', -}); + }), +); -describe('DefaultGithubCredentialsProvider tests', () => { +const github = SingleInstanceGithubCredentialsProvider.create(integrations); + +describe('SingleInstanceGithubCredentialsProvider tests', () => { beforeEach(() => { jest.resetAllMocks(); }); @@ -204,11 +217,22 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return the default token if no app is configured', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [], - token: 'fallback_token', - }); + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [], + token: 'fallback_token', + }, + ], + }, + }), + ); + + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); await expect( githubProvider.getCredentials({ @@ -218,19 +242,29 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return the configured token if there are no installations', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], }, - ], - token: 'hardcoded_token', - }); + }), + ); + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); octokit.apps.listInstallations.mockResolvedValue({ data: [], } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); @@ -243,9 +277,19 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return undefined if no token or apps are configured', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - }); + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }), + ); + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); await expect( githubProvider.getCredentials({ diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 5b6fa39e75..5b7aea02ec 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -22,9 +22,9 @@ import { DateTime } from 'luxon'; import { GithubCredentials, GithubCredentialsProvider, - GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; +import { ScmIntegrations } from '../ScmIntegrations'; type InstallationData = { installationId: number; @@ -232,17 +232,11 @@ export class GithubAppCredentialsMux { export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { - static create: GithubCredentialsProviderFactory = config => { - return new SingleInstanceGithubCredentialsProvider( - new GithubAppCredentialsMux(config), - config.token, - ); - }; + static create(integrations: ScmIntegrations) { + return new SingleInstanceGithubCredentialsProvider(integrations); + } - private constructor( - private readonly githubAppCredentialsMux: GithubAppCredentialsMux, - private readonly token?: string, - ) {} + private constructor(private readonly integrations: ScmIntegrations) {} /** * Returns {@link GithubCredentials} for a given URL. @@ -266,15 +260,24 @@ export class SingleInstanceGithubCredentialsProvider */ async getCredentials(opts: { url: string }): Promise { const parsed = parseGitUrl(opts.url); + const gitHubConfig = this.integrations.github.byUrl(opts.url)?.config; + if (!gitHubConfig) { + throw new Error( + `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, + ); + } + + const githubAppCredentialsMux = new GithubAppCredentialsMux(gitHubConfig); + const defaultToken = gitHubConfig.token; const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; let type: GithubCredentialType = 'app'; - let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); + let token = await githubAppCredentialsMux.getAppToken(owner, repo); if (!token) { type = 'token'; - token = this.token; + token = defaultToken; } return { diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index fa8dfb4b86..429a2b1c76 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -27,7 +27,6 @@ export { export type { GithubCredentials, GithubCredentialsProvider, - GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; export { GitHubIntegration, replaceGitHubUrlType } from './GitHubIntegration'; diff --git a/packages/integration/src/github/types.ts b/packages/integration/src/github/types.ts index d8c1f0ee5e..15f5375570 100644 --- a/packages/integration/src/github/types.ts +++ b/packages/integration/src/github/types.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { GitHubIntegrationConfig } from './config'; - /** * The type of credentials produced by the credential provider. * @@ -43,13 +41,3 @@ export type GithubCredentials = { export interface GithubCredentialsProvider { getCredentials(opts: { url: string }): Promise; } - -/** - * This allows implementations to be provided to create credentials providers. - * - * @public - * - */ -export type GithubCredentialsProviderFactory = ( - config: GitHubIntegrationConfig, -) => GithubCredentialsProvider; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 59c25bb051..06886ecb4f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -19,6 +19,10 @@ import { LocationSpec } from '@backstage/catalog-model'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from './github'; import { ConfigReader } from '@backstage/config'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; jest.mock('./github'); const mockGetOrganizationRepositories = @@ -67,14 +71,18 @@ describe('GithubDiscoveryProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'blob' }], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); const location: LocationSpec = { type: 'not-github-discovery', target: 'https://github.com', @@ -85,17 +93,21 @@ describe('GithubDiscoveryProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { host: 'github.com', token: 'blob' }, - { host: 'ghe.example.net', token: 'blob' }, - ], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'blob' }, + { host: 'ghe.example.net', token: 'blob' }, + ], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); const location: LocationSpec = { type: 'github-discovery', target: 'https://not.github.com/apa', @@ -109,14 +121,18 @@ describe('GithubDiscoveryProcessor', () => { }); describe('handles repositories', () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'blob' }], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); beforeEach(() => { mockGetOrganizationRepositories.mockClear(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index b1a1a948ba..01ba40b1fd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -17,9 +17,8 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, ScmIntegrations, - GithubCredentialsProviderFactory, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -44,36 +43,31 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; - private githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + private readonly githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, options: { logger: Logger; - githubCredentialsProviderFactory?: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }, ) { const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProviderFactory = - options.githubCredentialsProviderFactory || - SingleInstanceGithubCredentialsProvider.create; return new GithubDiscoveryProcessor({ ...options, integrations, - githubCredentialsProviderFactory, }); } constructor(options: { integrations: ScmIntegrations; logger: Logger; - githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; - this.githubCredentialsProviderFactory = - options.githubCredentialsProviderFactory; + this.githubCredentialsProvider = options.githubCredentialsProvider; } async readLocation( @@ -102,9 +96,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { // about how to handle the wild card which is special for this processor. const orgUrl = `https://${host}/${org}`; - const { headers } = await this.githubCredentialsProviderFactory( - gitHubConfig, - ).getCredentials({ url: orgUrl }); + const { headers } = await this.githubCredentialsProvider.getCredentials({ + url: orgUrl, + }); const client = graphql.defaults({ baseUrl: gitHubConfig.apiBaseUrl, diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts index 5ad735ceeb..7a25b35078 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts @@ -18,7 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GithubAppCredentialsMux, - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, } from '@backstage/integration'; @@ -44,8 +44,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly orgs: GithubMultiOrgConfig; private readonly logger: Logger; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, + ) { const c = config.getOptionalConfig('catalog.processors.githubMultiOrg'); const integrations = ScmIntegrations.fromConfig(config); @@ -60,10 +67,12 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { integrations: ScmIntegrations; logger: Logger; orgs: GithubMultiOrgConfig; + githubCredentialsProvider: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; this.orgs = options.orgs; + this.githubCredentialsProvider = options.githubCredentialsProvider; } async readLocation( @@ -86,8 +95,6 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const allUsersMap = new Map(); const baseUrl = new URL(location.target).origin; - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(gitHubConfig); const orgsToProcess = this.orgs.length ? this.orgs @@ -96,7 +103,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { for (const orgConfig of orgsToProcess) { try { const { headers, type: tokenType } = - await credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: `${baseUrl}/${orgConfig.name}`, }); const client = graphql.defaults({ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 9124637e52..953c926c71 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -17,8 +17,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, ScmIntegrations, + GithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; @@ -39,6 +39,14 @@ describe('GithubOrgReaderProcessor', () => { }, }), ); + let githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + type: 'app', + headers: { token: 'blah' }, + }); + }, + }; beforeEach(() => { jest.resetAllMocks(); @@ -48,6 +56,7 @@ describe('GithubOrgReaderProcessor', () => { const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', @@ -61,10 +70,14 @@ describe('GithubOrgReaderProcessor', () => { }); it('should not query for email addresses when GitHub Apps is used for authentication', async () => { - const mockGetCredentials = jest.fn().mockReturnValue({ - headers: { token: 'blah' }, - type: 'app', - }); + githubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + headers: { token: 'blah' }, + type: 'app', + }); + }, + }; const mockClient = jest.fn(); @@ -87,15 +100,10 @@ describe('GithubOrgReaderProcessor', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); - const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', @@ -111,10 +119,14 @@ describe('GithubOrgReaderProcessor', () => { }); it('should query for email addresses when token is used for authentication', async () => { - const mockGetCredentials = jest.fn().mockReturnValue({ - headers: { token: 'blah' }, - type: 'token', - }); + githubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + type: 'token', + headers: { token: 'blah' }, + }); + }, + }; const mockClient = jest.fn(); @@ -137,15 +149,10 @@ describe('GithubOrgReaderProcessor', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); - const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 95db344818..4c56b30c90 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -17,9 +17,9 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialType, ScmIntegrations, + GithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -40,8 +40,15 @@ type GraphQL = typeof graphql; export class GithubOrgReaderProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, + ) { const integrations = ScmIntegrations.fromConfig(config); return new GithubOrgReaderProcessor({ @@ -50,8 +57,13 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { }); } - constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }) { this.integrations = options.integrations; + this.githubCredentialsProvider = options.githubCredentialsProvider; this.logger = options.logger; } @@ -107,10 +119,8 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { ); } - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(gitHubConfig); const { headers, type: tokenType } = - await credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: orgUrl, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 6a85cc07c7..83f22249d7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -140,3 +140,9 @@ export class UrlReaderProcessor implements CatalogProcessor { return { response: [{ url: location, data }] }; } } + +export class RoadieDemoDataReaderProcessor extends UrlReaderProcessor { + getProcessorName() { + return 'roadie-demo-data-reader'; + } +} diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts index 6fde975a2b..e7aec1520d 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; import { GitHubOrgEntityProvider } from '.'; @@ -93,14 +93,13 @@ describe('GitHubOrgEntityProvider', () => { type: 'app', }); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; const entityProvider = new GitHubOrgEntityProvider({ id: 'my-id', + githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index 7d49a295ef..bc40ca1da5 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -20,7 +20,6 @@ import { } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, @@ -40,11 +39,16 @@ import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; export class GitHubOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; - private readonly credentialsProvider: GithubCredentialsProvider; + private githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, - options: { id: string; orgUrl: string; logger: Logger }, + options: { + id: string; + orgUrl: string; + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, ) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -64,6 +68,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: options.orgUrl, logger, gitHubConfig, + githubCredentialsProvider: options.githubCredentialsProvider, }); } @@ -73,11 +78,10 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: string; gitHubConfig: GitHubIntegrationConfig; logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; }, ) { - this.credentialsProvider = SingleInstanceGithubCredentialsProvider.create( - options.gitHubConfig, - ); + this.githubCredentialsProvider = options.githubCredentialsProvider; } getProviderName() { @@ -96,7 +100,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { const { markReadComplete } = trackProgress(this.options.logger); const { headers, type: tokenType } = - await this.credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: this.options.orgUrl, }); const client = graphql.defaults({ diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index 6fb52e245e..6fda605ae3 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -24,7 +24,10 @@ import { SchemaValidEntityPolicy, Validators, } from '@backstage/catalog-model'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import lodash from 'lodash'; import { EntitiesCatalog } from '../../catalog'; import { @@ -291,6 +294,8 @@ export class CatalogBuilder { private buildProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); this.checkDeprecatedReaderProcessors(); @@ -317,9 +322,15 @@ export class CatalogBuilder { processors.push( new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), - GithubDiscoveryProcessor.fromConfig(config, { logger }), + GithubDiscoveryProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), - GithubOrgReaderProcessor.fromConfig(config, { logger }), + GithubOrgReaderProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index 1618ef8d7b..24aefb2d96 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -25,7 +25,11 @@ import { SchemaValidEntityPolicy, Validators, } from '@backstage/catalog-model'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; import lodash from 'lodash'; @@ -289,13 +293,21 @@ export class NextCatalogBuilder { getDefaultProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider: GithubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); return [ new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), - GithubDiscoveryProcessor.fromConfig(config, { logger }), - GithubOrgReaderProcessor.fromConfig(config, { logger }), + GithubDiscoveryProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), + GithubOrgReaderProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 93ecceb76a..6975f1aa16 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -16,7 +16,11 @@ import { ContainerRunner, UrlReader } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { Config } from '@backstage/config'; import { createCatalogWriteAction, @@ -52,6 +56,8 @@ export const createBuiltinActions = (options: { }) => { const { reader, integrations, containerRunner, catalogClient, config } = options; + const githubCredentialsProvider: GithubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); const actions = [ createFetchPlainAction({ @@ -65,9 +71,11 @@ export const createBuiltinActions = (options: { createPublishGithubAction({ integrations, config, + githubCredentialsProvider, }), createPublishGithubPullRequestAction({ integrations, + githubCredentialsProvider, }), createPublishGitlabAction({ integrations, @@ -91,9 +99,11 @@ export const createBuiltinActions = (options: { createFilesystemRenameAction(), createGithubActionsDispatchAction({ integrations, + githubCredentialsProvider, }), createGithubWebhookAction({ integrations, + githubCredentialsProvider, }), ]; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts index 91b23f40dd..39a1ff6ad8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts @@ -15,7 +15,10 @@ */ import { OctokitProvider } from './OctokitProvider'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; describe('getOctokit', () => { @@ -29,7 +32,12 @@ describe('getOctokit', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const octokitProvider = new OctokitProvider(integrations); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); beforeEach(() => { jest.resetAllMocks(); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts index 9fdc6ced17..4a8bb8bbf6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts @@ -16,7 +16,6 @@ import { InputError } from '@backstage/errors'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; @@ -35,18 +34,14 @@ export type OctokitIntegration = { */ export class OctokitProvider { private readonly integrations: ScmIntegrationRegistry; - private readonly credentialsProviders: Map; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - constructor(integrations: ScmIntegrationRegistry) { + constructor( + integrations: ScmIntegrationRegistry, + githubCredentialsProvider: GithubCredentialsProvider, + ) { this.integrations = integrations; - this.credentialsProviders = new Map( - integrations.github.list().map(integration => { - const provider = SingleInstanceGithubCredentialsProvider.create( - integration.config, - ); - return [integration.config.host, provider]; - }), - ); + this.githubCredentialsProvider = githubCredentialsProvider; } /** @@ -67,17 +62,9 @@ export class OctokitProvider { throw new InputError(`No integration for host ${host}`); } - const credentialsProvider = this.credentialsProviders.get(host); - - if (!credentialsProvider) { - throw new InputError( - `No matching credentials for host ${host}, please check your integrations config`, - ); - } - // TODO(blam): Consider changing this API to have owner, repo interface instead of URL as the it's // needless to create URL and then parse again the other side. - const { token } = await credentialsProvider.getCredentials({ + const { token } = await this.githubCredentialsProvider.getCredentials({ url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent( repo, )}`, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts index ab1bbfc488..6720de4e95 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts @@ -17,7 +17,10 @@ jest.mock('@octokit/rest'); import { createGithubActionsDispatchAction } from './githubActionsDispatch'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -33,7 +36,12 @@ describe('github:actions:dispatch', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const action = createGithubActionsDispatchAction({ integrations }); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const action = createGithubActionsDispatchAction({ + integrations, + githubCredentialsProvider, + }); const mockContext = { input: { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts index 661101d184..a88698ed07 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts @@ -13,15 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; export function createGithubActionsDispatchAction(options: { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, githubCredentialsProvider } = options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); return createTemplateAction<{ repoUrl: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts index 0da08bbf9d..0c2daeb42f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts @@ -17,7 +17,10 @@ jest.mock('@octokit/rest'); import { createGithubWebhookAction } from './githubWebhook'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -33,10 +36,13 @@ describe('github:repository:webhook:create', () => { }); const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); const defaultWebhookSecret = 'aafdfdivierernfdk23f'; const action = createGithubWebhookAction({ integrations, defaultWebhookSecret, + githubCredentialsProvider, }); const mockContext = { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts index 0761f65f09..d341b59011 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; import { emitterEventNames } from '@octokit/webhooks'; @@ -24,9 +27,14 @@ type ContentType = 'form' | 'json'; export function createGithubWebhookAction(options: { integrations: ScmIntegrationRegistry; defaultWebhookSecret?: string; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations, defaultWebhookSecret } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, defaultWebhookSecret, githubCredentialsProvider } = + options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); const eventNames = emitterEventNames.filter(event => !event.includes('.')); return createTemplateAction<{ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 6a3ccf5e38..74d31ecc7a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -18,7 +18,10 @@ jest.mock('../helpers'); jest.mock('@octokit/rest'); import { createPublishGithubAction } from './github'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -39,7 +42,13 @@ describe('publish:github', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const action = createPublishGithubAction({ integrations, config }); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const action = createPublishGithubAction({ + integrations, + config, + githubCredentialsProvider, + }); const mockContext = { input: { repoUrl: 'github.com?repo=repo&owner=owner', @@ -201,6 +210,7 @@ describe('publish:github', () => { const customAuthorAction = createPublishGithubAction({ integrations: customAuthorIntegrations, config: customAuthorConfig, + githubCredentialsProvider, }); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -244,6 +254,7 @@ describe('publish:github', () => { const customAuthorAction = createPublishGithubAction({ integrations: customAuthorIntegrations, config: customAuthorConfig, + githubCredentialsProvider, }); mockGithubClient.users.getByUsername.mockResolvedValue({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index ad6a4a7857..66a45eaba1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { enableBranchProtectionOnDefaultRepoBranch, initRepoAndPush, @@ -30,9 +33,13 @@ type Collaborator = { access: Permission; username: string }; export function createPublishGithubAction(options: { integrations: ScmIntegrationRegistry; config: Config; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations, config } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, config, githubCredentialsProvider } = options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); return createTemplateAction<{ repoUrl: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts index 0d4deb87e0..20ea4b03da 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts @@ -16,7 +16,10 @@ import { getRootLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; import mockFs from 'mock-fs'; import os from 'os'; import { resolve as resolvePath } from 'path'; @@ -53,9 +56,13 @@ describe('createPublishGithubPullRequestAction', () => { }), }; clientFactory = jest.fn(async () => fakeClient); + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: jest.fn(), + }; instance = createPublishGithubPullRequestAction({ integrations, + githubCredentialsProvider, clientFactory, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts index 299eb07f2e..e657c36e50 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts @@ -18,7 +18,7 @@ import fs from 'fs-extra'; import { parseRepoUrl, isExecutable } from './util'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; import { zipObject } from 'lodash'; @@ -58,6 +58,7 @@ export type GithubPullRequestActionInput = { export type ClientFactoryInput = { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; host: string; owner: string; repo: string; @@ -65,6 +66,7 @@ export type ClientFactoryInput = { export const defaultClientFactory = async ({ integrations, + githubCredentialsProvider, owner, repo, host = 'github.com', @@ -75,16 +77,7 @@ export const defaultClientFactory = async ({ throw new InputError(`No integration for host ${host}`); } - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrationConfig); - - if (!credentialsProvider) { - throw new InputError( - `No matching credentials for host ${host}, please check your integrations config`, - ); - } - - const { token } = await credentialsProvider.getCredentials({ + const { token } = await githubCredentialsProvider.getCredentials({ url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent( repo, )}`, @@ -106,11 +99,13 @@ export const defaultClientFactory = async ({ interface CreateGithubPullRequestActionOptions { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; clientFactory?: (input: ClientFactoryInput) => Promise; } export const createPublishGithubPullRequestAction = ({ integrations, + githubCredentialsProvider, clientFactory = defaultClientFactory, }: CreateGithubPullRequestActionOptions) => { return createTemplateAction({ @@ -183,7 +178,13 @@ export const createPublishGithubPullRequestAction = ({ ); } - const client = await clientFactory({ integrations, host, owner, repo }); + const client = await clientFactory({ + integrations, + githubCredentialsProvider, + host, + owner, + repo, + }); const fileRoot = sourcePath ? resolveSafeChildPath(ctx.workspacePath, sourcePath) : ctx.workspacePath; From bb8ce88f2ea27d0cf4db945d0b662942976d8ed7 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 4 Jan 2022 15:57:07 +0000 Subject: [PATCH 03/14] api reports Signed-off-by: Brian Fletcher --- packages/integration/api-report.md | 9 +++------ plugins/catalog-backend/api-report.md | 19 ++++++++++++++----- plugins/scaffolder-backend/api-report.md | 10 +++++++++- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index ca24848db8..efcaa07c88 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -203,11 +203,6 @@ export interface GithubCredentialsProvider { getCredentials(opts: { url: string }): Promise; } -// @public -export type GithubCredentialsProviderFactory = ( - config: GitHubIntegrationConfig, -) => GithubCredentialsProvider; - // @public export type GithubCredentialType = 'app' | 'token'; @@ -432,7 +427,9 @@ export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { // (undocumented) - static create: GithubCredentialsProviderFactory; + static create( + integrations: ScmIntegrations, + ): SingleInstanceGithubCredentialsProvider; getCredentials(opts: { url: string }): Promise; } diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 13a7ec9a9e..1fe38051c9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,7 +16,7 @@ import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityRelationSpec } from '@backstage/catalog-model'; import express from 'express'; -import { GithubCredentialsProviderFactory } from '@backstage/integration'; +import { GithubCredentialsProvider } from '@backstage/integration'; import { GitHubIntegrationConfig } from '@backstage/integration'; import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/types'; @@ -1012,14 +1012,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; logger: Logger_2; - githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; - githubCredentialsProviderFactory?: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }, ): GithubDiscoveryProcessor; // (undocumented) @@ -1036,12 +1036,14 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { integrations: ScmIntegrations; logger: Logger_2; orgs: GithubMultiOrgConfig; + githubCredentialsProvider: GithubCredentialsProvider; }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; + githubCredentialsProvider: GithubCredentialsProvider; }, ): GithubMultiOrgReaderProcessor; // (undocumented) @@ -1061,6 +1063,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: string; gitHubConfig: GitHubIntegrationConfig; logger: Logger_2; + githubCredentialsProvider: GithubCredentialsProvider; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -1071,6 +1074,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { id: string; orgUrl: string; logger: Logger_2; + githubCredentialsProvider: GithubCredentialsProvider; }, ): GitHubOrgEntityProvider; // (undocumented) @@ -1083,12 +1087,17 @@ export class GitHubOrgEntityProvider implements EntityProvider { // // @public export class GithubOrgReaderProcessor implements CatalogProcessor { - constructor(options: { integrations: ScmIntegrations; logger: Logger_2 }); + constructor(options: { + integrations: ScmIntegrations; + logger: Logger_2; + githubCredentialsProvider: GithubCredentialsProvider; + }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; + githubCredentialsProvider: GithubCredentialsProvider; }, ): GithubOrgReaderProcessor; // (undocumented) @@ -1577,7 +1586,7 @@ export class UrlReaderProcessor implements CatalogProcessor { // src/catalog/types.d.ts:94:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/catalog/types.d.ts:95:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/catalog/types.d.ts:96:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts +// src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:25:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts // src/ingestion/types.d.ts:8:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/legacy/database/types.d.ts:98:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/legacy/database/types.d.ts:104:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index b7c1a9146c..a292388909 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -14,6 +14,7 @@ import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-back import { createPullRequest } from 'octokit-plugin-create-pull-request'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; +import { GithubCredentialsProvider } from '@backstage/integration'; import { JsonObject } from '@backstage/types'; import { JsonValue } from '@backstage/types'; import { Knex } from 'knex'; @@ -123,6 +124,7 @@ export const createFilesystemRenameAction: () => TemplateAction; // @public (undocumented) export function createGithubActionsDispatchAction(options: { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-missing-release-tag) "createGithubWebhookAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -131,6 +133,7 @@ export function createGithubActionsDispatchAction(options: { export function createGithubWebhookAction(options: { integrations: ScmIntegrationRegistry; defaultWebhookSecret?: string; + githubCredentialsProvider: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-missing-release-tag) "createPublishAzureAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -160,6 +163,7 @@ export function createPublishFileAction(): TemplateAction; export function createPublishGithubAction(options: { integrations: ScmIntegrationRegistry; config: Config; + githubCredentialsProvider: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-forgotten-export) The symbol "CreateGithubPullRequestActionOptions" needs to be exported by the entry point index.d.ts @@ -168,6 +172,7 @@ export function createPublishGithubAction(options: { // @public (undocumented) export const createPublishGithubPullRequestAction: ({ integrations, + githubCredentialsProvider, clientFactory, }: CreateGithubPullRequestActionOptions) => TemplateAction; @@ -281,7 +286,10 @@ export function fetchContents({ // // @public export class OctokitProvider { - constructor(integrations: ScmIntegrationRegistry); + constructor( + integrations: ScmIntegrationRegistry, + githubCredentialsProvider: GithubCredentialsProvider, + ); // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-forgotten-export) The symbol "OctokitIntegration" needs to be exported by the entry point index.d.ts getOctokit(repoUrl: string): Promise; From 595dab2935701efd0d2328ca43d3c1d92ce87c1f Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 5 Jan 2022 08:23:04 +0000 Subject: [PATCH 04/14] Creates new provider for compatability There is now two credentials providers: - SingleInstanceGithubCredentialsProvider can be created with a single GitHubIntegrationConfig. - DefaultGithubCredentialsProvider is created from the full integrations config. Signed-off-by: Brian Fletcher --- .../src/reading/GithubUrlReader.ts | 4 +- .../DefaultGithubCredentialsProvider.test.ts | 347 ++++++++++++++++++ .../DefaultGithubCredentialsProvider.ts | 111 ++++++ .../src/github/GithubAppCredentialsMux.ts | 214 +++++++++++ ...eInstanceGithubCredentialsProvider.test.ts | 113 ++---- ...SingleInstanceGithubCredentialsProvider.ts | 234 +----------- packages/integration/src/github/index.ts | 8 +- .../GithubDiscoveryProcessor.test.ts | 8 +- .../src/legacy/service/CatalogBuilder.ts | 4 +- .../src/service/NextCatalogBuilder.ts | 4 +- .../actions/builtin/createBuiltinActions.ts | 4 +- .../builtin/github/OctokitProvider.test.ts | 4 +- .../github/githubActionsDispatch.test.ts | 20 +- .../builtin/github/githubWebhook.test.ts | 26 +- .../actions/builtin/publish/github.test.ts | 22 +- 15 files changed, 785 insertions(+), 338 deletions(-) create mode 100644 packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts create mode 100644 packages/integration/src/github/DefaultGithubCredentialsProvider.ts create mode 100644 packages/integration/src/github/GithubAppCredentialsMux.ts diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 160a1a533b..59e136c138 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -16,7 +16,7 @@ import { getGitHubFileFetchUrl, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, GithubCredentialsProvider, GitHubIntegration, ScmIntegrations, @@ -59,7 +59,7 @@ export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); return integrations.github.list().map(integration => { const reader = new GithubUrlReader(integration, { treeResponseFactory, diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts new file mode 100644 index 0000000000..f537fee783 --- /dev/null +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts @@ -0,0 +1,347 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScmIntegrations } from '../ScmIntegrations'; + +const octokit = { + paginate: async (fn: any) => (await fn()).data, + apps: { + listInstallations: jest.fn(), + createInstallationAccessToken: jest.fn(), + }, +}; + +jest.doMock('@octokit/rest', () => { + class Octokit { + constructor() { + return octokit; + } + } + return { Octokit }; +}); + +import { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; +import { RestEndpointMethodTypes } from '@octokit/rest'; +import { DateTime } from 'luxon'; +import { ConfigReader } from '@backstage/config'; + +let integrations: ScmIntegrations; + +describe('DefaultGithubCredentialsProvider tests', () => { + beforeEach(() => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], + }, + }), + ); + jest.resetAllMocks(); + }); + + it('create repository specific tokens', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'selected', + account: null, + }, + { + id: 2, + repository_selection: 'selected', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + const { token, headers, type } = await github.getCredentials({ + url: 'https://github.com/backstage/foobar', + }); + expect(type).toEqual('app'); + expect(token).toEqual('secret_token'); + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + + // fallback to the configured token if no application is matching + await expect( + github.getCredentials({ + url: 'https://github.com/404/foobar', + }), + ).resolves.toEqual({ + headers: { + Authorization: 'Bearer hardcoded_token', + }, + token: 'hardcoded_token', + type: 'token', + }); + }); + + it('creates tokens for an organization', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'all', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + expect(token).toEqual('secret_token'); + }); + + it('should not fail to issue tokens for an organization when the app is installed for a single repo', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'selected', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + const expectedToken = 'secret_token'; + expect(headers).toEqual({ Authorization: `Bearer ${expectedToken}` }); + expect(token).toEqual('secret_token'); + }); + + it('should throw if the app is suspended', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + suspended_by: { + login: 'admin', + }, + repository_selection: 'all', + account: { + login: 'backstage', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + github.getCredentials({ + url: 'https://github.com/backstage', + }), + ).rejects.toThrow('The GitHub application for backstage is suspended'); + }); + + it('should return the default token when the call to github return a status that is not recognized', async () => { + octokit.apps.listInstallations.mockRejectedValue({ + status: 404, + message: 'NotFound', + }); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + await expect( + github.getCredentials({ + url: 'https://github.com/backstage', + }), + ).rejects.toEqual({ status: 404, message: 'NotFound' }); + }); + + it('should return the default token if no app is configured', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [], + token: 'fallback_token', + }, + ], + }, + }), + ); + + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/404/foobar', + }), + ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' })); + }); + + it('should return the configured token if there are no installations', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], + }, + }), + ); + + octokit.apps.listInstallations.mockResolvedValue({ + data: [], + } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/backstage', + }), + ).resolves.toEqual(expect.objectContaining({ token: 'hardcoded_token' })); + }); + + it('should return undefined if no token or apps are configured', async () => { + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }), + ); + const githubProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + + await expect( + githubProvider.getCredentials({ + url: 'https://github.com/backstage', + }), + ).resolves.toEqual({ headers: undefined, token: undefined, type: 'token' }); + }); + + it('should to create a token for the organization ignoring case sensitive', async () => { + octokit.apps.listInstallations.mockResolvedValue({ + headers: { + etag: '123', + }, + data: [ + { + id: 1, + repository_selection: 'all', + account: { + login: 'BACKSTAGE', + }, + }, + ], + } as RestEndpointMethodTypes['apps']['listInstallations']['response']); + + octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ + data: { + expires_at: DateTime.local().plus({ hours: 1 }).toString(), + token: 'secret_token', + }, + } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); + + const github = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const { token, headers } = await github.getCredentials({ + url: 'https://github.com/backstage', + }); + + expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); + expect(token).toEqual('secret_token'); + }); +}); diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts new file mode 100644 index 0000000000..37aa23698e --- /dev/null +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import parseGitUrl from 'git-url-parse'; +import { + GithubCredentials, + GithubCredentialsProvider, + GithubCredentialType, +} from './types'; +import { ScmIntegrations } from '../ScmIntegrations'; +import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; + +type MuxCollection = { + [url: string]: { + githubAppCredentialsMux: GithubAppCredentialsMux; + token?: string; + }; +}; + +/** + * Handles the creation and caching of credentials for GitHub integrations. + * + * @public + * @remarks + * + * TODO: Possibly move this to a backend only package so that it's not used in the frontend by mistake + */ +export class DefaultGithubCredentialsProvider + implements GithubCredentialsProvider +{ + static fromIntegrations(integrations: ScmIntegrations) { + const muxen: MuxCollection = {}; + + integrations.github.list().forEach(integration => { + muxen[integration.config.host] = { + githubAppCredentialsMux: new GithubAppCredentialsMux( + integration.config, + ), + token: integration.config.token, + }; + }); + return new DefaultGithubCredentialsProvider(muxen); + } + + private constructor( + private readonly githubAppCredentialsMuxen: MuxCollection, + ) {} + + /** + * Returns {@link GithubCredentials} for a given URL. + * + * @remarks + * + * Consecutive calls to this method with the same URL will return cached + * credentials. + * + * The shortest lifetime for a token returned is 10 minutes. + * + * @example + * ```ts + * const { token, headers } = await getCredentials({ + * url: 'github.com/backstage/foobar' + * }) + * ``` + * + * @param opts - The organization or repository URL + * @returns A promise of {@link GithubCredentials}. + */ + async getCredentials(opts: { url: string }): Promise { + const parsed = parseGitUrl(opts.url); + + if (!this.githubAppCredentialsMuxen[parsed.resource]) { + throw new Error( + `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, + ); + } + + const githubAppCredentialsMux = + this.githubAppCredentialsMuxen[parsed.resource].githubAppCredentialsMux; + const defaultToken = this.githubAppCredentialsMuxen[parsed.resource].token; + + const owner = parsed.owner || parsed.name; + const repo = parsed.owner ? parsed.name : undefined; + + let type: GithubCredentialType = 'app'; + let token = await githubAppCredentialsMux.getAppToken(owner, repo); + if (!token) { + type = 'token'; + token = defaultToken; + } + + return { + headers: token ? { Authorization: `Bearer ${token}` } : undefined, + token, + type, + }; + } +} diff --git a/packages/integration/src/github/GithubAppCredentialsMux.ts b/packages/integration/src/github/GithubAppCredentialsMux.ts new file mode 100644 index 0000000000..c6d055a213 --- /dev/null +++ b/packages/integration/src/github/GithubAppCredentialsMux.ts @@ -0,0 +1,214 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GithubAppConfig, GitHubIntegrationConfig } from './config'; +import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; +import { createAppAuth } from '@octokit/auth-app'; +import { DateTime } from 'luxon'; + +/** + * This accept header is required when calling App APIs in GitHub Enterprise. + * It has no effect on calls to github.com and can probably be removed entirely + * once GitHub Apps is out of preview. + */ +const HEADERS = { + Accept: 'application/vnd.github.machine-man-preview+json', +}; + +type InstallationData = { + installationId: number; + suspended: boolean; +}; + +class Cache { + private readonly tokenCache = new Map< + string, + { token: string; expiresAt: DateTime } + >(); + + async getOrCreateToken( + key: string, + supplier: () => Promise<{ token: string; expiresAt: DateTime }>, + ): Promise<{ accessToken: string }> { + const item = this.tokenCache.get(key); + if (item && this.isNotExpired(item.expiresAt)) { + return { accessToken: item.token }; + } + + const result = await supplier(); + this.tokenCache.set(key, result); + return { accessToken: result.token }; + } + + // consider timestamps older than 50 minutes to be expired. + private isNotExpired = (date: DateTime) => + date.diff(DateTime.local(), 'minutes').minutes > 50; +} + +/** + * GithubAppManager issues and caches tokens for a specific GitHub App. + */ +class GithubAppManager { + private readonly appClient: Octokit; + private readonly baseUrl?: string; + private readonly baseAuthConfig: { appId: number; privateKey: string }; + private readonly cache = new Cache(); + private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations + + constructor(config: GithubAppConfig, baseUrl?: string) { + this.allowedInstallationOwners = config.allowedInstallationOwners; + this.baseUrl = baseUrl; + this.baseAuthConfig = { + appId: config.appId, + privateKey: config.privateKey.replace(/\\n/gm, '\n'), + }; + this.appClient = new Octokit({ + baseUrl, + headers: HEADERS, + authStrategy: createAppAuth, + auth: this.baseAuthConfig, + }); + } + + async getInstallationCredentials( + owner: string, + repo?: string, + ): Promise<{ accessToken: string }> { + const { installationId, suspended } = await this.getInstallationData(owner); + if (this.allowedInstallationOwners) { + if (!this.allowedInstallationOwners?.includes(owner)) { + throw new Error( + `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, + ); + } + } + if (suspended) { + throw new Error(`The GitHub application for ${owner} is suspended`); + } + + const cacheKey = repo ? `${owner}/${repo}` : owner; + + // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. + return this.cache.getOrCreateToken(cacheKey, async () => { + const result = await this.appClient.apps.createInstallationAccessToken({ + installation_id: installationId, + headers: HEADERS, + }); + if (repo && result.data.repository_selection === 'selected') { + const installationClient = new Octokit({ + baseUrl: this.baseUrl, + auth: result.data.token, + }); + const repos = await installationClient.paginate( + installationClient.apps.listReposAccessibleToInstallation, + ); + const hasRepo = repos.some(repository => { + return repository.name === repo; + }); + if (!hasRepo) { + throw new Error( + `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, + ); + } + } + return { + token: result.data.token, + expiresAt: DateTime.fromISO(result.data.expires_at), + }; + }); + } + + getInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + return this.appClient.paginate(this.appClient.apps.listInstallations); + } + + private async getInstallationData(owner: string): Promise { + const allInstallations = await this.getInstallations(); + const installation = allInstallations.find( + inst => + inst.account?.login?.toLocaleLowerCase('en-US') === + owner.toLocaleLowerCase('en-US'), + ); + if (installation) { + return { + installationId: installation.id, + suspended: Boolean(installation.suspended_by), + }; + } + const notFoundError = new Error( + `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, + ); + notFoundError.name = 'NotFoundError'; + throw notFoundError; + } +} + +/** + * Corresponds to a Github installation which internally could hold several GitHub Apps. + * + * @public + */ +export class GithubAppCredentialsMux { + private readonly apps: GithubAppManager[]; + + constructor(config: GitHubIntegrationConfig) { + this.apps = + config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; + } + + async getAllInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + if (!this.apps.length) { + return []; + } + + const installs = await Promise.all( + this.apps.map(app => app.getInstallations()), + ); + + return installs.flat(); + } + + async getAppToken(owner: string, repo?: string): Promise { + if (this.apps.length === 0) { + return undefined; + } + + const results = await Promise.all( + this.apps.map(app => + app.getInstallationCredentials(owner, repo).then( + credentials => ({ credentials, error: undefined }), + error => ({ credentials: undefined, error }), + ), + ), + ); + + const result = results.find(resultItem => resultItem.credentials); + if (result) { + return result.credentials!.accessToken; + } + + const errors = results.map(r => r.error); + const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); + if (notNotFoundError) { + throw notNotFoundError; + } + + return undefined; + } +} diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts index 4278218a2c..e5c6307338 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ScmIntegrations } from '../ScmIntegrations'; +import { GithubCredentialsProvider } from './types'; const octokit = { paginate: async (fn: any) => (await fn()).data, @@ -36,35 +36,25 @@ jest.doMock('@octokit/rest', () => { import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; import { RestEndpointMethodTypes } from '@octokit/rest'; import { DateTime } from 'luxon'; -import { ConfigReader } from '@backstage/config'; - -let integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', - }, - ], - token: 'hardcoded_token', - }, - ], - }, - }), -); - -const github = SingleInstanceGithubCredentialsProvider.create(integrations); describe('SingleInstanceGithubCredentialsProvider tests', () => { + let github: GithubCredentialsProvider; + beforeEach(() => { jest.resetAllMocks(); + github = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }); }); it('create repository specific tokens', async () => { octokit.apps.listInstallations.mockResolvedValue({ @@ -217,22 +207,11 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return the default token if no app is configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [], - token: 'fallback_token', - }, - ], - }, - }), - ); - - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [], + token: 'fallback_token', + }); await expect( githubProvider.getCredentials({ @@ -242,29 +221,19 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return the configured token if there are no installations', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', - }, - ], - token: 'hardcoded_token', - }, - ], + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', }, - }), - ); - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + ], + token: 'hardcoded_token', + }); octokit.apps.listInstallations.mockResolvedValue({ data: [], } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); @@ -277,19 +246,9 @@ describe('SingleInstanceGithubCredentialsProvider tests', () => { }); it('should return undefined if no token or apps are configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - }, - ], - }, - }), - ); - const githubProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + const githubProvider = SingleInstanceGithubCredentialsProvider.create({ + host: 'github.com', + }); await expect( githubProvider.getCredentials({ diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 5b7aea02ec..5f04a3ed6e 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,211 +15,14 @@ */ import parseGitUrl from 'git-url-parse'; -import { GithubAppConfig, GitHubIntegrationConfig } from './config'; -import { createAppAuth } from '@octokit/auth-app'; -import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import { DateTime } from 'luxon'; + import { GithubCredentials, GithubCredentialsProvider, GithubCredentialType, } from './types'; -import { ScmIntegrations } from '../ScmIntegrations'; - -type InstallationData = { - installationId: number; - suspended: boolean; -}; - -class Cache { - private readonly tokenCache = new Map< - string, - { token: string; expiresAt: DateTime } - >(); - - async getOrCreateToken( - key: string, - supplier: () => Promise<{ token: string; expiresAt: DateTime }>, - ): Promise<{ accessToken: string }> { - const item = this.tokenCache.get(key); - if (item && this.isNotExpired(item.expiresAt)) { - return { accessToken: item.token }; - } - - const result = await supplier(); - this.tokenCache.set(key, result); - return { accessToken: result.token }; - } - - // consider timestamps older than 50 minutes to be expired. - private isNotExpired = (date: DateTime) => - date.diff(DateTime.local(), 'minutes').minutes > 50; -} - -/** - * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entirely - * once GitHub Apps is out of preview. - */ -const HEADERS = { - Accept: 'application/vnd.github.machine-man-preview+json', -}; - -/** - * GithubAppManager issues and caches tokens for a specific GitHub App. - */ -class GithubAppManager { - private readonly appClient: Octokit; - private readonly baseUrl?: string; - private readonly baseAuthConfig: { appId: number; privateKey: string }; - private readonly cache = new Cache(); - private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations - - constructor(config: GithubAppConfig, baseUrl?: string) { - this.allowedInstallationOwners = config.allowedInstallationOwners; - this.baseUrl = baseUrl; - this.baseAuthConfig = { - appId: config.appId, - privateKey: config.privateKey.replace(/\\n/gm, '\n'), - }; - this.appClient = new Octokit({ - baseUrl, - headers: HEADERS, - authStrategy: createAppAuth, - auth: this.baseAuthConfig, - }); - } - - async getInstallationCredentials( - owner: string, - repo?: string, - ): Promise<{ accessToken: string }> { - const { installationId, suspended } = await this.getInstallationData(owner); - if (this.allowedInstallationOwners) { - if (!this.allowedInstallationOwners?.includes(owner)) { - throw new Error( - `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, - ); - } - } - if (suspended) { - throw new Error(`The GitHub application for ${owner} is suspended`); - } - - const cacheKey = repo ? `${owner}/${repo}` : owner; - - // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. - return this.cache.getOrCreateToken(cacheKey, async () => { - const result = await this.appClient.apps.createInstallationAccessToken({ - installation_id: installationId, - headers: HEADERS, - }); - if (repo && result.data.repository_selection === 'selected') { - const installationClient = new Octokit({ - baseUrl: this.baseUrl, - auth: result.data.token, - }); - const repos = await installationClient.paginate( - installationClient.apps.listReposAccessibleToInstallation, - ); - const hasRepo = repos.some(repository => { - return repository.name === repo; - }); - if (!hasRepo) { - throw new Error( - `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, - ); - } - } - return { - token: result.data.token, - expiresAt: DateTime.fromISO(result.data.expires_at), - }; - }); - } - - getInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - return this.appClient.paginate(this.appClient.apps.listInstallations); - } - - private async getInstallationData(owner: string): Promise { - const allInstallations = await this.getInstallations(); - const installation = allInstallations.find( - inst => - inst.account?.login?.toLocaleLowerCase('en-US') === - owner.toLocaleLowerCase('en-US'), - ); - if (installation) { - return { - installationId: installation.id, - suspended: Boolean(installation.suspended_by), - }; - } - const notFoundError = new Error( - `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, - ); - notFoundError.name = 'NotFoundError'; - throw notFoundError; - } -} - -/** - * Corresponds to a Github installation which internally could hold several GitHub Apps. - * - * @public - */ -export class GithubAppCredentialsMux { - private readonly apps: GithubAppManager[]; - - constructor(config: GitHubIntegrationConfig) { - this.apps = - config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; - } - - async getAllInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - if (!this.apps.length) { - return []; - } - - const installs = await Promise.all( - this.apps.map(app => app.getInstallations()), - ); - - return installs.flat(); - } - - async getAppToken(owner: string, repo?: string): Promise { - if (this.apps.length === 0) { - return undefined; - } - - const results = await Promise.all( - this.apps.map(app => - app.getInstallationCredentials(owner, repo).then( - credentials => ({ credentials, error: undefined }), - error => ({ credentials: undefined, error }), - ), - ), - ); - - const result = results.find(resultItem => resultItem.credentials); - if (result) { - return result.credentials!.accessToken; - } - - const errors = results.map(r => r.error); - const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); - if (notNotFoundError) { - throw notNotFoundError; - } - - return undefined; - } -} +import { GitHubIntegrationConfig } from './config'; +import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; /** * Handles the creation and caching of credentials for GitHub integrations. @@ -232,11 +35,19 @@ export class GithubAppCredentialsMux { export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { - static create(integrations: ScmIntegrations) { - return new SingleInstanceGithubCredentialsProvider(integrations); - } + static create: ( + config: GitHubIntegrationConfig, + ) => GithubCredentialsProvider = config => { + return new SingleInstanceGithubCredentialsProvider( + new GithubAppCredentialsMux(config), + config.token, + ); + }; - private constructor(private readonly integrations: ScmIntegrations) {} + private constructor( + private readonly githubAppCredentialsMux: GithubAppCredentialsMux, + private readonly token?: string, + ) {} /** * Returns {@link GithubCredentials} for a given URL. @@ -260,24 +71,15 @@ export class SingleInstanceGithubCredentialsProvider */ async getCredentials(opts: { url: string }): Promise { const parsed = parseGitUrl(opts.url); - const gitHubConfig = this.integrations.github.byUrl(opts.url)?.config; - if (!gitHubConfig) { - throw new Error( - `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, - ); - } - - const githubAppCredentialsMux = new GithubAppCredentialsMux(gitHubConfig); - const defaultToken = gitHubConfig.token; const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; let type: GithubCredentialType = 'app'; - let token = await githubAppCredentialsMux.getAppToken(owner, repo); + let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); if (!token) { type = 'token'; - token = defaultToken; + token = this.token; } return { diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 429a2b1c76..51d7bc9dd2 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -20,10 +20,10 @@ export { } from './config'; export type { GithubAppConfig, GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; -export { - GithubAppCredentialsMux, - SingleInstanceGithubCredentialsProvider, -} from './SingleInstanceGithubCredentialsProvider'; +export { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; +export { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; +export { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; + export type { GithubCredentials, GithubCredentialsProvider, diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 06886ecb4f..d6746f13da 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -21,7 +21,7 @@ import { getOrganizationRepositories } from './github'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; jest.mock('./github'); @@ -78,7 +78,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, @@ -103,7 +103,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, @@ -128,7 +128,7 @@ describe('GithubDiscoveryProcessor', () => { }); const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const processor = GithubDiscoveryProcessor.fromConfig(config, { logger: getVoidLogger(), githubCredentialsProvider, diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index 6fda605ae3..cd1966441b 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -26,7 +26,7 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import lodash from 'lodash'; import { EntitiesCatalog } from '../../catalog'; @@ -295,7 +295,7 @@ export class CatalogBuilder { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); this.checkDeprecatedReaderProcessors(); diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index 24aefb2d96..5b76866bd2 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -28,7 +28,7 @@ import { import { GithubCredentialsProvider, ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; @@ -294,7 +294,7 @@ export class NextCatalogBuilder { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider: GithubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); return [ new FileReaderProcessor(), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 6975f1aa16..ce1e2a7f49 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -19,7 +19,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { GithubCredentialsProvider, ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { Config } from '@backstage/config'; import { @@ -57,7 +57,7 @@ export const createBuiltinActions = (options: { const { reader, integrations, containerRunner, catalogClient, config } = options; const githubCredentialsProvider: GithubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const actions = [ createFetchPlainAction({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts index 39a1ff6ad8..0d4353c857 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts @@ -17,7 +17,7 @@ import { OctokitProvider } from './OctokitProvider'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; @@ -33,7 +33,7 @@ describe('getOctokit', () => { const integrations = ScmIntegrations.fromConfig(config); const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const octokitProvider = new OctokitProvider( integrations, githubCredentialsProvider, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts index 6720de4e95..b725f825fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts @@ -15,11 +15,13 @@ */ jest.mock('@octokit/rest'); - +import { TemplateAction } from '../../types'; import { createGithubActionsDispatchAction } from './githubActionsDispatch'; + import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; @@ -36,12 +38,8 @@ describe('github:actions:dispatch', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); - const action = createGithubActionsDispatchAction({ - integrations, - githubCredentialsProvider, - }); + let githubCredentialsProvider: GithubCredentialsProvider; + let action: TemplateAction; const mockContext = { input: { @@ -60,6 +58,12 @@ describe('github:actions:dispatch', () => { beforeEach(() => { jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createGithubActionsDispatchAction({ + integrations, + githubCredentialsProvider, + }); }); it('should call the githubApis for creating WorkflowDispatch', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts index 0c2daeb42f..fe2535ca42 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts @@ -19,11 +19,13 @@ jest.mock('@octokit/rest'); import { createGithubWebhookAction } from './githubWebhook'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; +import { TemplateAction } from '../..'; describe('github:repository:webhook:create', () => { const config = new ConfigReader({ @@ -36,13 +38,19 @@ describe('github:repository:webhook:create', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); + let githubCredentialsProvider: GithubCredentialsProvider; const defaultWebhookSecret = 'aafdfdivierernfdk23f'; - const action = createGithubWebhookAction({ - integrations, - defaultWebhookSecret, - githubCredentialsProvider, + let action: TemplateAction; + + beforeEach(() => { + jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createGithubWebhookAction({ + integrations, + defaultWebhookSecret, + githubCredentialsProvider, + }); }); const mockContext = { @@ -59,10 +67,6 @@ describe('github:repository:webhook:create', () => { const { mockGithubClient } = require('@octokit/rest'); - beforeEach(() => { - jest.resetAllMocks(); - }); - it('should call the githubApi for creating repository Webhook', async () => { const repoUrl = 'github.com?repo=repo&owner=owner'; const webhookUrl = 'https://example.com/payload'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 74d31ecc7a..7724dec85e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -14,13 +14,16 @@ * limitations under the License. */ +import { TemplateAction } from '../../types'; + jest.mock('../helpers'); jest.mock('@octokit/rest'); import { createPublishGithubAction } from './github'; import { ScmIntegrations, - SingleInstanceGithubCredentialsProvider, + DefaultGithubCredentialsProvider, + GithubCredentialsProvider, } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; @@ -42,13 +45,9 @@ describe('publish:github', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrations); - const action = createPublishGithubAction({ - integrations, - config, - githubCredentialsProvider, - }); + let githubCredentialsProvider: GithubCredentialsProvider; + let action: TemplateAction; + const mockContext = { input: { repoUrl: 'github.com?repo=repo&owner=owner', @@ -67,6 +66,13 @@ describe('publish:github', () => { beforeEach(() => { jest.resetAllMocks(); + githubCredentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + action = createPublishGithubAction({ + integrations, + config, + githubCredentialsProvider, + }); }); it('should call the githubApis with the correct values for createInOrg', async () => { From 41ced545f38eaa9bca1745bc4ea8c1924798eb1b Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 5 Jan 2022 08:30:11 +0000 Subject: [PATCH 05/14] api-reports Signed-off-by: Brian Fletcher --- packages/integration/api-report.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index efcaa07c88..ae1b49b8d4 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -92,6 +92,17 @@ export type BitbucketIntegrationConfig = { appPassword?: string; }; +// @public +export class DefaultGithubCredentialsProvider + implements GithubCredentialsProvider +{ + // (undocumented) + static fromIntegrations( + integrations: ScmIntegrations, + ): DefaultGithubCredentialsProvider; + getCredentials(opts: { url: string }): Promise; +} + // @public export function defaultScmResolveUrl(options: { url: string; @@ -427,9 +438,7 @@ export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { // (undocumented) - static create( - integrations: ScmIntegrations, - ): SingleInstanceGithubCredentialsProvider; + static create: (config: GitHubIntegrationConfig) => GithubCredentialsProvider; getCredentials(opts: { url: string }): Promise; } From 2f1d27133b9fe11d1f4851c323ae9c4c2c0ffb14 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 5 Jan 2022 13:52:01 +0000 Subject: [PATCH 06/14] use single instance provider in default one Also changes how the GitHub url is parsed. Signed-off-by: Brian Fletcher --- .../DefaultGithubCredentialsProvider.ts | 56 +++++-------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts index 37aa23698e..1d1b6db593 100644 --- a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -14,20 +14,12 @@ * limitations under the License. */ -import parseGitUrl from 'git-url-parse'; -import { - GithubCredentials, - GithubCredentialsProvider, - GithubCredentialType, -} from './types'; +import { GithubCredentials, GithubCredentialsProvider } from './types'; import { ScmIntegrations } from '../ScmIntegrations'; -import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; +import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; -type MuxCollection = { - [url: string]: { - githubAppCredentialsMux: GithubAppCredentialsMux; - token?: string; - }; +type SingleInstanceGithubCredentialsProviderCollection = { + [url: string]: GithubCredentialsProvider; }; /** @@ -42,21 +34,19 @@ export class DefaultGithubCredentialsProvider implements GithubCredentialsProvider { static fromIntegrations(integrations: ScmIntegrations) { - const muxen: MuxCollection = {}; + const credentialsProviders: SingleInstanceGithubCredentialsProviderCollection = + {}; integrations.github.list().forEach(integration => { - muxen[integration.config.host] = { - githubAppCredentialsMux: new GithubAppCredentialsMux( - integration.config, - ), - token: integration.config.token, - }; + const credentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integration.config); + credentialsProviders[integration.config.host] = credentialsProvider; }); - return new DefaultGithubCredentialsProvider(muxen); + return new DefaultGithubCredentialsProvider(credentialsProviders); } private constructor( - private readonly githubAppCredentialsMuxen: MuxCollection, + private readonly providers: SingleInstanceGithubCredentialsProviderCollection, ) {} /** @@ -80,32 +70,14 @@ export class DefaultGithubCredentialsProvider * @returns A promise of {@link GithubCredentials}. */ async getCredentials(opts: { url: string }): Promise { - const parsed = parseGitUrl(opts.url); + const parsed = new URL(opts.url); - if (!this.githubAppCredentialsMuxen[parsed.resource]) { + if (!this.providers[parsed.host]) { throw new Error( `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, ); } - const githubAppCredentialsMux = - this.githubAppCredentialsMuxen[parsed.resource].githubAppCredentialsMux; - const defaultToken = this.githubAppCredentialsMuxen[parsed.resource].token; - - const owner = parsed.owner || parsed.name; - const repo = parsed.owner ? parsed.name : undefined; - - let type: GithubCredentialType = 'app'; - let token = await githubAppCredentialsMux.getAppToken(owner, repo); - if (!token) { - type = 'token'; - token = defaultToken; - } - - return { - headers: token ? { Authorization: `Bearer ${token}` } : undefined, - token, - type, - }; + return this.providers[parsed.host].getCredentials(opts); } } From fb8ca7c1bfc89952e8af4e8e211d43ff02fc42ae Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 5 Jan 2022 16:53:30 +0000 Subject: [PATCH 07/14] remove code accidently added in Signed-off-by: Brian Fletcher --- .../src/github/GithubAppCredentialsMux.ts | 214 ------------------ .../processors/UrlReaderProcessor.ts | 6 - 2 files changed, 220 deletions(-) delete mode 100644 packages/integration/src/github/GithubAppCredentialsMux.ts diff --git a/packages/integration/src/github/GithubAppCredentialsMux.ts b/packages/integration/src/github/GithubAppCredentialsMux.ts deleted file mode 100644 index c6d055a213..0000000000 --- a/packages/integration/src/github/GithubAppCredentialsMux.ts +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { GithubAppConfig, GitHubIntegrationConfig } from './config'; -import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import { createAppAuth } from '@octokit/auth-app'; -import { DateTime } from 'luxon'; - -/** - * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entirely - * once GitHub Apps is out of preview. - */ -const HEADERS = { - Accept: 'application/vnd.github.machine-man-preview+json', -}; - -type InstallationData = { - installationId: number; - suspended: boolean; -}; - -class Cache { - private readonly tokenCache = new Map< - string, - { token: string; expiresAt: DateTime } - >(); - - async getOrCreateToken( - key: string, - supplier: () => Promise<{ token: string; expiresAt: DateTime }>, - ): Promise<{ accessToken: string }> { - const item = this.tokenCache.get(key); - if (item && this.isNotExpired(item.expiresAt)) { - return { accessToken: item.token }; - } - - const result = await supplier(); - this.tokenCache.set(key, result); - return { accessToken: result.token }; - } - - // consider timestamps older than 50 minutes to be expired. - private isNotExpired = (date: DateTime) => - date.diff(DateTime.local(), 'minutes').minutes > 50; -} - -/** - * GithubAppManager issues and caches tokens for a specific GitHub App. - */ -class GithubAppManager { - private readonly appClient: Octokit; - private readonly baseUrl?: string; - private readonly baseAuthConfig: { appId: number; privateKey: string }; - private readonly cache = new Cache(); - private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations - - constructor(config: GithubAppConfig, baseUrl?: string) { - this.allowedInstallationOwners = config.allowedInstallationOwners; - this.baseUrl = baseUrl; - this.baseAuthConfig = { - appId: config.appId, - privateKey: config.privateKey.replace(/\\n/gm, '\n'), - }; - this.appClient = new Octokit({ - baseUrl, - headers: HEADERS, - authStrategy: createAppAuth, - auth: this.baseAuthConfig, - }); - } - - async getInstallationCredentials( - owner: string, - repo?: string, - ): Promise<{ accessToken: string }> { - const { installationId, suspended } = await this.getInstallationData(owner); - if (this.allowedInstallationOwners) { - if (!this.allowedInstallationOwners?.includes(owner)) { - throw new Error( - `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, - ); - } - } - if (suspended) { - throw new Error(`The GitHub application for ${owner} is suspended`); - } - - const cacheKey = repo ? `${owner}/${repo}` : owner; - - // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. - return this.cache.getOrCreateToken(cacheKey, async () => { - const result = await this.appClient.apps.createInstallationAccessToken({ - installation_id: installationId, - headers: HEADERS, - }); - if (repo && result.data.repository_selection === 'selected') { - const installationClient = new Octokit({ - baseUrl: this.baseUrl, - auth: result.data.token, - }); - const repos = await installationClient.paginate( - installationClient.apps.listReposAccessibleToInstallation, - ); - const hasRepo = repos.some(repository => { - return repository.name === repo; - }); - if (!hasRepo) { - throw new Error( - `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, - ); - } - } - return { - token: result.data.token, - expiresAt: DateTime.fromISO(result.data.expires_at), - }; - }); - } - - getInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - return this.appClient.paginate(this.appClient.apps.listInstallations); - } - - private async getInstallationData(owner: string): Promise { - const allInstallations = await this.getInstallations(); - const installation = allInstallations.find( - inst => - inst.account?.login?.toLocaleLowerCase('en-US') === - owner.toLocaleLowerCase('en-US'), - ); - if (installation) { - return { - installationId: installation.id, - suspended: Boolean(installation.suspended_by), - }; - } - const notFoundError = new Error( - `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, - ); - notFoundError.name = 'NotFoundError'; - throw notFoundError; - } -} - -/** - * Corresponds to a Github installation which internally could hold several GitHub Apps. - * - * @public - */ -export class GithubAppCredentialsMux { - private readonly apps: GithubAppManager[]; - - constructor(config: GitHubIntegrationConfig) { - this.apps = - config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; - } - - async getAllInstallations(): Promise< - RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] - > { - if (!this.apps.length) { - return []; - } - - const installs = await Promise.all( - this.apps.map(app => app.getInstallations()), - ); - - return installs.flat(); - } - - async getAppToken(owner: string, repo?: string): Promise { - if (this.apps.length === 0) { - return undefined; - } - - const results = await Promise.all( - this.apps.map(app => - app.getInstallationCredentials(owner, repo).then( - credentials => ({ credentials, error: undefined }), - error => ({ credentials: undefined, error }), - ), - ), - ); - - const result = results.find(resultItem => resultItem.credentials); - if (result) { - return result.credentials!.accessToken; - } - - const errors = results.map(r => r.error); - const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); - if (notNotFoundError) { - throw notNotFoundError; - } - - return undefined; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 83f22249d7..6a85cc07c7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -140,9 +140,3 @@ export class UrlReaderProcessor implements CatalogProcessor { return { response: [{ url: location, data }] }; } } - -export class RoadieDemoDataReaderProcessor extends UrlReaderProcessor { - getProcessorName() { - return 'roadie-demo-data-reader'; - } -} From 0a21e4bcc4aa6e801c83df1dfe9e42e55e3dd533 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 11:46:39 +0000 Subject: [PATCH 08/14] Make the credentials provider optional.. ..and other review comments improvements. Signed-off-by: Brian Fletcher --- .../DefaultGithubCredentialsProvider.test.ts | 358 ++++-------------- .../DefaultGithubCredentialsProvider.ts | 17 +- ...SingleInstanceGithubCredentialsProvider.ts | 202 +++++++++- packages/integration/src/github/index.ts | 6 +- plugins/catalog-backend/api-report.md | 14 +- .../processors/GithubDiscoveryProcessor.ts | 9 +- .../GithubMultiOrgReaderProcessor.ts | 9 +- .../processors/GithubOrgReaderProcessor.ts | 9 +- .../providers/GitHubOrgEntityProvider.ts | 7 +- plugins/scaffolder-backend/api-report.md | 12 +- .../builtin/github/githubActionsDispatch.ts | 10 +- .../actions/builtin/github/githubWebhook.ts | 10 +- .../actions/builtin/publish/github.ts | 10 +- 13 files changed, 332 insertions(+), 341 deletions(-) diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts index f537fee783..7c2b53e822 100644 --- a/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.test.ts @@ -15,28 +15,24 @@ */ import { ScmIntegrations } from '../ScmIntegrations'; - -const octokit = { - paginate: async (fn: any) => (await fn()).data, - apps: { - listInstallations: jest.fn(), - createInstallationAccessToken: jest.fn(), - }, -}; - -jest.doMock('@octokit/rest', () => { - class Octokit { - constructor() { - return octokit; - } - } - return { Octokit }; -}); +import { GitHubIntegrationConfig } from './config'; +import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; import { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; -import { RestEndpointMethodTypes } from '@octokit/rest'; -import { DateTime } from 'luxon'; import { ConfigReader } from '@backstage/config'; +import { GithubCredentials } from './types'; + +const resultBuilder = (host: string): GithubCredentials => { + return { + type: 'token', + token: `${host}-token`, + headers: { + token: `${host}-token`, + }, + }; +}; + +jest.mock('./SingleInstanceGithubCredentialsProvider'); let integrations: ScmIntegrations; @@ -59,289 +55,69 @@ describe('DefaultGithubCredentialsProvider tests', () => { ], token: 'hardcoded_token', }, - ], - }, - }), - ); - jest.resetAllMocks(); - }); - - it('create repository specific tokens', async () => { - octokit.apps.listInstallations.mockResolvedValue({ - headers: { - etag: '123', - }, - data: [ - { - id: 1, - repository_selection: 'selected', - account: null, - }, - { - id: 2, - repository_selection: 'selected', - account: { - login: 'backstage', - }, - }, - ], - } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ - data: { - expires_at: DateTime.local().plus({ hours: 1 }).toString(), - token: 'secret_token', - }, - } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - const { token, headers, type } = await github.getCredentials({ - url: 'https://github.com/backstage/foobar', - }); - expect(type).toEqual('app'); - expect(token).toEqual('secret_token'); - expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); - - // fallback to the configured token if no application is matching - await expect( - github.getCredentials({ - url: 'https://github.com/404/foobar', - }), - ).resolves.toEqual({ - headers: { - Authorization: 'Bearer hardcoded_token', - }, - token: 'hardcoded_token', - type: 'token', - }); - }); - - it('creates tokens for an organization', async () => { - octokit.apps.listInstallations.mockResolvedValue({ - headers: { - etag: '123', - }, - data: [ - { - id: 1, - repository_selection: 'all', - account: { - login: 'backstage', - }, - }, - ], - } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ - data: { - expires_at: DateTime.local().plus({ hours: 1 }).toString(), - token: 'secret_token', - }, - } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - const { token, headers } = await github.getCredentials({ - url: 'https://github.com/backstage', - }); - - expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); - expect(token).toEqual('secret_token'); - }); - - it('should not fail to issue tokens for an organization when the app is installed for a single repo', async () => { - octokit.apps.listInstallations.mockResolvedValue({ - headers: { - etag: '123', - }, - data: [ - { - id: 1, - repository_selection: 'selected', - account: { - login: 'backstage', - }, - }, - ], - } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ - data: { - expires_at: DateTime.local().plus({ hours: 1 }).toString(), - token: 'secret_token', - }, - } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - const { token, headers } = await github.getCredentials({ - url: 'https://github.com/backstage', - }); - const expectedToken = 'secret_token'; - expect(headers).toEqual({ Authorization: `Bearer ${expectedToken}` }); - expect(token).toEqual('secret_token'); - }); - - it('should throw if the app is suspended', async () => { - octokit.apps.listInstallations.mockResolvedValue({ - headers: { - etag: '123', - }, - data: [ - { - id: 1, - suspended_by: { - login: 'admin', - }, - repository_selection: 'all', - account: { - login: 'backstage', - }, - }, - ], - } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - await expect( - github.getCredentials({ - url: 'https://github.com/backstage', - }), - ).rejects.toThrow('The GitHub application for backstage is suspended'); - }); - - it('should return the default token when the call to github return a status that is not recognized', async () => { - octokit.apps.listInstallations.mockRejectedValue({ - status: 404, - message: 'NotFound', - }); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - await expect( - github.getCredentials({ - url: 'https://github.com/backstage', - }), - ).rejects.toEqual({ status: 404, message: 'NotFound' }); - }); - - it('should return the default token if no app is configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ { - host: 'github.com', - apps: [], - token: 'fallback_token', - }, - ], - }, - }), - ); - - const githubProvider = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - await expect( - githubProvider.getCredentials({ - url: 'https://github.com/404/foobar', - }), - ).resolves.toEqual(expect.objectContaining({ token: 'fallback_token' })); - }); - - it('should return the configured token if there are no installations', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', - }, - ], + host: 'grithub.com', token: 'hardcoded_token', }, ], }, }), ); - - octokit.apps.listInstallations.mockResolvedValue({ - data: [], - } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - const githubProvider = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - await expect( - githubProvider.getCredentials({ - url: 'https://github.com/backstage', - }), - ).resolves.toEqual(expect.objectContaining({ token: 'hardcoded_token' })); + jest.resetAllMocks(); + SingleInstanceGithubCredentialsProvider.create = ( + config: GitHubIntegrationConfig, + ) => { + return { + getCredentials: (_opts: { url: string }) => { + return Promise.resolve(resultBuilder(config.host)); + }, + }; + }; + jest.spyOn(SingleInstanceGithubCredentialsProvider, 'create'); }); - it('should return undefined if no token or apps are configured', async () => { - integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { - host: 'github.com', - }, - ], - }, - }), - ); - const githubProvider = + describe('.create', () => { + it('passes the config through to the single provider', () => { DefaultGithubCredentialsProvider.fromIntegrations(integrations); - - await expect( - githubProvider.getCredentials({ - url: 'https://github.com/backstage', - }), - ).resolves.toEqual({ headers: undefined, token: undefined, type: 'token' }); - }); - - it('should to create a token for the organization ignoring case sensitive', async () => { - octokit.apps.listInstallations.mockResolvedValue({ - headers: { - etag: '123', - }, - data: [ - { - id: 1, - repository_selection: 'all', - account: { - login: 'BACKSTAGE', - }, - }, - ], - } as RestEndpointMethodTypes['apps']['listInstallations']['response']); - - octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ - data: { - expires_at: DateTime.local().plus({ hours: 1 }).toString(), - token: 'secret_token', - }, - } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); - - const github = - DefaultGithubCredentialsProvider.fromIntegrations(integrations); - const { token, headers } = await github.getCredentials({ - url: 'https://github.com/backstage', + const githubIntegration = + integrations.github.byHost('github.com')?.config; + const grithubIntegration = + integrations.github.byHost('grithub.com')?.config; + expect( + SingleInstanceGithubCredentialsProvider.create, + ).toHaveBeenCalledWith(githubIntegration); + expect( + SingleInstanceGithubCredentialsProvider.create, + ).toHaveBeenCalledWith(grithubIntegration); }); + }); - expect(headers).toEqual({ Authorization: 'Bearer secret_token' }); - expect(token).toEqual('secret_token'); + describe('#getCredentials', () => { + it('returns the data verbatim from the creds provider', async () => { + const provider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const gitHubCredentials = await provider.getCredentials({ + url: 'https://github.com/blah', + }); + const gritHubCredentials = await provider.getCredentials({ + url: 'https://grithub.com/blah', + }); + + expect(gitHubCredentials).toEqual({ + type: 'token', + token: 'github.com-token', + headers: { + token: 'github.com-token', + }, + }); + + expect(gritHubCredentials).toEqual({ + type: 'token', + token: 'grithub.com-token', + headers: { + token: 'grithub.com-token', + }, + }); + }); }); }); diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts index 1d1b6db593..aabcac53c7 100644 --- a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -18,10 +18,6 @@ import { GithubCredentials, GithubCredentialsProvider } from './types'; import { ScmIntegrations } from '../ScmIntegrations'; import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; -type SingleInstanceGithubCredentialsProviderCollection = { - [url: string]: GithubCredentialsProvider; -}; - /** * Handles the creation and caching of credentials for GitHub integrations. * @@ -34,19 +30,19 @@ export class DefaultGithubCredentialsProvider implements GithubCredentialsProvider { static fromIntegrations(integrations: ScmIntegrations) { - const credentialsProviders: SingleInstanceGithubCredentialsProviderCollection = - {}; + const credentialsProviders: Map = + new Map(); integrations.github.list().forEach(integration => { const credentialsProvider = SingleInstanceGithubCredentialsProvider.create(integration.config); - credentialsProviders[integration.config.host] = credentialsProvider; + credentialsProviders.set(integration.config.host, credentialsProvider); }); return new DefaultGithubCredentialsProvider(credentialsProviders); } private constructor( - private readonly providers: SingleInstanceGithubCredentialsProviderCollection, + private readonly providers: Map, ) {} /** @@ -71,13 +67,14 @@ export class DefaultGithubCredentialsProvider */ async getCredentials(opts: { url: string }): Promise { const parsed = new URL(opts.url); + const provider = this.providers.get(parsed.host); - if (!this.providers[parsed.host]) { + if (!provider) { throw new Error( `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, ); } - return this.providers[parsed.host].getCredentials(opts); + return provider.getCredentials(opts); } } diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 5f04a3ed6e..6b5e3318db 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -21,8 +21,206 @@ import { GithubCredentialsProvider, GithubCredentialType, } from './types'; -import { GitHubIntegrationConfig } from './config'; -import { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; + +import { GithubAppConfig, GitHubIntegrationConfig } from './config'; +import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; +import { createAppAuth } from '@octokit/auth-app'; +import { DateTime } from 'luxon'; + +/** + * This accept header is required when calling App APIs in GitHub Enterprise. + * It has no effect on calls to github.com and can probably be removed entirely + * once GitHub Apps is out of preview. + */ +const HEADERS = { + Accept: 'application/vnd.github.machine-man-preview+json', +}; + +type InstallationData = { + installationId: number; + suspended: boolean; +}; + +class Cache { + private readonly tokenCache = new Map< + string, + { token: string; expiresAt: DateTime } + >(); + + async getOrCreateToken( + key: string, + supplier: () => Promise<{ token: string; expiresAt: DateTime }>, + ): Promise<{ accessToken: string }> { + const item = this.tokenCache.get(key); + if (item && this.isNotExpired(item.expiresAt)) { + return { accessToken: item.token }; + } + + const result = await supplier(); + this.tokenCache.set(key, result); + return { accessToken: result.token }; + } + + // consider timestamps older than 50 minutes to be expired. + private isNotExpired = (date: DateTime) => + date.diff(DateTime.local(), 'minutes').minutes > 50; +} + +/** + * GithubAppManager issues and caches tokens for a specific GitHub App. + */ +class GithubAppManager { + private readonly appClient: Octokit; + private readonly baseUrl?: string; + private readonly baseAuthConfig: { appId: number; privateKey: string }; + private readonly cache = new Cache(); + private readonly allowedInstallationOwners: string[] | undefined; // undefined allows all installations + + constructor(config: GithubAppConfig, baseUrl?: string) { + this.allowedInstallationOwners = config.allowedInstallationOwners; + this.baseUrl = baseUrl; + this.baseAuthConfig = { + appId: config.appId, + privateKey: config.privateKey.replace(/\\n/gm, '\n'), + }; + this.appClient = new Octokit({ + baseUrl, + headers: HEADERS, + authStrategy: createAppAuth, + auth: this.baseAuthConfig, + }); + } + + async getInstallationCredentials( + owner: string, + repo?: string, + ): Promise<{ accessToken: string }> { + const { installationId, suspended } = await this.getInstallationData(owner); + if (this.allowedInstallationOwners) { + if (!this.allowedInstallationOwners?.includes(owner)) { + throw new Error( + `The GitHub application for ${owner} is not included in the allowed installation list (${installationId}).`, + ); + } + } + if (suspended) { + throw new Error(`The GitHub application for ${owner} is suspended`); + } + + const cacheKey = repo ? `${owner}/${repo}` : owner; + + // Go and grab an access token for the app scoped to a repository if provided, if not use the organisation installation. + return this.cache.getOrCreateToken(cacheKey, async () => { + const result = await this.appClient.apps.createInstallationAccessToken({ + installation_id: installationId, + headers: HEADERS, + }); + if (repo && result.data.repository_selection === 'selected') { + const installationClient = new Octokit({ + baseUrl: this.baseUrl, + auth: result.data.token, + }); + const repos = await installationClient.paginate( + installationClient.apps.listReposAccessibleToInstallation, + ); + const hasRepo = repos.some(repository => { + return repository.name === repo; + }); + if (!hasRepo) { + throw new Error( + `The Backstage GitHub application used in the ${owner} organization does not have access to a repository with the name ${repo}`, + ); + } + } + return { + token: result.data.token, + expiresAt: DateTime.fromISO(result.data.expires_at), + }; + }); + } + + getInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + return this.appClient.paginate(this.appClient.apps.listInstallations); + } + + private async getInstallationData(owner: string): Promise { + const allInstallations = await this.getInstallations(); + const installation = allInstallations.find( + inst => + inst.account?.login?.toLocaleLowerCase('en-US') === + owner.toLocaleLowerCase('en-US'), + ); + if (installation) { + return { + installationId: installation.id, + suspended: Boolean(installation.suspended_by), + }; + } + const notFoundError = new Error( + `No app installation found for ${owner} in ${this.baseAuthConfig.appId}`, + ); + notFoundError.name = 'NotFoundError'; + throw notFoundError; + } +} + +/** + * Corresponds to a Github installation which internally could hold several GitHub Apps. + * + * @public + */ +export class GithubAppCredentialsMux { + private readonly apps: GithubAppManager[]; + + constructor(config: GitHubIntegrationConfig) { + this.apps = + config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; + } + + async getAllInstallations(): Promise< + RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] + > { + if (!this.apps.length) { + return []; + } + + const installs = await Promise.all( + this.apps.map(app => app.getInstallations()), + ); + + return installs.flat(); + } + + async getAppToken(owner: string, repo?: string): Promise { + if (this.apps.length === 0) { + return undefined; + } + + const results = await Promise.all( + this.apps.map(app => + app.getInstallationCredentials(owner, repo).then( + credentials => ({ credentials, error: undefined }), + error => ({ credentials: undefined, error }), + ), + ), + ); + + const result = results.find(resultItem => resultItem.credentials); + if (result) { + return result.credentials!.accessToken; + } + + const errors = results.map(r => r.error); + const notNotFoundError = errors.find(err => err.name !== 'NotFoundError'); + if (notNotFoundError) { + throw notNotFoundError; + } + + return undefined; + } +} /** * Handles the creation and caching of credentials for GitHub integrations. diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 51d7bc9dd2..b9fbb845f1 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -20,9 +20,11 @@ export { } from './config'; export type { GithubAppConfig, GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; -export { GithubAppCredentialsMux } from './GithubAppCredentialsMux'; export { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; -export { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; +export { + SingleInstanceGithubCredentialsProvider, + GithubAppCredentialsMux, +} from './SingleInstanceGithubCredentialsProvider'; export type { GithubCredentials, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 1fe38051c9..2d5e29fea0 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -1012,14 +1012,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ): GithubDiscoveryProcessor; // (undocumented) @@ -1036,14 +1036,14 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { integrations: ScmIntegrations; logger: Logger_2; orgs: GithubMultiOrgConfig; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ): GithubMultiOrgReaderProcessor; // (undocumented) @@ -1074,7 +1074,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { id: string; orgUrl: string; logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ): GitHubOrgEntityProvider; // (undocumented) @@ -1090,14 +1090,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }); // (undocumented) static fromConfig( config: Config, options: { logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ): GithubOrgReaderProcessor; // (undocumented) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 01ba40b1fd..3196c4e161 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -17,6 +17,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, ScmIntegrations, } from '@backstage/integration'; @@ -49,7 +50,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { config: Config, options: { logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ) { const integrations = ScmIntegrations.fromConfig(config); @@ -63,11 +64,13 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; - this.githubCredentialsProvider = options.githubCredentialsProvider; + this.githubCredentialsProvider = + options.githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(this.integrations); } async readLocation( diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts index 7a25b35078..e07de8ac0c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts @@ -17,6 +17,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { + DefaultGithubCredentialsProvider, GithubAppCredentialsMux, GithubCredentialsProvider, GitHubIntegrationConfig, @@ -50,7 +51,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { config: Config, options: { logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ) { const c = config.getOptionalConfig('catalog.processors.githubMultiOrg'); @@ -67,12 +68,14 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { integrations: ScmIntegrations; logger: Logger; orgs: GithubMultiOrgConfig; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; this.orgs = options.orgs; - this.githubCredentialsProvider = options.githubCredentialsProvider; + this.githubCredentialsProvider = + options.githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(this.integrations); } async readLocation( diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 4c56b30c90..474d08feee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -20,6 +20,7 @@ import { GithubCredentialType, ScmIntegrations, GithubCredentialsProvider, + DefaultGithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -46,7 +47,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { config: Config, options: { logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ) { const integrations = ScmIntegrations.fromConfig(config); @@ -60,10 +61,12 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrations; logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }) { this.integrations = options.integrations; - this.githubCredentialsProvider = options.githubCredentialsProvider; + this.githubCredentialsProvider = + options.githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(this.integrations); this.logger = options.logger; } diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index bc40ca1da5..c161074de4 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -20,6 +20,7 @@ import { } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, @@ -47,7 +48,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { id: string; orgUrl: string; logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ) { const integrations = ScmIntegrations.fromConfig(config); @@ -68,7 +69,9 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: options.orgUrl, logger, gitHubConfig, - githubCredentialsProvider: options.githubCredentialsProvider, + githubCredentialsProvider: + options.githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(integrations), }); } diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index a292388909..e0710647de 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -123,17 +123,17 @@ export const createFilesystemRenameAction: () => TemplateAction; // // @public (undocumented) export function createGithubActionsDispatchAction(options: { - integrations: ScmIntegrationRegistry; - githubCredentialsProvider: GithubCredentialsProvider; + integrations: ScmIntegrations; + githubCredentialsProvider?: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-missing-release-tag) "createGithubWebhookAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export function createGithubWebhookAction(options: { - integrations: ScmIntegrationRegistry; + integrations: ScmIntegrations; defaultWebhookSecret?: string; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-missing-release-tag) "createPublishAzureAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -161,9 +161,9 @@ export function createPublishFileAction(): TemplateAction; // // @public (undocumented) export function createPublishGithubAction(options: { - integrations: ScmIntegrationRegistry; + integrations: ScmIntegrations; config: Config; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }): TemplateAction; // Warning: (ae-forgotten-export) The symbol "CreateGithubPullRequestActionOptions" needs to be exported by the entry point index.d.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts index a88698ed07..eed74f2d04 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts @@ -14,20 +14,22 @@ * limitations under the License. */ import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, - ScmIntegrationRegistry, + ScmIntegrations, } from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; export function createGithubActionsDispatchAction(options: { - integrations: ScmIntegrationRegistry; - githubCredentialsProvider: GithubCredentialsProvider; + integrations: ScmIntegrations; + githubCredentialsProvider?: GithubCredentialsProvider; }) { const { integrations, githubCredentialsProvider } = options; const octokitProvider = new OctokitProvider( integrations, - githubCredentialsProvider, + githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(integrations), ); return createTemplateAction<{ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts index d341b59011..95950627f9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts @@ -14,8 +14,9 @@ * limitations under the License. */ import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, - ScmIntegrationRegistry, + ScmIntegrations, } from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; @@ -25,15 +26,16 @@ import { assertError } from '@backstage/errors'; type ContentType = 'form' | 'json'; export function createGithubWebhookAction(options: { - integrations: ScmIntegrationRegistry; + integrations: ScmIntegrations; defaultWebhookSecret?: string; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }) { const { integrations, defaultWebhookSecret, githubCredentialsProvider } = options; const octokitProvider = new OctokitProvider( integrations, - githubCredentialsProvider, + githubCredentialsProvider ?? + DefaultGithubCredentialsProvider.fromIntegrations(integrations), ); const eventNames = emitterEventNames.filter(event => !event.includes('.')); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 66a45eaba1..24f3c5c48b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -14,8 +14,9 @@ * limitations under the License. */ import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, - ScmIntegrationRegistry, + ScmIntegrations, } from '@backstage/integration'; import { enableBranchProtectionOnDefaultRepoBranch, @@ -31,14 +32,15 @@ type Permission = 'pull' | 'push' | 'admin' | 'maintain' | 'triage'; type Collaborator = { access: Permission; username: string }; export function createPublishGithubAction(options: { - integrations: ScmIntegrationRegistry; + integrations: ScmIntegrations; config: Config; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }) { const { integrations, config, githubCredentialsProvider } = options; const octokitProvider = new OctokitProvider( integrations, - githubCredentialsProvider, + githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(integrations), ); return createTemplateAction<{ From be15350198199e6b5f41f3bf4d491be7da607c95 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 11:55:15 +0000 Subject: [PATCH 09/14] change single instance provider back to the way it was there was no changes to it, so there was no need for it. Signed-off-by: Brian Fletcher --- ...SingleInstanceGithubCredentialsProvider.ts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 6b5e3318db..bc15007f9d 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -15,27 +15,16 @@ */ import parseGitUrl from 'git-url-parse'; - +import { GithubAppConfig, GitHubIntegrationConfig } from './config'; +import { createAppAuth } from '@octokit/auth-app'; +import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; +import { DateTime } from 'luxon'; import { GithubCredentials, GithubCredentialsProvider, GithubCredentialType, } from './types'; -import { GithubAppConfig, GitHubIntegrationConfig } from './config'; -import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import { createAppAuth } from '@octokit/auth-app'; -import { DateTime } from 'luxon'; - -/** - * This accept header is required when calling App APIs in GitHub Enterprise. - * It has no effect on calls to github.com and can probably be removed entirely - * once GitHub Apps is out of preview. - */ -const HEADERS = { - Accept: 'application/vnd.github.machine-man-preview+json', -}; - type InstallationData = { installationId: number; suspended: boolean; @@ -66,6 +55,15 @@ class Cache { date.diff(DateTime.local(), 'minutes').minutes > 50; } +/** + * This accept header is required when calling App APIs in GitHub Enterprise. + * It has no effect on calls to github.com and can probably be removed entirely + * once GitHub Apps is out of preview. + */ +const HEADERS = { + Accept: 'application/vnd.github.machine-man-preview+json', +}; + /** * GithubAppManager issues and caches tokens for a specific GitHub App. */ From 3b4d8caff6917f062b9ba63da14a14f0ea313588 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 12:03:52 +0000 Subject: [PATCH 10/14] adds changesets Signed-off-by: Brian Fletcher --- .changeset/cold-toes-sniff.md | 5 +++++ .changeset/late-plums-act.md | 5 +++++ .changeset/silent-nails-sleep.md | 5 +++++ .changeset/stale-gorillas-mix.md | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 .changeset/cold-toes-sniff.md create mode 100644 .changeset/late-plums-act.md create mode 100644 .changeset/silent-nails-sleep.md create mode 100644 .changeset/stale-gorillas-mix.md diff --git a/.changeset/cold-toes-sniff.md b/.changeset/cold-toes-sniff.md new file mode 100644 index 0000000000..0b79abd5aa --- /dev/null +++ b/.changeset/cold-toes-sniff.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Adds a new GitHub credentials provider. It handles multiple app configurations. It looks up the app configuration based on the url. diff --git a/.changeset/late-plums-act.md b/.changeset/late-plums-act.md new file mode 100644 index 0000000000..6822f40eeb --- /dev/null +++ b/.changeset/late-plums-act.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Allow a custom GithubCredentialsProvider to be passed to the GitHub processors. diff --git a/.changeset/silent-nails-sleep.md b/.changeset/silent-nails-sleep.md new file mode 100644 index 0000000000..a1587f8427 --- /dev/null +++ b/.changeset/silent-nails-sleep.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Allow a GitHubCredentialsProvider to be passed to the GitHub scaffolder tasks actions. diff --git a/.changeset/stale-gorillas-mix.md b/.changeset/stale-gorillas-mix.md new file mode 100644 index 0000000000..4ffa1fd553 --- /dev/null +++ b/.changeset/stale-gorillas-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +The GithubUrlReader is switched to use the DefaultGithubCredentialsProvider From ffda3d4490f81503af776c3ea190e3814bde6fb1 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 12:11:05 +0000 Subject: [PATCH 11/14] tweaks to remove uneeded differences Signed-off-by: Brian Fletcher --- .../src/github/SingleInstanceGithubCredentialsProvider.ts | 2 +- packages/integration/src/github/index.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index bc15007f9d..408c5b1348 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2022 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index b9fbb845f1..c6a7799c3d 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -22,10 +22,9 @@ export type { GithubAppConfig, GitHubIntegrationConfig } from './config'; export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; export { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider'; export { - SingleInstanceGithubCredentialsProvider, GithubAppCredentialsMux, + SingleInstanceGithubCredentialsProvider, } from './SingleInstanceGithubCredentialsProvider'; - export type { GithubCredentials, GithubCredentialsProvider, From 95b369d808a44425ce4fc071c24e0c2cfc26d40e Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 14:21:56 +0000 Subject: [PATCH 12/14] use scm integrations interface rather than implementation Signed-off-by: Brian Fletcher --- .../src/github/DefaultGithubCredentialsProvider.ts | 4 ++-- .../src/scaffolder/actions/builtin/github/githubWebhook.ts | 4 ++-- .../src/scaffolder/actions/builtin/publish/github.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts index aabcac53c7..f43eabe52f 100644 --- a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -15,7 +15,7 @@ */ import { GithubCredentials, GithubCredentialsProvider } from './types'; -import { ScmIntegrations } from '../ScmIntegrations'; +import { ScmIntegrationRegistry } from '../registry'; import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; /** @@ -29,7 +29,7 @@ import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubC export class DefaultGithubCredentialsProvider implements GithubCredentialsProvider { - static fromIntegrations(integrations: ScmIntegrations) { + static fromIntegrations(integrations: ScmIntegrationRegistry) { const credentialsProviders: Map = new Map(); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts index 95950627f9..95f1888882 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts @@ -16,7 +16,7 @@ import { DefaultGithubCredentialsProvider, GithubCredentialsProvider, - ScmIntegrations, + ScmIntegrationRegistry, } from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; @@ -26,7 +26,7 @@ import { assertError } from '@backstage/errors'; type ContentType = 'form' | 'json'; export function createGithubWebhookAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; defaultWebhookSecret?: string; githubCredentialsProvider?: GithubCredentialsProvider; }) { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 24f3c5c48b..3e529b1f17 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -16,7 +16,7 @@ import { DefaultGithubCredentialsProvider, GithubCredentialsProvider, - ScmIntegrations, + ScmIntegrationRegistry, } from '@backstage/integration'; import { enableBranchProtectionOnDefaultRepoBranch, @@ -32,7 +32,7 @@ type Permission = 'pull' | 'push' | 'admin' | 'maintain' | 'triage'; type Collaborator = { access: Permission; username: string }; export function createPublishGithubAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; config: Config; githubCredentialsProvider?: GithubCredentialsProvider; }) { From db49de335f4f13d3b2d720436f0fc8f2196ef748 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 6 Jan 2022 16:31:36 +0000 Subject: [PATCH 13/14] adds api-reports Signed-off-by: Brian Fletcher --- packages/integration/api-report.md | 2 +- plugins/scaffolder-backend/api-report.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index ae1b49b8d4..4d8a28ea8c 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -98,7 +98,7 @@ export class DefaultGithubCredentialsProvider { // (undocumented) static fromIntegrations( - integrations: ScmIntegrations, + integrations: ScmIntegrationRegistry, ): DefaultGithubCredentialsProvider; getCredentials(opts: { url: string }): Promise; } diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index e0710647de..f2f9830664 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -131,7 +131,7 @@ export function createGithubActionsDispatchAction(options: { // // @public (undocumented) export function createGithubWebhookAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; defaultWebhookSecret?: string; githubCredentialsProvider?: GithubCredentialsProvider; }): TemplateAction; @@ -161,7 +161,7 @@ export function createPublishFileAction(): TemplateAction; // // @public (undocumented) export function createPublishGithubAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; config: Config; githubCredentialsProvider?: GithubCredentialsProvider; }): TemplateAction; From ce82656901817c2072419b8913f03183d1043c08 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 7 Jan 2022 14:13:23 +0000 Subject: [PATCH 14/14] removes some breaking compatability changes Signed-off-by: Brian Fletcher --- .changeset/cold-toes-sniff.md | 2 +- .../src/github/DefaultGithubCredentialsProvider.ts | 6 +++++- plugins/catalog-backend/api-report.md | 2 +- .../ingestion/providers/GitHubOrgEntityProvider.ts | 7 +++++-- plugins/scaffolder-backend/api-report.md | 2 +- .../actions/builtin/github/OctokitProvider.ts | 7 +++++-- .../actions/builtin/publish/githubPullRequest.ts | 11 ++++++++--- 7 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.changeset/cold-toes-sniff.md b/.changeset/cold-toes-sniff.md index 0b79abd5aa..7fcfb54408 100644 --- a/.changeset/cold-toes-sniff.md +++ b/.changeset/cold-toes-sniff.md @@ -2,4 +2,4 @@ '@backstage/integration': patch --- -Adds a new GitHub credentials provider. It handles multiple app configurations. It looks up the app configuration based on the url. +Adds a new GitHub credentials provider (DefaultGithubCredentialsProvider). It handles multiple app configurations. It looks up the app configuration based on the url. diff --git a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts index f43eabe52f..112fd83db7 100644 --- a/packages/integration/src/github/DefaultGithubCredentialsProvider.ts +++ b/packages/integration/src/github/DefaultGithubCredentialsProvider.ts @@ -58,7 +58,11 @@ export class DefaultGithubCredentialsProvider * @example * ```ts * const { token, headers } = await getCredentials({ - * url: 'github.com/backstage/foobar' + * url: 'https://github.com/backstage/foobar' + * }) + * + * const { token, headers } = await getCredentials({ + * url: 'https://github.com/backstage' * }) * ``` * diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 2d5e29fea0..e9a985914f 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -1063,7 +1063,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: string; gitHubConfig: GitHubIntegrationConfig; logger: Logger_2; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index c161074de4..b1d44a55b5 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -24,6 +24,7 @@ import { GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, + SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; @@ -81,10 +82,12 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: string; gitHubConfig: GitHubIntegrationConfig; logger: Logger; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; }, ) { - this.githubCredentialsProvider = options.githubCredentialsProvider; + this.githubCredentialsProvider = + options.githubCredentialsProvider || + SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig); } getProviderName() { diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index f2f9830664..fde7d7d265 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -288,7 +288,7 @@ export function fetchContents({ export class OctokitProvider { constructor( integrations: ScmIntegrationRegistry, - githubCredentialsProvider: GithubCredentialsProvider, + githubCredentialsProvider?: GithubCredentialsProvider, ); // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-forgotten-export) The symbol "OctokitIntegration" needs to be exported by the entry point index.d.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts index 4a8bb8bbf6..40c723005e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts @@ -16,6 +16,7 @@ import { InputError } from '@backstage/errors'; import { + DefaultGithubCredentialsProvider, GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; @@ -38,10 +39,12 @@ export class OctokitProvider { constructor( integrations: ScmIntegrationRegistry, - githubCredentialsProvider: GithubCredentialsProvider, + githubCredentialsProvider?: GithubCredentialsProvider, ) { this.integrations = integrations; - this.githubCredentialsProvider = githubCredentialsProvider; + this.githubCredentialsProvider = + githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(this.integrations); } /** diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts index e657c36e50..7b3e89ac2a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts @@ -20,6 +20,7 @@ import { parseRepoUrl, isExecutable } from './util'; import { GithubCredentialsProvider, ScmIntegrationRegistry, + SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; import { zipObject } from 'lodash'; import { createTemplateAction } from '../../createTemplateAction'; @@ -58,7 +59,7 @@ export type GithubPullRequestActionInput = { export type ClientFactoryInput = { integrations: ScmIntegrationRegistry; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; host: string; owner: string; repo: string; @@ -77,7 +78,11 @@ export const defaultClientFactory = async ({ throw new InputError(`No integration for host ${host}`); } - const { token } = await githubCredentialsProvider.getCredentials({ + const credentialsProvider = + githubCredentialsProvider || + SingleInstanceGithubCredentialsProvider.create(integrationConfig); + + const { token } = await credentialsProvider.getCredentials({ url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent( repo, )}`, @@ -99,7 +104,7 @@ export const defaultClientFactory = async ({ interface CreateGithubPullRequestActionOptions { integrations: ScmIntegrationRegistry; - githubCredentialsProvider: GithubCredentialsProvider; + githubCredentialsProvider?: GithubCredentialsProvider; clientFactory?: (input: ClientFactoryInput) => Promise; }