diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 6b18cfbe1a..160a1a533b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -58,9 +58,9 @@ export type GhBlobResponse = export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); + const credentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); return integrations.github.list().map(integration => { - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integration.config); const reader = new GithubUrlReader(integration, { treeResponseFactory, credentialsProvider, diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts index 6bea9647a1..4278218a2c 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.test.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { ScmIntegrations } from '../ScmIntegrations'; + const octokit = { paginate: async (fn: any) => (await fn()).data, apps: { @@ -34,22 +36,33 @@ jest.doMock('@octokit/rest', () => { import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider'; import { RestEndpointMethodTypes } from '@octokit/rest'; import { DateTime } from 'luxon'; +import { ConfigReader } from '@backstage/config'; -const github = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', +let integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], }, - ], - token: 'hardcoded_token', -}); + }), +); -describe('DefaultGithubCredentialsProvider tests', () => { +const github = SingleInstanceGithubCredentialsProvider.create(integrations); + +describe('SingleInstanceGithubCredentialsProvider tests', () => { beforeEach(() => { jest.resetAllMocks(); }); @@ -204,11 +217,22 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return the default token if no app is configured', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [], - token: 'fallback_token', - }); + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [], + token: 'fallback_token', + }, + ], + }, + }), + ); + + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); await expect( githubProvider.getCredentials({ @@ -218,19 +242,29 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return the configured token if there are no installations', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - apps: [ - { - appId: 1, - privateKey: 'privateKey', - webhookSecret: '123', - clientId: 'CLIENT_ID', - clientSecret: 'CLIENT_SECRET', + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + apps: [ + { + appId: 1, + privateKey: 'privateKey', + webhookSecret: '123', + clientId: 'CLIENT_ID', + clientSecret: 'CLIENT_SECRET', + }, + ], + token: 'hardcoded_token', + }, + ], }, - ], - token: 'hardcoded_token', - }); + }), + ); + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); octokit.apps.listInstallations.mockResolvedValue({ data: [], } as unknown as RestEndpointMethodTypes['apps']['listInstallations']['response']); @@ -243,9 +277,19 @@ describe('DefaultGithubCredentialsProvider tests', () => { }); it('should return undefined if no token or apps are configured', async () => { - const githubProvider = SingleInstanceGithubCredentialsProvider.create({ - host: 'github.com', - }); + integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { + host: 'github.com', + }, + ], + }, + }), + ); + const githubProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); await expect( githubProvider.getCredentials({ diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index 5b6fa39e75..5b7aea02ec 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -22,9 +22,9 @@ import { DateTime } from 'luxon'; import { GithubCredentials, GithubCredentialsProvider, - GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; +import { ScmIntegrations } from '../ScmIntegrations'; type InstallationData = { installationId: number; @@ -232,17 +232,11 @@ export class GithubAppCredentialsMux { export class SingleInstanceGithubCredentialsProvider implements GithubCredentialsProvider { - static create: GithubCredentialsProviderFactory = config => { - return new SingleInstanceGithubCredentialsProvider( - new GithubAppCredentialsMux(config), - config.token, - ); - }; + static create(integrations: ScmIntegrations) { + return new SingleInstanceGithubCredentialsProvider(integrations); + } - private constructor( - private readonly githubAppCredentialsMux: GithubAppCredentialsMux, - private readonly token?: string, - ) {} + private constructor(private readonly integrations: ScmIntegrations) {} /** * Returns {@link GithubCredentials} for a given URL. @@ -266,15 +260,24 @@ export class SingleInstanceGithubCredentialsProvider */ async getCredentials(opts: { url: string }): Promise { const parsed = parseGitUrl(opts.url); + const gitHubConfig = this.integrations.github.byUrl(opts.url)?.config; + if (!gitHubConfig) { + throw new Error( + `There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`, + ); + } + + const githubAppCredentialsMux = new GithubAppCredentialsMux(gitHubConfig); + const defaultToken = gitHubConfig.token; const owner = parsed.owner || parsed.name; const repo = parsed.owner ? parsed.name : undefined; let type: GithubCredentialType = 'app'; - let token = await this.githubAppCredentialsMux.getAppToken(owner, repo); + let token = await githubAppCredentialsMux.getAppToken(owner, repo); if (!token) { type = 'token'; - token = this.token; + token = defaultToken; } return { diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index fa8dfb4b86..429a2b1c76 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -27,7 +27,6 @@ export { export type { GithubCredentials, GithubCredentialsProvider, - GithubCredentialsProviderFactory, GithubCredentialType, } from './types'; export { GitHubIntegration, replaceGitHubUrlType } from './GitHubIntegration'; diff --git a/packages/integration/src/github/types.ts b/packages/integration/src/github/types.ts index d8c1f0ee5e..15f5375570 100644 --- a/packages/integration/src/github/types.ts +++ b/packages/integration/src/github/types.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { GitHubIntegrationConfig } from './config'; - /** * The type of credentials produced by the credential provider. * @@ -43,13 +41,3 @@ export type GithubCredentials = { export interface GithubCredentialsProvider { getCredentials(opts: { url: string }): Promise; } - -/** - * This allows implementations to be provided to create credentials providers. - * - * @public - * - */ -export type GithubCredentialsProviderFactory = ( - config: GitHubIntegrationConfig, -) => GithubCredentialsProvider; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 59c25bb051..06886ecb4f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -19,6 +19,10 @@ import { LocationSpec } from '@backstage/catalog-model'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from './github'; import { ConfigReader } from '@backstage/config'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; jest.mock('./github'); const mockGetOrganizationRepositories = @@ -67,14 +71,18 @@ describe('GithubDiscoveryProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'blob' }], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); const location: LocationSpec = { type: 'not-github-discovery', target: 'https://github.com', @@ -85,17 +93,21 @@ describe('GithubDiscoveryProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { host: 'github.com', token: 'blob' }, - { host: 'ghe.example.net', token: 'blob' }, - ], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'blob' }, + { host: 'ghe.example.net', token: 'blob' }, + ], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); const location: LocationSpec = { type: 'github-discovery', target: 'https://not.github.com/apa', @@ -109,14 +121,18 @@ describe('GithubDiscoveryProcessor', () => { }); describe('handles repositories', () => { - const processor = GithubDiscoveryProcessor.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'blob' }], - }, - }), - { logger: getVoidLogger() }, - ); + const config = new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }); + const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const processor = GithubDiscoveryProcessor.fromConfig(config, { + logger: getVoidLogger(), + githubCredentialsProvider, + }); beforeEach(() => { mockGetOrganizationRepositories.mockClear(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index b1a1a948ba..01ba40b1fd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -17,9 +17,8 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, ScmIntegrations, - GithubCredentialsProviderFactory, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -44,36 +43,31 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; - private githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + private readonly githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, options: { logger: Logger; - githubCredentialsProviderFactory?: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }, ) { const integrations = ScmIntegrations.fromConfig(config); - const githubCredentialsProviderFactory = - options.githubCredentialsProviderFactory || - SingleInstanceGithubCredentialsProvider.create; return new GithubDiscoveryProcessor({ ...options, integrations, - githubCredentialsProviderFactory, }); } constructor(options: { integrations: ScmIntegrations; logger: Logger; - githubCredentialsProviderFactory: GithubCredentialsProviderFactory; + githubCredentialsProvider: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; - this.githubCredentialsProviderFactory = - options.githubCredentialsProviderFactory; + this.githubCredentialsProvider = options.githubCredentialsProvider; } async readLocation( @@ -102,9 +96,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { // about how to handle the wild card which is special for this processor. const orgUrl = `https://${host}/${org}`; - const { headers } = await this.githubCredentialsProviderFactory( - gitHubConfig, - ).getCredentials({ url: orgUrl }); + const { headers } = await this.githubCredentialsProvider.getCredentials({ + url: orgUrl, + }); const client = graphql.defaults({ baseUrl: gitHubConfig.apiBaseUrl, diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts index 5ad735ceeb..7a25b35078 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts @@ -18,7 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GithubAppCredentialsMux, - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, } from '@backstage/integration'; @@ -44,8 +44,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly orgs: GithubMultiOrgConfig; private readonly logger: Logger; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, + ) { const c = config.getOptionalConfig('catalog.processors.githubMultiOrg'); const integrations = ScmIntegrations.fromConfig(config); @@ -60,10 +67,12 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { integrations: ScmIntegrations; logger: Logger; orgs: GithubMultiOrgConfig; + githubCredentialsProvider: GithubCredentialsProvider; }) { this.integrations = options.integrations; this.logger = options.logger; this.orgs = options.orgs; + this.githubCredentialsProvider = options.githubCredentialsProvider; } async readLocation( @@ -86,8 +95,6 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const allUsersMap = new Map(); const baseUrl = new URL(location.target).origin; - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(gitHubConfig); const orgsToProcess = this.orgs.length ? this.orgs @@ -96,7 +103,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { for (const orgConfig of orgsToProcess) { try { const { headers, type: tokenType } = - await credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: `${baseUrl}/${orgConfig.name}`, }); const client = graphql.defaults({ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 9124637e52..953c926c71 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -17,8 +17,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, ScmIntegrations, + GithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; @@ -39,6 +39,14 @@ describe('GithubOrgReaderProcessor', () => { }, }), ); + let githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + type: 'app', + headers: { token: 'blah' }, + }); + }, + }; beforeEach(() => { jest.resetAllMocks(); @@ -48,6 +56,7 @@ describe('GithubOrgReaderProcessor', () => { const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', @@ -61,10 +70,14 @@ describe('GithubOrgReaderProcessor', () => { }); it('should not query for email addresses when GitHub Apps is used for authentication', async () => { - const mockGetCredentials = jest.fn().mockReturnValue({ - headers: { token: 'blah' }, - type: 'app', - }); + githubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + headers: { token: 'blah' }, + type: 'app', + }); + }, + }; const mockClient = jest.fn(); @@ -87,15 +100,10 @@ describe('GithubOrgReaderProcessor', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); - const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', @@ -111,10 +119,14 @@ describe('GithubOrgReaderProcessor', () => { }); it('should query for email addresses when token is used for authentication', async () => { - const mockGetCredentials = jest.fn().mockReturnValue({ - headers: { token: 'blah' }, - type: 'token', - }); + githubCredentialsProvider = { + getCredentials() { + return Promise.resolve({ + type: 'token', + headers: { token: 'blah' }, + }); + }, + }; const mockClient = jest.fn(); @@ -137,15 +149,10 @@ describe('GithubOrgReaderProcessor', () => { (graphql.defaults as jest.Mock).mockReturnValue(mockClient); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); - const processor = new GithubOrgReaderProcessor({ integrations, logger, + githubCredentialsProvider, }); const location: LocationSpec = { type: 'github-org', diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 95db344818..4c56b30c90 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -17,9 +17,9 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialType, ScmIntegrations, + GithubCredentialsProvider, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -40,8 +40,15 @@ type GraphQL = typeof graphql; export class GithubOrgReaderProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, + ) { const integrations = ScmIntegrations.fromConfig(config); return new GithubOrgReaderProcessor({ @@ -50,8 +57,13 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { }); } - constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + constructor(options: { + integrations: ScmIntegrations; + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }) { this.integrations = options.integrations; + this.githubCredentialsProvider = options.githubCredentialsProvider; this.logger = options.logger; } @@ -107,10 +119,8 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { ); } - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(gitHubConfig); const { headers, type: tokenType } = - await credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: orgUrl, }); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 6a85cc07c7..83f22249d7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -140,3 +140,9 @@ export class UrlReaderProcessor implements CatalogProcessor { return { response: [{ url: location, data }] }; } } + +export class RoadieDemoDataReaderProcessor extends UrlReaderProcessor { + getProcessorName() { + return 'roadie-demo-data-reader'; + } +} diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts index 6fde975a2b..e7aec1520d 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; import { GitHubOrgEntityProvider } from '.'; @@ -93,14 +93,13 @@ describe('GitHubOrgEntityProvider', () => { type: 'app', }); - jest - .spyOn(SingleInstanceGithubCredentialsProvider, 'create') - .mockReturnValue({ - getCredentials: mockGetCredentials, - } as any); + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; const entityProvider = new GitHubOrgEntityProvider({ id: 'my-id', + githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index 7d49a295ef..bc40ca1da5 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -20,7 +20,6 @@ import { } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialsProvider, GitHubIntegrationConfig, ScmIntegrations, @@ -40,11 +39,16 @@ import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; export class GitHubOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; - private readonly credentialsProvider: GithubCredentialsProvider; + private githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, - options: { id: string; orgUrl: string; logger: Logger }, + options: { + id: string; + orgUrl: string; + logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; + }, ) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -64,6 +68,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: options.orgUrl, logger, gitHubConfig, + githubCredentialsProvider: options.githubCredentialsProvider, }); } @@ -73,11 +78,10 @@ export class GitHubOrgEntityProvider implements EntityProvider { orgUrl: string; gitHubConfig: GitHubIntegrationConfig; logger: Logger; + githubCredentialsProvider: GithubCredentialsProvider; }, ) { - this.credentialsProvider = SingleInstanceGithubCredentialsProvider.create( - options.gitHubConfig, - ); + this.githubCredentialsProvider = options.githubCredentialsProvider; } getProviderName() { @@ -96,7 +100,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { const { markReadComplete } = trackProgress(this.options.logger); const { headers, type: tokenType } = - await this.credentialsProvider.getCredentials({ + await this.githubCredentialsProvider.getCredentials({ url: this.options.orgUrl, }); const client = graphql.defaults({ diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index 6fb52e245e..6fda605ae3 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -24,7 +24,10 @@ import { SchemaValidEntityPolicy, Validators, } from '@backstage/catalog-model'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import lodash from 'lodash'; import { EntitiesCatalog } from '../../catalog'; import { @@ -291,6 +294,8 @@ export class CatalogBuilder { private buildProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); this.checkDeprecatedReaderProcessors(); @@ -317,9 +322,15 @@ export class CatalogBuilder { processors.push( new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), - GithubDiscoveryProcessor.fromConfig(config, { logger }), + GithubDiscoveryProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), - GithubOrgReaderProcessor.fromConfig(config, { logger }), + GithubOrgReaderProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index 1618ef8d7b..24aefb2d96 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -25,7 +25,11 @@ import { SchemaValidEntityPolicy, Validators, } from '@backstage/catalog-model'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { createHash } from 'crypto'; import { Router } from 'express'; import lodash from 'lodash'; @@ -289,13 +293,21 @@ export class NextCatalogBuilder { getDefaultProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider: GithubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); return [ new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), - GithubDiscoveryProcessor.fromConfig(config, { logger }), - GithubOrgReaderProcessor.fromConfig(config, { logger }), + GithubDiscoveryProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), + GithubOrgReaderProcessor.fromConfig(config, { + logger, + githubCredentialsProvider, + }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 93ecceb76a..6975f1aa16 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -16,7 +16,11 @@ import { ContainerRunner, UrlReader } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { Config } from '@backstage/config'; import { createCatalogWriteAction, @@ -52,6 +56,8 @@ export const createBuiltinActions = (options: { }) => { const { reader, integrations, containerRunner, catalogClient, config } = options; + const githubCredentialsProvider: GithubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); const actions = [ createFetchPlainAction({ @@ -65,9 +71,11 @@ export const createBuiltinActions = (options: { createPublishGithubAction({ integrations, config, + githubCredentialsProvider, }), createPublishGithubPullRequestAction({ integrations, + githubCredentialsProvider, }), createPublishGitlabAction({ integrations, @@ -91,9 +99,11 @@ export const createBuiltinActions = (options: { createFilesystemRenameAction(), createGithubActionsDispatchAction({ integrations, + githubCredentialsProvider, }), createGithubWebhookAction({ integrations, + githubCredentialsProvider, }), ]; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts index 91b23f40dd..39a1ff6ad8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.test.ts @@ -15,7 +15,10 @@ */ import { OctokitProvider } from './OctokitProvider'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; describe('getOctokit', () => { @@ -29,7 +32,12 @@ describe('getOctokit', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const octokitProvider = new OctokitProvider(integrations); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); beforeEach(() => { jest.resetAllMocks(); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts index 9fdc6ced17..4a8bb8bbf6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/OctokitProvider.ts @@ -16,7 +16,6 @@ import { InputError } from '@backstage/errors'; import { - SingleInstanceGithubCredentialsProvider, GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; @@ -35,18 +34,14 @@ export type OctokitIntegration = { */ export class OctokitProvider { private readonly integrations: ScmIntegrationRegistry; - private readonly credentialsProviders: Map; + private readonly githubCredentialsProvider: GithubCredentialsProvider; - constructor(integrations: ScmIntegrationRegistry) { + constructor( + integrations: ScmIntegrationRegistry, + githubCredentialsProvider: GithubCredentialsProvider, + ) { this.integrations = integrations; - this.credentialsProviders = new Map( - integrations.github.list().map(integration => { - const provider = SingleInstanceGithubCredentialsProvider.create( - integration.config, - ); - return [integration.config.host, provider]; - }), - ); + this.githubCredentialsProvider = githubCredentialsProvider; } /** @@ -67,17 +62,9 @@ export class OctokitProvider { throw new InputError(`No integration for host ${host}`); } - const credentialsProvider = this.credentialsProviders.get(host); - - if (!credentialsProvider) { - throw new InputError( - `No matching credentials for host ${host}, please check your integrations config`, - ); - } - // TODO(blam): Consider changing this API to have owner, repo interface instead of URL as the it's // needless to create URL and then parse again the other side. - const { token } = await credentialsProvider.getCredentials({ + const { token } = await this.githubCredentialsProvider.getCredentials({ url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent( repo, )}`, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts index ab1bbfc488..6720de4e95 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.test.ts @@ -17,7 +17,10 @@ jest.mock('@octokit/rest'); import { createGithubActionsDispatchAction } from './githubActionsDispatch'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -33,7 +36,12 @@ describe('github:actions:dispatch', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const action = createGithubActionsDispatchAction({ integrations }); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const action = createGithubActionsDispatchAction({ + integrations, + githubCredentialsProvider, + }); const mockContext = { input: { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts index 661101d184..a88698ed07 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubActionsDispatch.ts @@ -13,15 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; export function createGithubActionsDispatchAction(options: { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, githubCredentialsProvider } = options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); return createTemplateAction<{ repoUrl: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts index 0da08bbf9d..0c2daeb42f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.test.ts @@ -17,7 +17,10 @@ jest.mock('@octokit/rest'); import { createGithubWebhookAction } from './githubWebhook'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -33,10 +36,13 @@ describe('github:repository:webhook:create', () => { }); const integrations = ScmIntegrations.fromConfig(config); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); const defaultWebhookSecret = 'aafdfdivierernfdk23f'; const action = createGithubWebhookAction({ integrations, defaultWebhookSecret, + githubCredentialsProvider, }); const mockContext = { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts index 0761f65f09..d341b59011 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubWebhook.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { createTemplateAction } from '../../createTemplateAction'; import { OctokitProvider } from './OctokitProvider'; import { emitterEventNames } from '@octokit/webhooks'; @@ -24,9 +27,14 @@ type ContentType = 'form' | 'json'; export function createGithubWebhookAction(options: { integrations: ScmIntegrationRegistry; defaultWebhookSecret?: string; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations, defaultWebhookSecret } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, defaultWebhookSecret, githubCredentialsProvider } = + options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); const eventNames = emitterEventNames.filter(event => !event.includes('.')); return createTemplateAction<{ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 6a3ccf5e38..74d31ecc7a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -18,7 +18,10 @@ jest.mock('../helpers'); jest.mock('@octokit/rest'); import { createPublishGithubAction } from './github'; -import { ScmIntegrations } from '@backstage/integration'; +import { + ScmIntegrations, + SingleInstanceGithubCredentialsProvider, +} from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; @@ -39,7 +42,13 @@ describe('publish:github', () => { }); const integrations = ScmIntegrations.fromConfig(config); - const action = createPublishGithubAction({ integrations, config }); + const githubCredentialsProvider = + SingleInstanceGithubCredentialsProvider.create(integrations); + const action = createPublishGithubAction({ + integrations, + config, + githubCredentialsProvider, + }); const mockContext = { input: { repoUrl: 'github.com?repo=repo&owner=owner', @@ -201,6 +210,7 @@ describe('publish:github', () => { const customAuthorAction = createPublishGithubAction({ integrations: customAuthorIntegrations, config: customAuthorConfig, + githubCredentialsProvider, }); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -244,6 +254,7 @@ describe('publish:github', () => { const customAuthorAction = createPublishGithubAction({ integrations: customAuthorIntegrations, config: customAuthorConfig, + githubCredentialsProvider, }); mockGithubClient.users.getByUsername.mockResolvedValue({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index ad6a4a7857..66a45eaba1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ScmIntegrationRegistry } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; import { enableBranchProtectionOnDefaultRepoBranch, initRepoAndPush, @@ -30,9 +33,13 @@ type Collaborator = { access: Permission; username: string }; export function createPublishGithubAction(options: { integrations: ScmIntegrationRegistry; config: Config; + githubCredentialsProvider: GithubCredentialsProvider; }) { - const { integrations, config } = options; - const octokitProvider = new OctokitProvider(integrations); + const { integrations, config, githubCredentialsProvider } = options; + const octokitProvider = new OctokitProvider( + integrations, + githubCredentialsProvider, + ); return createTemplateAction<{ repoUrl: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts index 0d4deb87e0..20ea4b03da 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts @@ -16,7 +16,10 @@ import { getRootLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; import mockFs from 'mock-fs'; import os from 'os'; import { resolve as resolvePath } from 'path'; @@ -53,9 +56,13 @@ describe('createPublishGithubPullRequestAction', () => { }), }; clientFactory = jest.fn(async () => fakeClient); + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: jest.fn(), + }; instance = createPublishGithubPullRequestAction({ integrations, + githubCredentialsProvider, clientFactory, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts index 299eb07f2e..e657c36e50 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts @@ -18,7 +18,7 @@ import fs from 'fs-extra'; import { parseRepoUrl, isExecutable } from './util'; import { - SingleInstanceGithubCredentialsProvider, + GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; import { zipObject } from 'lodash'; @@ -58,6 +58,7 @@ export type GithubPullRequestActionInput = { export type ClientFactoryInput = { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; host: string; owner: string; repo: string; @@ -65,6 +66,7 @@ export type ClientFactoryInput = { export const defaultClientFactory = async ({ integrations, + githubCredentialsProvider, owner, repo, host = 'github.com', @@ -75,16 +77,7 @@ export const defaultClientFactory = async ({ throw new InputError(`No integration for host ${host}`); } - const credentialsProvider = - SingleInstanceGithubCredentialsProvider.create(integrationConfig); - - if (!credentialsProvider) { - throw new InputError( - `No matching credentials for host ${host}, please check your integrations config`, - ); - } - - const { token } = await credentialsProvider.getCredentials({ + const { token } = await githubCredentialsProvider.getCredentials({ url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent( repo, )}`, @@ -106,11 +99,13 @@ export const defaultClientFactory = async ({ interface CreateGithubPullRequestActionOptions { integrations: ScmIntegrationRegistry; + githubCredentialsProvider: GithubCredentialsProvider; clientFactory?: (input: ClientFactoryInput) => Promise; } export const createPublishGithubPullRequestAction = ({ integrations, + githubCredentialsProvider, clientFactory = defaultClientFactory, }: CreateGithubPullRequestActionOptions) => { return createTemplateAction({ @@ -183,7 +178,13 @@ export const createPublishGithubPullRequestAction = ({ ); } - const client = await clientFactory({ integrations, host, owner, repo }); + const client = await clientFactory({ + integrations, + githubCredentialsProvider, + host, + owner, + repo, + }); const fileRoot = sourcePath ? resolveSafeChildPath(ctx.workspacePath, sourcePath) : ctx.workspacePath;