Merge pull request #8619 from RoadieHQ/demo-of-processor-using-creds-provider-interface

processor, actions and readers to use gh creds interface
This commit is contained in:
Ben Lambert
2022-01-11 10:30:44 +01:00
committed by GitHub
32 changed files with 608 additions and 165 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': patch
---
Adds a new GitHub credentials provider (DefaultGithubCredentialsProvider). It handles multiple app configurations. It looks up the app configuration based on the url.
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
Allow a custom GithubCredentialsProvider to be passed to the GitHub processors.
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-scaffolder-backend': patch
---
Allow a GitHubCredentialsProvider to be passed to the GitHub scaffolder tasks actions.
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': patch
---
The GithubUrlReader is switched to use the DefaultGithubCredentialsProvider
@@ -16,7 +16,7 @@
import {
getGitHubFileFetchUrl,
SingleInstanceGithubCredentialsProvider,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
GitHubIntegration,
ScmIntegrations,
@@ -58,9 +58,9 @@ export type GhBlobResponse =
export class GithubUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const integrations = ScmIntegrations.fromConfig(config);
const credentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
return integrations.github.list().map(integration => {
const credentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
const reader = new GithubUrlReader(integration, {
treeResponseFactory,
credentialsProvider,
+11
View File
@@ -94,6 +94,17 @@ export type BitbucketIntegrationConfig = {
appPassword?: string;
};
// @public
export class DefaultGithubCredentialsProvider
implements GithubCredentialsProvider
{
// (undocumented)
static fromIntegrations(
integrations: ScmIntegrationRegistry,
): DefaultGithubCredentialsProvider;
getCredentials(opts: { url: string }): Promise<GithubCredentials>;
}
// @public
export function defaultScmResolveUrl(options: {
url: string;
@@ -0,0 +1,123 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrations } from '../ScmIntegrations';
import { GitHubIntegrationConfig } from './config';
import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider';
import { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider';
import { ConfigReader } from '@backstage/config';
import { GithubCredentials } from './types';
const resultBuilder = (host: string): GithubCredentials => {
return {
type: 'token',
token: `${host}-token`,
headers: {
token: `${host}-token`,
},
};
};
jest.mock('./SingleInstanceGithubCredentialsProvider');
let integrations: ScmIntegrations;
describe('DefaultGithubCredentialsProvider tests', () => {
beforeEach(() => {
integrations = ScmIntegrations.fromConfig(
new ConfigReader({
integrations: {
github: [
{
host: 'github.com',
apps: [
{
appId: 1,
privateKey: 'privateKey',
webhookSecret: '123',
clientId: 'CLIENT_ID',
clientSecret: 'CLIENT_SECRET',
},
],
token: 'hardcoded_token',
},
{
host: 'grithub.com',
token: 'hardcoded_token',
},
],
},
}),
);
jest.resetAllMocks();
SingleInstanceGithubCredentialsProvider.create = (
config: GitHubIntegrationConfig,
) => {
return {
getCredentials: (_opts: { url: string }) => {
return Promise.resolve(resultBuilder(config.host));
},
};
};
jest.spyOn(SingleInstanceGithubCredentialsProvider, 'create');
});
describe('.create', () => {
it('passes the config through to the single provider', () => {
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const githubIntegration =
integrations.github.byHost('github.com')?.config;
const grithubIntegration =
integrations.github.byHost('grithub.com')?.config;
expect(
SingleInstanceGithubCredentialsProvider.create,
).toHaveBeenCalledWith(githubIntegration);
expect(
SingleInstanceGithubCredentialsProvider.create,
).toHaveBeenCalledWith(grithubIntegration);
});
});
describe('#getCredentials', () => {
it('returns the data verbatim from the creds provider', async () => {
const provider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const gitHubCredentials = await provider.getCredentials({
url: 'https://github.com/blah',
});
const gritHubCredentials = await provider.getCredentials({
url: 'https://grithub.com/blah',
});
expect(gitHubCredentials).toEqual({
type: 'token',
token: 'github.com-token',
headers: {
token: 'github.com-token',
},
});
expect(gritHubCredentials).toEqual({
type: 'token',
token: 'grithub.com-token',
headers: {
token: 'grithub.com-token',
},
});
});
});
});
@@ -0,0 +1,84 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GithubCredentials, GithubCredentialsProvider } from './types';
import { ScmIntegrationRegistry } from '../registry';
import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubCredentialsProvider';
/**
* Handles the creation and caching of credentials for GitHub integrations.
*
* @public
* @remarks
*
* TODO: Possibly move this to a backend only package so that it's not used in the frontend by mistake
*/
export class DefaultGithubCredentialsProvider
implements GithubCredentialsProvider
{
static fromIntegrations(integrations: ScmIntegrationRegistry) {
const credentialsProviders: Map<string, GithubCredentialsProvider> =
new Map<string, GithubCredentialsProvider>();
integrations.github.list().forEach(integration => {
const credentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
credentialsProviders.set(integration.config.host, credentialsProvider);
});
return new DefaultGithubCredentialsProvider(credentialsProviders);
}
private constructor(
private readonly providers: Map<string, GithubCredentialsProvider>,
) {}
/**
* Returns {@link GithubCredentials} for a given URL.
*
* @remarks
*
* Consecutive calls to this method with the same URL will return cached
* credentials.
*
* The shortest lifetime for a token returned is 10 minutes.
*
* @example
* ```ts
* const { token, headers } = await getCredentials({
* url: 'https://github.com/backstage/foobar'
* })
*
* const { token, headers } = await getCredentials({
* url: 'https://github.com/backstage'
* })
* ```
*
* @param opts - The organization or repository URL
* @returns A promise of {@link GithubCredentials}.
*/
async getCredentials(opts: { url: string }): Promise<GithubCredentials> {
const parsed = new URL(opts.url);
const provider = this.providers.get(parsed.host);
if (!provider) {
throw new Error(
`There is no GitHub integration that matches ${opts.url}. Please add a configuration for an integration.`,
);
}
return provider.getCredentials(opts);
}
}
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { GithubCredentialsProvider } from './types';
const octokit = {
paginate: async (fn: any) => (await fn()).data,
apps: {
@@ -35,23 +37,24 @@ import { SingleInstanceGithubCredentialsProvider } from './SingleInstanceGithubC
import { RestEndpointMethodTypes } from '@octokit/rest';
import { DateTime } from 'luxon';
const github = SingleInstanceGithubCredentialsProvider.create({
host: 'github.com',
apps: [
{
appId: 1,
privateKey: 'privateKey',
webhookSecret: '123',
clientId: 'CLIENT_ID',
clientSecret: 'CLIENT_SECRET',
},
],
token: 'hardcoded_token',
});
describe('SingleInstanceGithubCredentialsProvider tests', () => {
let github: GithubCredentialsProvider;
describe('DefaultGithubCredentialsProvider tests', () => {
beforeEach(() => {
jest.resetAllMocks();
github = SingleInstanceGithubCredentialsProvider.create({
host: 'github.com',
apps: [
{
appId: 1,
privateKey: 'privateKey',
webhookSecret: '123',
clientId: 'CLIENT_ID',
clientSecret: 'CLIENT_SECRET',
},
],
token: 'hardcoded_token',
});
});
it('create repository specific tokens', async () => {
octokit.apps.listInstallations.mockResolvedValue({
+1
View File
@@ -20,6 +20,7 @@ export {
} from './config';
export type { GithubAppConfig, GitHubIntegrationConfig } from './config';
export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
export { DefaultGithubCredentialsProvider } from './DefaultGithubCredentialsProvider';
export {
GithubAppCredentialsMux,
SingleInstanceGithubCredentialsProvider,
+18 -3
View File
@@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model';
import { EntityPolicy } from '@backstage/catalog-model';
import { EntityRelationSpec } from '@backstage/catalog-model';
import express from 'express';
import { GithubCredentialsProvider } from '@backstage/integration';
import { GitHubIntegrationConfig } from '@backstage/integration';
import { IndexableDocument } from '@backstage/search-common';
import { JsonObject } from '@backstage/types';
@@ -1005,12 +1006,17 @@ function generalError(
//
// @public
export class GithubDiscoveryProcessor implements CatalogProcessor {
constructor(options: { integrations: ScmIntegrations; logger: Logger_2 });
constructor(options: {
integrations: ScmIntegrations;
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubDiscoveryProcessor;
// (undocumented)
@@ -1027,12 +1033,14 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
integrations: ScmIntegrations;
logger: Logger_2;
orgs: GithubMultiOrgConfig;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubMultiOrgReaderProcessor;
// (undocumented)
@@ -1052,6 +1060,7 @@ export class GitHubOrgEntityProvider implements EntityProvider {
orgUrl: string;
gitHubConfig: GitHubIntegrationConfig;
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
@@ -1062,6 +1071,7 @@ export class GitHubOrgEntityProvider implements EntityProvider {
id: string;
orgUrl: string;
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GitHubOrgEntityProvider;
// (undocumented)
@@ -1074,12 +1084,17 @@ export class GitHubOrgEntityProvider implements EntityProvider {
//
// @public
export class GithubOrgReaderProcessor implements CatalogProcessor {
constructor(options: { integrations: ScmIntegrations; logger: Logger_2 });
constructor(options: {
integrations: ScmIntegrations;
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
});
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger_2;
githubCredentialsProvider?: GithubCredentialsProvider;
},
): GithubOrgReaderProcessor;
// (undocumented)
@@ -1589,5 +1604,5 @@ export class UrlReaderProcessor implements CatalogProcessor {
// Warnings were encountered during analysis:
//
// src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:23:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts
// src/ingestion/processors/GithubMultiOrgReaderProcessor.d.ts:25:9 - (ae-forgotten-export) The symbol "GithubMultiOrgConfig" needs to be exported by the entry point index.d.ts
```
@@ -19,6 +19,10 @@ import { LocationSpec } from '@backstage/catalog-model';
import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
import { getOrganizationRepositories } from './github';
import { ConfigReader } from '@backstage/config';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
jest.mock('./github');
const mockGetOrganizationRepositories =
@@ -67,14 +71,18 @@ describe('GithubDiscoveryProcessor', () => {
describe('reject unrelated entries', () => {
it('rejects unknown types', async () => {
const processor = GithubDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
github: [{ host: 'github.com', token: 'blob' }],
},
}),
{ logger: getVoidLogger() },
);
const config = new ConfigReader({
integrations: {
github: [{ host: 'github.com', token: 'blob' }],
},
});
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const processor = GithubDiscoveryProcessor.fromConfig(config, {
logger: getVoidLogger(),
githubCredentialsProvider,
});
const location: LocationSpec = {
type: 'not-github-discovery',
target: 'https://github.com',
@@ -85,17 +93,21 @@ describe('GithubDiscoveryProcessor', () => {
});
it('rejects unknown targets', async () => {
const processor = GithubDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
github: [
{ host: 'github.com', token: 'blob' },
{ host: 'ghe.example.net', token: 'blob' },
],
},
}),
{ logger: getVoidLogger() },
);
const config = new ConfigReader({
integrations: {
github: [
{ host: 'github.com', token: 'blob' },
{ host: 'ghe.example.net', token: 'blob' },
],
},
});
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const processor = GithubDiscoveryProcessor.fromConfig(config, {
logger: getVoidLogger(),
githubCredentialsProvider,
});
const location: LocationSpec = {
type: 'github-discovery',
target: 'https://not.github.com/apa',
@@ -109,14 +121,18 @@ describe('GithubDiscoveryProcessor', () => {
});
describe('handles repositories', () => {
const processor = GithubDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
github: [{ host: 'github.com', token: 'blob' }],
},
}),
{ logger: getVoidLogger() },
);
const config = new ConfigReader({
integrations: {
github: [{ host: 'github.com', token: 'blob' }],
},
});
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const processor = GithubDiscoveryProcessor.fromConfig(config, {
logger: getVoidLogger(),
githubCredentialsProvider,
});
beforeEach(() => {
mockGetOrganizationRepositories.mockClear();
@@ -17,7 +17,8 @@
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
SingleInstanceGithubCredentialsProvider,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrations,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
@@ -43,8 +44,15 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types';
export class GithubDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
private readonly logger: Logger;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
) {
const integrations = ScmIntegrations.fromConfig(config);
return new GithubDiscoveryProcessor({
@@ -53,9 +61,16 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
});
}
constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
constructor(options: {
integrations: ScmIntegrations;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
this.integrations = options.integrations;
this.logger = options.logger;
this.githubCredentialsProvider =
options.githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
}
async readLocation(
@@ -84,9 +99,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
// about how to handle the wild card which is special for this processor.
const orgUrl = `https://${host}/${org}`;
const { headers } = await SingleInstanceGithubCredentialsProvider.create(
gitHubConfig,
).getCredentials({ url: orgUrl });
const { headers } = await this.githubCredentialsProvider.getCredentials({
url: orgUrl,
});
const client = graphql.defaults({
baseUrl: gitHubConfig.apiBaseUrl,
@@ -17,8 +17,9 @@
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
DefaultGithubCredentialsProvider,
GithubAppCredentialsMux,
SingleInstanceGithubCredentialsProvider,
GithubCredentialsProvider,
GitHubIntegrationConfig,
ScmIntegrations,
} from '@backstage/integration';
@@ -44,8 +45,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
private readonly orgs: GithubMultiOrgConfig;
private readonly logger: Logger;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
) {
const c = config.getOptionalConfig('catalog.processors.githubMultiOrg');
const integrations = ScmIntegrations.fromConfig(config);
@@ -60,10 +68,14 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
integrations: ScmIntegrations;
logger: Logger;
orgs: GithubMultiOrgConfig;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
this.integrations = options.integrations;
this.logger = options.logger;
this.orgs = options.orgs;
this.githubCredentialsProvider =
options.githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
}
async readLocation(
@@ -86,8 +98,6 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
const allUsersMap = new Map();
const baseUrl = new URL(location.target).origin;
const credentialsProvider =
SingleInstanceGithubCredentialsProvider.create(gitHubConfig);
const orgsToProcess = this.orgs.length
? this.orgs
@@ -96,7 +106,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
for (const orgConfig of orgsToProcess) {
try {
const { headers, type: tokenType } =
await credentialsProvider.getCredentials({
await this.githubCredentialsProvider.getCredentials({
url: `${baseUrl}/${orgConfig.name}`,
});
const client = graphql.defaults({
@@ -17,8 +17,8 @@ import { getVoidLogger } from '@backstage/backend-common';
import { LocationSpec } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import {
SingleInstanceGithubCredentialsProvider,
ScmIntegrations,
GithubCredentialsProvider,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
@@ -39,6 +39,14 @@ describe('GithubOrgReaderProcessor', () => {
},
}),
);
let githubCredentialsProvider: GithubCredentialsProvider = {
getCredentials() {
return Promise.resolve({
type: 'app',
headers: { token: 'blah' },
});
},
};
beforeEach(() => {
jest.resetAllMocks();
@@ -48,6 +56,7 @@ describe('GithubOrgReaderProcessor', () => {
const processor = new GithubOrgReaderProcessor({
integrations,
logger,
githubCredentialsProvider,
});
const location: LocationSpec = {
type: 'github-org',
@@ -61,10 +70,14 @@ describe('GithubOrgReaderProcessor', () => {
});
it('should not query for email addresses when GitHub Apps is used for authentication', async () => {
const mockGetCredentials = jest.fn().mockReturnValue({
headers: { token: 'blah' },
type: 'app',
});
githubCredentialsProvider = {
getCredentials() {
return Promise.resolve({
headers: { token: 'blah' },
type: 'app',
});
},
};
const mockClient = jest.fn();
@@ -87,15 +100,10 @@ describe('GithubOrgReaderProcessor', () => {
(graphql.defaults as jest.Mock).mockReturnValue(mockClient);
jest
.spyOn(SingleInstanceGithubCredentialsProvider, 'create')
.mockReturnValue({
getCredentials: mockGetCredentials,
} as any);
const processor = new GithubOrgReaderProcessor({
integrations,
logger,
githubCredentialsProvider,
});
const location: LocationSpec = {
type: 'github-org',
@@ -111,10 +119,14 @@ describe('GithubOrgReaderProcessor', () => {
});
it('should query for email addresses when token is used for authentication', async () => {
const mockGetCredentials = jest.fn().mockReturnValue({
headers: { token: 'blah' },
type: 'token',
});
githubCredentialsProvider = {
getCredentials() {
return Promise.resolve({
type: 'token',
headers: { token: 'blah' },
});
},
};
const mockClient = jest.fn();
@@ -137,15 +149,10 @@ describe('GithubOrgReaderProcessor', () => {
(graphql.defaults as jest.Mock).mockReturnValue(mockClient);
jest
.spyOn(SingleInstanceGithubCredentialsProvider, 'create')
.mockReturnValue({
getCredentials: mockGetCredentials,
} as any);
const processor = new GithubOrgReaderProcessor({
integrations,
logger,
githubCredentialsProvider,
});
const location: LocationSpec = {
type: 'github-org',
@@ -17,9 +17,10 @@
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
SingleInstanceGithubCredentialsProvider,
GithubCredentialType,
ScmIntegrations,
GithubCredentialsProvider,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
@@ -40,8 +41,15 @@ type GraphQL = typeof graphql;
export class GithubOrgReaderProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
private readonly logger: Logger;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: {
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
) {
const integrations = ScmIntegrations.fromConfig(config);
return new GithubOrgReaderProcessor({
@@ -50,8 +58,15 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
});
}
constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
constructor(options: {
integrations: ScmIntegrations;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
this.integrations = options.integrations;
this.githubCredentialsProvider =
options.githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
this.logger = options.logger;
}
@@ -107,10 +122,8 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
);
}
const credentialsProvider =
SingleInstanceGithubCredentialsProvider.create(gitHubConfig);
const { headers, type: tokenType } =
await credentialsProvider.getCredentials({
await this.githubCredentialsProvider.getCredentials({
url: orgUrl,
});
@@ -17,7 +17,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import {
SingleInstanceGithubCredentialsProvider,
GithubCredentialsProvider,
GitHubIntegrationConfig,
} from '@backstage/integration';
import { GitHubOrgEntityProvider } from '.';
@@ -93,14 +93,13 @@ describe('GitHubOrgEntityProvider', () => {
type: 'app',
});
jest
.spyOn(SingleInstanceGithubCredentialsProvider, 'create')
.mockReturnValue({
getCredentials: mockGetCredentials,
} as any);
const githubCredentialsProvider: GithubCredentialsProvider = {
getCredentials: mockGetCredentials,
};
const entityProvider = new GitHubOrgEntityProvider({
id: 'my-id',
githubCredentialsProvider,
orgUrl: 'https://github.com/backstage',
gitHubConfig,
logger,
@@ -20,18 +20,16 @@ import {
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
SingleInstanceGithubCredentialsProvider,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
GitHubIntegrationConfig,
ScmIntegrations,
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import { merge } from 'lodash';
import { Logger } from 'winston';
import {
EntityProvider,
EntityProviderConnection,
} from '../../providers/types';
import { EntityProvider, EntityProviderConnection } from '../../providers';
import {
getOrganizationTeams,
getOrganizationUsers,
@@ -43,11 +41,16 @@ import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org';
export class GitHubOrgEntityProvider implements EntityProvider {
private connection?: EntityProviderConnection;
private readonly credentialsProvider: GithubCredentialsProvider;
private githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(
config: Config,
options: { id: string; orgUrl: string; logger: Logger },
options: {
id: string;
orgUrl: string;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
) {
const integrations = ScmIntegrations.fromConfig(config);
const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config;
@@ -67,6 +70,9 @@ export class GitHubOrgEntityProvider implements EntityProvider {
orgUrl: options.orgUrl,
logger,
gitHubConfig,
githubCredentialsProvider:
options.githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(integrations),
});
}
@@ -76,11 +82,12 @@ export class GitHubOrgEntityProvider implements EntityProvider {
orgUrl: string;
gitHubConfig: GitHubIntegrationConfig;
logger: Logger;
githubCredentialsProvider?: GithubCredentialsProvider;
},
) {
this.credentialsProvider = SingleInstanceGithubCredentialsProvider.create(
options.gitHubConfig,
);
this.githubCredentialsProvider =
options.githubCredentialsProvider ||
SingleInstanceGithubCredentialsProvider.create(options.gitHubConfig);
}
getProviderName() {
@@ -99,7 +106,7 @@ export class GitHubOrgEntityProvider implements EntityProvider {
const { markReadComplete } = trackProgress(this.options.logger);
const { headers, type: tokenType } =
await this.credentialsProvider.getCredentials({
await this.githubCredentialsProvider.getCredentials({
url: this.options.orgUrl,
});
const client = graphql.defaults({
@@ -24,7 +24,10 @@ import {
SchemaValidEntityPolicy,
Validators,
} from '@backstage/catalog-model';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import lodash from 'lodash';
import { EntitiesCatalog } from '../../catalog';
import {
@@ -291,6 +294,8 @@ export class CatalogBuilder {
private buildProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
this.checkDeprecatedReaderProcessors();
@@ -317,9 +322,15 @@ export class CatalogBuilder {
processors.push(
new FileReaderProcessor(),
BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
GithubDiscoveryProcessor.fromConfig(config, { logger }),
GithubDiscoveryProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
GitLabDiscoveryProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
@@ -26,7 +26,11 @@ import {
SchemaValidEntityPolicy,
Validators,
} from '@backstage/catalog-model';
import { ScmIntegrations } from '@backstage/integration';
import {
GithubCredentialsProvider,
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { createHash } from 'crypto';
import { Router } from 'express';
import lodash from 'lodash';
@@ -301,13 +305,21 @@ export class NextCatalogBuilder {
getDefaultProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const integrations = ScmIntegrations.fromConfig(config);
const githubCredentialsProvider: GithubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
return [
new FileReaderProcessor(),
BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
GithubDiscoveryProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
GithubDiscoveryProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
GithubOrgReaderProcessor.fromConfig(config, {
logger,
githubCredentialsProvider,
}),
GitLabDiscoveryProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
+10 -2
View File
@@ -14,6 +14,7 @@ import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-back
import { createPullRequest } from 'octokit-plugin-create-pull-request';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import { GithubCredentialsProvider } from '@backstage/integration';
import { JsonObject } from '@backstage/types';
import { JsonValue } from '@backstage/types';
import { Knex } from 'knex';
@@ -126,7 +127,8 @@ export const createFilesystemRenameAction: () => TemplateAction<any>;
//
// @public (undocumented)
export function createGithubActionsDispatchAction(options: {
integrations: ScmIntegrationRegistry;
integrations: ScmIntegrations;
githubCredentialsProvider?: GithubCredentialsProvider;
}): TemplateAction<any>;
// Warning: (ae-missing-release-tag) "createGithubWebhookAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -135,6 +137,7 @@ export function createGithubActionsDispatchAction(options: {
export function createGithubWebhookAction(options: {
integrations: ScmIntegrationRegistry;
defaultWebhookSecret?: string;
githubCredentialsProvider?: GithubCredentialsProvider;
}): TemplateAction<any>;
// Warning: (ae-missing-release-tag) "createPublishAzureAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -164,6 +167,7 @@ export function createPublishFileAction(): TemplateAction<any>;
export function createPublishGithubAction(options: {
integrations: ScmIntegrationRegistry;
config: Config;
githubCredentialsProvider?: GithubCredentialsProvider;
}): TemplateAction<any>;
// Warning: (ae-forgotten-export) The symbol "CreateGithubPullRequestActionOptions" needs to be exported by the entry point index.d.ts
@@ -172,6 +176,7 @@ export function createPublishGithubAction(options: {
// @public (undocumented)
export const createPublishGithubPullRequestAction: ({
integrations,
githubCredentialsProvider,
clientFactory,
}: CreateGithubPullRequestActionOptions) => TemplateAction<any>;
@@ -285,7 +290,10 @@ export function fetchContents({
//
// @public
export class OctokitProvider {
constructor(integrations: ScmIntegrationRegistry);
constructor(
integrations: ScmIntegrationRegistry,
githubCredentialsProvider?: GithubCredentialsProvider,
);
// Warning: (ae-forgotten-export) The symbol "OctokitIntegration" needs to be exported by the entry point index.d.ts
getOctokit(repoUrl: string): Promise<OctokitIntegration>;
}
@@ -16,7 +16,11 @@
import { ContainerRunner, UrlReader } from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
import { ScmIntegrations } from '@backstage/integration';
import {
GithubCredentialsProvider,
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { Config } from '@backstage/config';
import {
createCatalogWriteAction,
@@ -52,6 +56,8 @@ export const createBuiltinActions = (options: {
}) => {
const { reader, integrations, containerRunner, catalogClient, config } =
options;
const githubCredentialsProvider: GithubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const actions = [
createFetchPlainAction({
@@ -65,9 +71,11 @@ export const createBuiltinActions = (options: {
createPublishGithubAction({
integrations,
config,
githubCredentialsProvider,
}),
createPublishGithubPullRequestAction({
integrations,
githubCredentialsProvider,
}),
createPublishGitlabAction({
integrations,
@@ -91,9 +99,11 @@ export const createBuiltinActions = (options: {
createFilesystemRenameAction(),
createGithubActionsDispatchAction({
integrations,
githubCredentialsProvider,
}),
createGithubWebhookAction({
integrations,
githubCredentialsProvider,
}),
];
@@ -15,7 +15,10 @@
*/
import { OctokitProvider } from './OctokitProvider';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
} from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
describe('getOctokit', () => {
@@ -29,7 +32,12 @@ describe('getOctokit', () => {
});
const integrations = ScmIntegrations.fromConfig(config);
const octokitProvider = new OctokitProvider(integrations);
const githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
const octokitProvider = new OctokitProvider(
integrations,
githubCredentialsProvider,
);
beforeEach(() => {
jest.resetAllMocks();
@@ -16,7 +16,7 @@
import { InputError } from '@backstage/errors';
import {
SingleInstanceGithubCredentialsProvider,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrationRegistry,
} from '@backstage/integration';
@@ -35,18 +35,16 @@ export type OctokitIntegration = {
*/
export class OctokitProvider {
private readonly integrations: ScmIntegrationRegistry;
private readonly credentialsProviders: Map<string, GithubCredentialsProvider>;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
constructor(integrations: ScmIntegrationRegistry) {
constructor(
integrations: ScmIntegrationRegistry,
githubCredentialsProvider?: GithubCredentialsProvider,
) {
this.integrations = integrations;
this.credentialsProviders = new Map(
integrations.github.list().map(integration => {
const provider = SingleInstanceGithubCredentialsProvider.create(
integration.config,
);
return [integration.config.host, provider];
}),
);
this.githubCredentialsProvider =
githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(this.integrations);
}
/**
@@ -67,17 +65,9 @@ export class OctokitProvider {
throw new InputError(`No integration for host ${host}`);
}
const credentialsProvider = this.credentialsProviders.get(host);
if (!credentialsProvider) {
throw new InputError(
`No matching credentials for host ${host}, please check your integrations config`,
);
}
// TODO(blam): Consider changing this API to have owner, repo interface instead of URL as the it's
// needless to create URL and then parse again the other side.
const { token } = await credentialsProvider.getCredentials({
const { token } = await this.githubCredentialsProvider.getCredentials({
url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent(
repo,
)}`,
@@ -15,9 +15,14 @@
*/
jest.mock('@octokit/rest');
import { TemplateAction } from '../../types';
import { createGithubActionsDispatchAction } from './githubActionsDispatch';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
} from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import { PassThrough } from 'stream';
@@ -33,7 +38,8 @@ describe('github:actions:dispatch', () => {
});
const integrations = ScmIntegrations.fromConfig(config);
const action = createGithubActionsDispatchAction({ integrations });
let githubCredentialsProvider: GithubCredentialsProvider;
let action: TemplateAction<any>;
const mockContext = {
input: {
@@ -52,6 +58,12 @@ describe('github:actions:dispatch', () => {
beforeEach(() => {
jest.resetAllMocks();
githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
action = createGithubActionsDispatchAction({
integrations,
githubCredentialsProvider,
});
});
it('should call the githubApis for creating WorkflowDispatch', async () => {
@@ -13,15 +13,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrations,
} from '@backstage/integration';
import { createTemplateAction } from '../../createTemplateAction';
import { OctokitProvider } from './OctokitProvider';
export function createGithubActionsDispatchAction(options: {
integrations: ScmIntegrationRegistry;
integrations: ScmIntegrations;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
const { integrations } = options;
const octokitProvider = new OctokitProvider(integrations);
const { integrations, githubCredentialsProvider } = options;
const octokitProvider = new OctokitProvider(
integrations,
githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(integrations),
);
return createTemplateAction<{
repoUrl: string;
@@ -17,10 +17,15 @@
jest.mock('@octokit/rest');
import { createGithubWebhookAction } from './githubWebhook';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
} from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import { PassThrough } from 'stream';
import { TemplateAction } from '../..';
describe('github:repository:webhook:create', () => {
const config = new ConfigReader({
@@ -33,10 +38,19 @@ describe('github:repository:webhook:create', () => {
});
const integrations = ScmIntegrations.fromConfig(config);
let githubCredentialsProvider: GithubCredentialsProvider;
const defaultWebhookSecret = 'aafdfdivierernfdk23f';
const action = createGithubWebhookAction({
integrations,
defaultWebhookSecret,
let action: TemplateAction<any>;
beforeEach(() => {
jest.resetAllMocks();
githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
action = createGithubWebhookAction({
integrations,
defaultWebhookSecret,
githubCredentialsProvider,
});
});
const mockContext = {
@@ -53,10 +67,6 @@ describe('github:repository:webhook:create', () => {
const { mockGithubClient } = require('@octokit/rest');
beforeEach(() => {
jest.resetAllMocks();
});
it('should call the githubApi for creating repository Webhook', async () => {
const repoUrl = 'github.com?repo=repo&owner=owner';
const webhookUrl = 'https://example.com/payload';
@@ -13,7 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrationRegistry,
} from '@backstage/integration';
import { createTemplateAction } from '../../createTemplateAction';
import { OctokitProvider } from './OctokitProvider';
import { emitterEventNames } from '@octokit/webhooks';
@@ -24,9 +28,15 @@ type ContentType = 'form' | 'json';
export function createGithubWebhookAction(options: {
integrations: ScmIntegrationRegistry;
defaultWebhookSecret?: string;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
const { integrations, defaultWebhookSecret } = options;
const octokitProvider = new OctokitProvider(integrations);
const { integrations, defaultWebhookSecret, githubCredentialsProvider } =
options;
const octokitProvider = new OctokitProvider(
integrations,
githubCredentialsProvider ??
DefaultGithubCredentialsProvider.fromIntegrations(integrations),
);
const eventNames = emitterEventNames.filter(event => !event.includes('.'));
return createTemplateAction<{
@@ -14,11 +14,17 @@
* limitations under the License.
*/
import { TemplateAction } from '../../types';
jest.mock('../helpers');
jest.mock('@octokit/rest');
import { createPublishGithubAction } from './github';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrations,
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
} from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger } from '@backstage/backend-common';
import { PassThrough } from 'stream';
@@ -39,7 +45,9 @@ describe('publish:github', () => {
});
const integrations = ScmIntegrations.fromConfig(config);
const action = createPublishGithubAction({ integrations, config });
let githubCredentialsProvider: GithubCredentialsProvider;
let action: TemplateAction<any>;
const mockContext = {
input: {
repoUrl: 'github.com?repo=repo&owner=owner',
@@ -58,6 +66,13 @@ describe('publish:github', () => {
beforeEach(() => {
jest.resetAllMocks();
githubCredentialsProvider =
DefaultGithubCredentialsProvider.fromIntegrations(integrations);
action = createPublishGithubAction({
integrations,
config,
githubCredentialsProvider,
});
});
it('should call the githubApis with the correct values for createInOrg', async () => {
@@ -201,6 +216,7 @@ describe('publish:github', () => {
const customAuthorAction = createPublishGithubAction({
integrations: customAuthorIntegrations,
config: customAuthorConfig,
githubCredentialsProvider,
});
mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -244,6 +260,7 @@ describe('publish:github', () => {
const customAuthorAction = createPublishGithubAction({
integrations: customAuthorIntegrations,
config: customAuthorConfig,
githubCredentialsProvider,
});
mockGithubClient.users.getByUsername.mockResolvedValue({
@@ -13,7 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
DefaultGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrationRegistry,
} from '@backstage/integration';
import {
enableBranchProtectionOnDefaultRepoBranch,
initRepoAndPush,
@@ -30,9 +34,14 @@ type Collaborator = { access: Permission; username: string };
export function createPublishGithubAction(options: {
integrations: ScmIntegrationRegistry;
config: Config;
githubCredentialsProvider?: GithubCredentialsProvider;
}) {
const { integrations, config } = options;
const octokitProvider = new OctokitProvider(integrations);
const { integrations, config, githubCredentialsProvider } = options;
const octokitProvider = new OctokitProvider(
integrations,
githubCredentialsProvider ||
DefaultGithubCredentialsProvider.fromIntegrations(integrations),
);
return createTemplateAction<{
repoUrl: string;
@@ -16,7 +16,10 @@
import { getRootLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import {
GithubCredentialsProvider,
ScmIntegrations,
} from '@backstage/integration';
import mockFs from 'mock-fs';
import os from 'os';
import { resolve as resolvePath } from 'path';
@@ -53,9 +56,13 @@ describe('createPublishGithubPullRequestAction', () => {
}),
};
clientFactory = jest.fn(async () => fakeClient);
const githubCredentialsProvider: GithubCredentialsProvider = {
getCredentials: jest.fn(),
};
instance = createPublishGithubPullRequestAction({
integrations,
githubCredentialsProvider,
clientFactory,
});
});
@@ -18,8 +18,9 @@ import fs from 'fs-extra';
import { parseRepoUrl, isExecutable } from './util';
import {
SingleInstanceGithubCredentialsProvider,
GithubCredentialsProvider,
ScmIntegrationRegistry,
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import { zipObject } from 'lodash';
import { createTemplateAction } from '../../createTemplateAction';
@@ -58,6 +59,7 @@ export type GithubPullRequestActionInput = {
export type ClientFactoryInput = {
integrations: ScmIntegrationRegistry;
githubCredentialsProvider?: GithubCredentialsProvider;
host: string;
owner: string;
repo: string;
@@ -65,6 +67,7 @@ export type ClientFactoryInput = {
export const defaultClientFactory = async ({
integrations,
githubCredentialsProvider,
owner,
repo,
host = 'github.com',
@@ -76,14 +79,9 @@ export const defaultClientFactory = async ({
}
const credentialsProvider =
githubCredentialsProvider ||
SingleInstanceGithubCredentialsProvider.create(integrationConfig);
if (!credentialsProvider) {
throw new InputError(
`No matching credentials for host ${host}, please check your integrations config`,
);
}
const { token } = await credentialsProvider.getCredentials({
url: `https://${host}/${encodeURIComponent(owner)}/${encodeURIComponent(
repo,
@@ -106,11 +104,13 @@ export const defaultClientFactory = async ({
interface CreateGithubPullRequestActionOptions {
integrations: ScmIntegrationRegistry;
githubCredentialsProvider?: GithubCredentialsProvider;
clientFactory?: (input: ClientFactoryInput) => Promise<PullRequestCreator>;
}
export const createPublishGithubPullRequestAction = ({
integrations,
githubCredentialsProvider,
clientFactory = defaultClientFactory,
}: CreateGithubPullRequestActionOptions) => {
return createTemplateAction<GithubPullRequestActionInput>({
@@ -183,7 +183,13 @@ export const createPublishGithubPullRequestAction = ({
);
}
const client = await clientFactory({ integrations, host, owner, repo });
const client = await clientFactory({
integrations,
githubCredentialsProvider,
host,
owner,
repo,
});
const fileRoot = sourcePath
? resolveSafeChildPath(ctx.workspacePath, sourcePath)
: ctx.workspacePath;