paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

techdocs: Use @backstage/integration for scm tokens and request options

Browse Source
This commit is contained in:
Himanshu Mishra
2020-12-21 23:09:41 +01:00
parent bd3bc94ff8
commit c91e4d6b04
13 changed files with 127 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/package.json
+1
View File
@@ -39,6 +39,7 @@
"@backstage/backend-common": "^0.4.1",
"@backstage/catalog-model": "^0.6.0",
"@backstage/config": "^0.1.2",
"@backstage/integration": "^0.1.4",
"@google-cloud/storage": "^5.6.0",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
packages/techdocs-common/src/default-branch.ts
+18 -62
View File
@@ -16,13 +16,17 @@
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { Config } from '@backstage/config';
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
import {
getAzureHostToken,
getGitHubRequestOptions,
getGitLabRequestOptions,
getAzureRequestOptions,
} from '@backstage/integration';
import {
getGitHost,
getGithubHostToken,
getGitlabHostToken,
getGitRepoType,
getGitHubIntegrationConfig,
getGitLabIntegrationConfig,
getAzureIntegrationConfig,
} from './git-auth';
interface IGitlabBranch {
@@ -86,62 +90,15 @@ function getAzureApiUrl(url: string): URL {
);
}
function getGithubRequestOptions(config: Config, host: string): RequestInit {
const headers: HeadersInit = {
Accept: 'application/vnd.github.v3.raw',
};
const token = getGithubHostToken(config, host);
if (token) {
headers.Authorization = `token ${token}`;
}
return {
headers,
};
}
function getGitlabRequestOptions(config: Config, host: string): RequestInit {
const headers: HeadersInit = {
'PRIVATE-TOKEN': '',
};
const token = getGitlabHostToken(config, host);
if (token) {
headers['PRIVATE-TOKEN'] = token;
}
return {
headers,
};
}
function getAzureRequestOptions(config: Config, host: string): RequestInit {
const headers: HeadersInit = {};
const token = getAzureHostToken(config, host);
if (token !== '') {
headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString(
'base64',
)}`;
}
const requestOptions: RequestInit = {
headers,
};
return requestOptions;
}
async function getGithubDefaultBranch(
repositoryUrl: string,
config: Config,
): Promise<string> {
const path = getGithubApiUrl(config, repositoryUrl).toString();
const host = getGitHost(repositoryUrl);
const options = getGithubRequestOptions(config, host);
const integrationConfig = getGitHubIntegrationConfig(config, host);
const options = getGitHubRequestOptions(integrationConfig);
try {
const raw = await fetch(path, options);
@@ -169,9 +126,10 @@ async function getGitlabDefaultBranch(
config: Config,
): Promise<string> {
const path = getGitlabApiUrl(repositoryUrl).toString();
const host = getGitHost(repositoryUrl);
const gitlabHost = getGitHost(repositoryUrl);
const options = getGitlabRequestOptions(config, gitlabHost);
const integrationConfig = getGitLabIntegrationConfig(config, host);
const options = getGitLabRequestOptions(integrationConfig);
try {
const raw = await fetch(path, options);
@@ -203,7 +161,9 @@ async function getAzureDefaultBranch(
): Promise<string> {
const path = getAzureApiUrl(repositoryUrl).toString();
const host = getGitHost(repositoryUrl);
const options = getAzureRequestOptions(config, host);
const integrationConfig = getAzureIntegrationConfig(config, host);
const options = getAzureRequestOptions(integrationConfig);
try {
const urlResponse = await fetch(path, options);
@@ -235,12 +195,8 @@ async function getAzureDefaultBranch(
export const getDefaultBranch = async (
repositoryUrl: string,
config: Config,
): Promise<string> => {
// TODO(Rugvip): Config should not be loaded here, pass it in instead
const config = await loadBackendConfig({
logger: getRootLogger(),
argv: process.argv,
});
const type = getGitRepoType(repositoryUrl);
try {
packages/techdocs-common/src/git-auth.ts
+45 -56
View File
@@ -15,7 +15,14 @@
*/
import parseGitUrl from 'git-url-parse';
import { Config } from '@backstage/config';
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
import {
readGitHubIntegrationConfigs,
readGitLabIntegrationConfigs,
readAzureIntegrationConfigs,
GitHubIntegrationConfig,
GitLabIntegrationConfig,
AzureIntegrationConfig,
} from '@backstage/integration';
export function getGitHost(url: string): string {
const { resource } = parseGitUrl(url);
@@ -34,85 +41,67 @@ export function getGitRepoType(url: string): string {
return type;
}
export function getGithubHostToken(
export const getGitHubIntegrationConfig = (
config: Config,
host: string,
): string | undefined {
const providerConfigs =
config.getOptionalConfigArray('integrations.github') ?? [];
const hostConfig = providerConfigs.filter(
providerConfig => providerConfig.getOptionalString('host') === host,
): GitHubIntegrationConfig => {
const allGitHubConfigs = readGitHubIntegrationConfigs(
config.getOptionalConfigArray('integrations.github') ?? [],
);
const token =
hostConfig[0]?.getOptionalString('token') ??
config.getOptionalString('catalog.processors.github.privateToken') ??
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
process.env.GITHUB_TOKEN;
const gitHubIntegrationConfig = allGitHubConfigs.find(v => v.host === host);
if (!gitHubIntegrationConfig) {
throw new Error(`Unable to locate GitHub integration for the host ${host}`);
}
return gitHubIntegrationConfig;
};
return token;
}
export function getGitlabHostToken(
export const getGitLabIntegrationConfig = (
config: Config,
host: string,
): string | undefined {
const providerConfigs =
config.getOptionalConfigArray('integrations.gitlab') ?? [];
const hostConfig = providerConfigs.filter(
providerConfig => providerConfig.getOptionalString('host') === host,
): GitLabIntegrationConfig => {
const allGitLabConfigs = readGitLabIntegrationConfigs(
config.getOptionalConfigArray('integrations.gitlab') ?? [],
);
const token =
hostConfig[0]?.getOptionalString('token') ??
config.getOptionalString('catalog.processors.gitlab.privateToken') ??
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
process.env.GITLAB_TOKEN;
const gitLabIntegrationConfig = allGitLabConfigs.find(v => v.host === host);
if (!gitLabIntegrationConfig) {
throw new Error(`Unable to locate GitLab integration for the host ${host}`);
}
return gitLabIntegrationConfig;
};
return token;
}
export function getAzureHostToken(
export const getAzureIntegrationConfig = (
config: Config,
host: string,
): string | undefined {
const providerConfigs =
config.getOptionalConfigArray('integrations.azure') ?? [];
const hostConfig = providerConfigs.filter(
providerConfig => providerConfig.getOptionalString('host') === host,
): AzureIntegrationConfig => {
const allAzureIntegrationConfig = readAzureIntegrationConfigs(
config.getOptionalConfigArray('integrations.azure') ?? [],
);
const token =
hostConfig[0]?.getOptionalString('token') ??
config.getOptionalString('catalog.processors.azureApi.privateToken') ??
process.env.AZURE_TOKEN;
return token;
}
const azureIntegrationConfig = allAzureIntegrationConfig.find(
v => v.host === host,
);
if (!azureIntegrationConfig) {
throw new Error(`Unable to locate Azure integration for the host ${host}`);
}
return azureIntegrationConfig;
};
export const getTokenForGitRepo = async (
repositoryUrl: string,
config: Config,
): Promise<string | undefined> => {
// TODO(Rugvip): Config should not be loaded here, pass it in instead
const config = await loadBackendConfig({
logger: getRootLogger(),
argv: process.argv,
});
const host = getGitHost(repositoryUrl);
const type = getGitRepoType(repositoryUrl);
try {
switch (type) {
case 'github':
return getGithubHostToken(config, host);
return getGitHubIntegrationConfig(config, host).token;
case 'gitlab':
return getGitlabHostToken(config, host);
return getGitLabIntegrationConfig(config, host).token;
case 'azure/api':
return getAzureHostToken(config, host);
return getAzureIntegrationConfig(config, host).token;
default:
throw new Error('Failed to get repository type');
throw new Error('Failed to get reository type');
}
} catch (error) {
throw error;
packages/techdocs-common/src/helpers.ts
+14 -5
View File
@@ -19,10 +19,11 @@ import path from 'path';
import parseGitUrl from 'git-url-parse';
import NodeGit, { Clone, Repository } from 'nodegit';
import fs from 'fs-extra';
import { InputError, UrlReader } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { getDefaultBranch } from './default-branch';
import { getGitRepoType, getTokenForGitRepo } from './git-auth';
import { Entity } from '@backstage/catalog-model';
import { InputError, UrlReader } from '@backstage/backend-common';
import { RemoteProtocol } from './stages/prepare/types';
import { Logger } from 'winston';
@@ -94,6 +95,7 @@ export const getLocationForEntity = (
export const getGitRepositoryTempFolder = async (
repositoryUrl: string,
config: Config,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repositoryUrl);
// removes .git from git location path
@@ -102,6 +104,7 @@ export const getGitRepositoryTempFolder = async (
if (!parsedGitLocation.ref) {
parsedGitLocation.ref = await getDefaultBranch(
parsedGitLocation.toString('https'),
config,
);
}
@@ -118,11 +121,12 @@ export const getGitRepositoryTempFolder = async (
export const checkoutGitRepository = async (
repoUrl: string,
config: Config,
logger: Logger,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repoUrl);
const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl);
const token = await getTokenForGitRepo(repoUrl);
const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl, config);
const token = await getTokenForGitRepo(repoUrl, config);
if (fs.existsSync(repositoryTmpPath)) {
try {
@@ -170,9 +174,14 @@ export const checkoutGitRepository = async (
export const getLastCommitTimestamp = async (
repositoryUrl: string,
config: Config,
logger: Logger,
): Promise<number> => {
const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger);
const repositoryLocation = await checkoutGitRepository(
repositoryUrl,
config,
logger,
);
const repository = await Repository.open(repositoryLocation);
const commit = await repository.getReferenceCommit('HEAD');
packages/techdocs-common/src/stages/prepare/commonGit.test.ts
+6 -3
View File
@@ -15,6 +15,7 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { CommonGitPreparer } from './commonGit';
import { checkoutGitRepository } from '../../helpers';
@@ -43,11 +44,13 @@ const createMockEntity = (annotations = {}) => {
};
};
const mockConfig = new ConfigReader({});
const logger = getVoidLogger();
describe('commonGit preparer', () => {
it('should prepare temp docs path from github repo', async () => {
const preparer = new CommonGitPreparer(logger);
const preparer = new CommonGitPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/techdocs-ref':
@@ -62,7 +65,7 @@ describe('commonGit preparer', () => {
});
it('should prepare temp docs path from gitlab repo', async () => {
const preparer = new CommonGitPreparer(logger);
const preparer = new CommonGitPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/techdocs-ref':
@@ -77,7 +80,7 @@ describe('commonGit preparer', () => {
});
it('should prepare temp docs path from azure repo', async () => {
const preparer = new CommonGitPreparer(logger);
const preparer = new CommonGitPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/techdocs-ref':
packages/techdocs-common/src/stages/prepare/commonGit.ts
+11 -4
View File
@@ -14,17 +14,20 @@
* limitations under the License.
*/
import path from 'path';
import { Entity } from '@backstage/catalog-model';
import { PreparerBase } from './types';
import parseGitUrl from 'git-url-parse';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { PreparerBase } from './types';
import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers';
import { Logger } from 'winston';
export class CommonGitPreparer implements PreparerBase {
private readonly config: Config;
private readonly logger: Logger;
constructor(logger: Logger) {
constructor(config: Config, logger: Logger) {
this.config = config;
this.logger = logger;
}
@@ -35,7 +38,11 @@ export class CommonGitPreparer implements PreparerBase {
);
try {
const repoPath = await checkoutGitRepository(target, this.logger);
const repoPath = await checkoutGitRepository(
target,
this.config,
this.logger,
);
const parsedGitLocation = parseGitUrl(target);
return path.join(repoPath, parsedGitLocation.filepath);
packages/techdocs-common/src/stages/prepare/dir.test.ts
+7 -4
View File
@@ -13,8 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DirectoryPreparer } from './dir';
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { DirectoryPreparer } from './dir';
import { checkoutGitRepository } from '../../helpers';
function normalizePath(path: string) {
@@ -44,9 +45,11 @@ const createMockEntity = (annotations: {}) => {
};
};
const mockConfig = new ConfigReader({});
describe('directory preparer', () => {
it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
const directoryPreparer = new DirectoryPreparer(logger);
const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/managed-by-location':
@@ -60,7 +63,7 @@ describe('directory preparer', () => {
});
it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
const directoryPreparer = new DirectoryPreparer(logger);
const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/managed-by-location':
@@ -74,7 +77,7 @@ describe('directory preparer', () => {
});
it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
const directoryPreparer = new DirectoryPreparer(logger);
const directoryPreparer = new DirectoryPreparer(mockConfig, logger);
const mockEntity = createMockEntity({
'backstage.io/managed-by-location':
packages/techdocs-common/src/stages/prepare/dir.ts
+9 -2
View File
@@ -15,6 +15,7 @@
*/
import { PreparerBase } from './types';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import path from 'path';
import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers';
import { InputError } from '@backstage/backend-common';
@@ -22,9 +23,11 @@ import parseGitUrl from 'git-url-parse';
import { Logger } from 'winston';
export class DirectoryPreparer implements PreparerBase {
private readonly config: Config;
private readonly logger: Logger;
constructor(logger: Logger) {
constructor(config: Config, logger: Logger) {
this.config = config;
this.logger = logger;
}
@@ -43,7 +46,11 @@ export class DirectoryPreparer implements PreparerBase {
case 'url':
case 'azure/api': {
const parsedGitLocation = parseGitUrl(target);
const repoLocation = await checkoutGitRepository(target, this.logger);
const repoLocation = await checkoutGitRepository(
target,
this.config,
this.logger,
);
return path.dirname(
path.join(repoLocation, parsedGitLocation.filepath),
packages/techdocs-common/src/stages/prepare/preparers.ts
+2 -5
View File
@@ -30,18 +30,15 @@ export class Preparers implements PreparerBuilder {
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
static async fromConfig(
// @ts-ignore
// Config not used now, but will be used in urlPreparer when it starts using
// @backstage/integration to get the tokens for providers.
config: Config,
{ logger, reader }: factoryOptions,
): Promise<PreparerBuilder> {
const preparers = new Preparers();
const directoryPreparer = new DirectoryPreparer(logger);
const directoryPreparer = new DirectoryPreparer(config, logger);
preparers.register('dir', directoryPreparer);
const commonGitPreparer = new CommonGitPreparer(logger);
const commonGitPreparer = new CommonGitPreparer(config, logger);
preparers.register('github', commonGitPreparer);
preparers.register('gitlab', commonGitPreparer);
preparers.register('azure/api', commonGitPreparer);
packages/techdocs-common/src/stages/prepare/url.ts
+2 -2
View File
@@ -13,11 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Logger } from 'winston';
import { Entity } from '@backstage/catalog-model';
import { UrlReader } from '@backstage/backend-common';
import { PreparerBase } from './types';
import { getDocFilesFromRepository } from '../../helpers';
import { Logger } from 'winston';
import { UrlReader } from '@backstage/backend-common';
export class UrlPreparer implements PreparerBase {
private readonly logger: Logger;