From 359f9d2d8cbe82e9c03230cc3915c35e2e248703 Mon Sep 17 00:00:00 2001 From: keshan Date: Mon, 21 Dec 2020 12:44:02 +0530 Subject: [PATCH 1/6] Introduced config schema for techdocs plugins --- .changeset/modern-timers-obey.md | 6 ++ plugins/techdocs-backend/config.d.ts | 46 ++++++++++++++ plugins/techdocs-backend/package.json | 6 +- plugins/techdocs/config.d.ts | 91 +++++++++++++++++++++++++++ plugins/techdocs/package.json | 83 +----------------------- 5 files changed, 150 insertions(+), 82 deletions(-) create mode 100644 .changeset/modern-timers-obey.md create mode 100644 plugins/techdocs-backend/config.d.ts create mode 100644 plugins/techdocs/config.d.ts diff --git a/.changeset/modern-timers-obey.md b/.changeset/modern-timers-obey.md new file mode 100644 index 0000000000..7268d4961d --- /dev/null +++ b/.changeset/modern-timers-obey.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Added configuration schema for the commonly used properties of techdocs and techdocs-backend plugins diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts new file mode 100644 index 0000000000..f3237c669a --- /dev/null +++ b/plugins/techdocs-backend/config.d.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the techdocs-backend plugin */ + techdocs: { + /** + * attr: 'storageUrl' - accepts a string value + * e.g. storageUrl: http://localhost:7000/api/techdocs/static/docs + */ + storageUrl: string; + /** + * documentation building process depends on the builder attr + * attr: 'builder' - accepts a string value + * e.g. builder: 'local' + * alternative: 'external' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + builder: string; + /** + * techdocs publisher information + */ + publisher: { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'local' + * aleternatives: 'googleGcs' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: string; + }; + }; +} diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 5552d87f3f..372c8aaa6e 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -48,6 +48,8 @@ "supertest": "^4.0.2" }, "files": [ - "dist" - ] + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts new file mode 100644 index 0000000000..a60fbed1b4 --- /dev/null +++ b/plugins/techdocs/config.d.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the techdocs plugin */ + techdocs: { + /** + * attr: 'requestUrl' - accepts a string value + * e.g. requestUrl: http://localhost:7000/api/techdocs + * @visibility frontend + */ + requestUrl: string; + /** + * attr: 'storageUrl' - accepts a string value + * e.g. storageUrl: http://localhost:7000/api/techdocs/static/docs + */ + storageUrl: string; + /** + * documentation building process depends on the builder attr + * attr: 'builder' - accepts a string value + * e.g. builder: 'local' + * alternative: 'external' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + * @visibility frontend + */ + builder: string; + + /** + * techdocs publisher information + */ + generators?: { + /** + * attr: 'techdocs' - accepts a string value + * e.g. type: 'docker' + * aleternatives: 'local' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + techdocs: string; + }; + + /** + * techdocs publisher information + */ + publisher?: { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'local' + * aleternatives: 'googleGcs' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: string; + + /** + * googleGcs required when 'type' is set to googleGcs, skip otherwise + */ + googleGcs?: { + /** + * API key used to write to a storage bucket. + * attr: 'credentials' - accepts a string value + * @visibility secret + */ + credentials: string; + /** + * GCP Project ID where the Cloud Storage Bucket is hosted. + * attr: 'projectId' - accepts a string value + * @visibility secret + */ + projectId: string; + /** + * Cloud Storage Bucket Name + * attr: 'bucketName' - accepts a string value + * @visibility secret + */ + bucketName: string; + }; + }; + }; +} diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 361825e71a..2baebdcad0 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -63,85 +63,8 @@ "msw": "^0.21.2" }, "files": [ - "dist" + "dist", + "config.d.ts" ], - "configSchema": { - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/techdocs", - "type": "object", - "properties": { - "techdocs": { - "type": "object", - "properties": { - "requestUrl": { - "type": "string", - "visibility": "frontend" - }, - "storageUrl": { - "type": "string", - "visibility": "backend" - }, - "generators": { - "type": "object", - "properties": { - "techdocs": { - "type": "string", - "visibility": "backend" - } - } - }, - "builder": { - "type": "string", - "visibility": "frontend" - }, - "publisher": { - "oneOf": [ - { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "local", - "visibility": "backend" - } - } - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "googleGcs", - "visibility": "backend" - }, - "googleGcs": { - "type": "object", - "properties": { - "credentials": { - "type": "string", - "visibility": "secret" - }, - "projectId": { - "type": "string", - "visibility": "secret" - }, - "bucketName": { - "type": "string", - "visibility": "secret" - } - } - } - } - } - ] - } - }, - "required": [ - "requestUrl", - "storageUrl", - "builder" - ] - } - } - } + "configSchema": "config.d.ts" } From e53d4a2ca823efd3eccbd91e0c962e5ee7ec94cc Mon Sep 17 00:00:00 2001 From: keshan Date: Mon, 21 Dec 2020 16:25:49 +0530 Subject: [PATCH 2/6] updated based on the feedbacks provided --- plugins/techdocs-backend/config.d.ts | 9 ++-- plugins/techdocs/config.d.ts | 76 ++++++++++++++++------------ 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index f3237c669a..bb76b21eae 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +/** + * techdocs schema below is an abstract of what's used within techdocs-backend and for its visisbility + * to view the complete techdoc schema please refer: plugins/techdocs/config.d.ts + * */ export interface Config { /** Configuration options for the techdocs-backend plugin */ techdocs: { @@ -29,7 +32,7 @@ export interface Config { * alternative: 'external' etc. * @see http://backstage.io/docs/features/techdocs/configuration */ - builder: string; + builder: 'local' | 'external'; /** * techdocs publisher information */ @@ -40,7 +43,7 @@ export interface Config { * aleternatives: 'googleGcs' etc. * @see http://backstage.io/docs/features/techdocs/configuration */ - type: string; + type: 'local' | 'googleGcs' | 'awsS3'; }; }; } diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index a60fbed1b4..10adcb9fbd 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -36,7 +36,7 @@ export interface Config { * @see http://backstage.io/docs/features/techdocs/configuration * @visibility frontend */ - builder: string; + builder: 'local' | 'external'; /** * techdocs publisher information @@ -54,38 +54,48 @@ export interface Config { /** * techdocs publisher information */ - publisher?: { - /** - * attr: 'type' - accepts a string value - * e.g. type: 'local' - * aleternatives: 'googleGcs' etc. - * @see http://backstage.io/docs/features/techdocs/configuration - */ - type: string; + publisher?: + | { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'local' + * aleternatives: 'googleGcs' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: 'local' | 'awsS3'; + } + | { + /** + * attr: 'type' - accepts a string value + * e.g. type: 'googleGcs' + * aleternatives: 'googleGcs' etc. + * @see http://backstage.io/docs/features/techdocs/configuration + */ + type: 'googleGcs'; - /** - * googleGcs required when 'type' is set to googleGcs, skip otherwise - */ - googleGcs?: { - /** - * API key used to write to a storage bucket. - * attr: 'credentials' - accepts a string value - * @visibility secret - */ - credentials: string; - /** - * GCP Project ID where the Cloud Storage Bucket is hosted. - * attr: 'projectId' - accepts a string value - * @visibility secret - */ - projectId: string; - /** - * Cloud Storage Bucket Name - * attr: 'bucketName' - accepts a string value - * @visibility secret - */ - bucketName: string; - }; - }; + /** + * googleGcs required when 'type' is set to googleGcs + */ + googleGcs?: { + /** + * API key used to write to a storage bucket. + * attr: 'credentials' - accepts a string value + * @visibility secret + */ + credentials: string; + /** + * GCP Project ID where the Cloud Storage Bucket is hosted. + * attr: 'projectId' - accepts a string value + * @visibility secret + */ + projectId: string; + /** + * Cloud Storage Bucket Name + * attr: 'bucketName' - accepts a string value + * @visibility secret + */ + bucketName: string; + }; + }; }; } From 851dde45f7c95520822423df8fa425e67f07f2cb Mon Sep 17 00:00:00 2001 From: keshan Date: Mon, 21 Dec 2020 16:31:01 +0530 Subject: [PATCH 3/6] updated the missed attr change --- plugins/techdocs/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index 10adcb9fbd..543579e85f 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -48,7 +48,7 @@ export interface Config { * aleternatives: 'local' etc. * @see http://backstage.io/docs/features/techdocs/configuration */ - techdocs: string; + techdocs: 'local' | 'docker'; }; /** From 23cee0d8313d03576c14f5af59faeed8b292ef27 Mon Sep 17 00:00:00 2001 From: keshan Date: Mon, 21 Dec 2020 16:33:39 +0530 Subject: [PATCH 4/6] typo corrected --- plugins/techdocs-backend/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index bb76b21eae..b51d86138a 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * techdocs schema below is an abstract of what's used within techdocs-backend and for its visisbility + * techdocs schema below is an abstract of what's used within techdocs-backend and for its visibility * to view the complete techdoc schema please refer: plugins/techdocs/config.d.ts * */ export interface Config { From c91e4d6b0468eee3bf273487b4d3505c8ff7c4f9 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Mon, 21 Dec 2020 23:09:41 +0100 Subject: [PATCH 5/6] techdocs: Use @backstage/integration for scm tokens and request options --- packages/techdocs-common/package.json | 1 + .../techdocs-common/src/default-branch.ts | 80 ++++---------- packages/techdocs-common/src/git-auth.ts | 101 ++++++++---------- packages/techdocs-common/src/helpers.ts | 19 +++- .../src/stages/prepare/commonGit.test.ts | 9 +- .../src/stages/prepare/commonGit.ts | 15 ++- .../src/stages/prepare/dir.test.ts | 11 +- .../techdocs-common/src/stages/prepare/dir.ts | 11 +- .../src/stages/prepare/preparers.ts | 7 +- .../techdocs-common/src/stages/prepare/url.ts | 4 +- .../src/DocsBuilder/builder.ts | 11 +- .../techdocs-backend/src/service/router.ts | 1 + .../src/service/standaloneServer.ts | 2 +- 13 files changed, 127 insertions(+), 145 deletions(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 68debc2448..20d0f68a98 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -39,6 +39,7 @@ "@backstage/backend-common": "^0.4.1", "@backstage/catalog-model": "^0.6.0", "@backstage/config": "^0.1.2", + "@backstage/integration": "^0.1.4", "@google-cloud/storage": "^5.6.0", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", diff --git a/packages/techdocs-common/src/default-branch.ts b/packages/techdocs-common/src/default-branch.ts index 1090fcb3d3..7dd460a9fe 100644 --- a/packages/techdocs-common/src/default-branch.ts +++ b/packages/techdocs-common/src/default-branch.ts @@ -16,13 +16,17 @@ import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; import { Config } from '@backstage/config'; -import { getRootLogger, loadBackendConfig } from '@backstage/backend-common'; import { - getAzureHostToken, + getGitHubRequestOptions, + getGitLabRequestOptions, + getAzureRequestOptions, +} from '@backstage/integration'; +import { getGitHost, - getGithubHostToken, - getGitlabHostToken, getGitRepoType, + getGitHubIntegrationConfig, + getGitLabIntegrationConfig, + getAzureIntegrationConfig, } from './git-auth'; interface IGitlabBranch { @@ -86,62 +90,15 @@ function getAzureApiUrl(url: string): URL { ); } -function getGithubRequestOptions(config: Config, host: string): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - const token = getGithubHostToken(config, host); - - if (token) { - headers.Authorization = `token ${token}`; - } - - return { - headers, - }; -} - -function getGitlabRequestOptions(config: Config, host: string): RequestInit { - const headers: HeadersInit = { - 'PRIVATE-TOKEN': '', - }; - - const token = getGitlabHostToken(config, host); - if (token) { - headers['PRIVATE-TOKEN'] = token; - } - - return { - headers, - }; -} - -function getAzureRequestOptions(config: Config, host: string): RequestInit { - const headers: HeadersInit = {}; - - const token = getAzureHostToken(config, host); - - if (token !== '') { - headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString( - 'base64', - )}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; -} - async function getGithubDefaultBranch( repositoryUrl: string, config: Config, ): Promise { const path = getGithubApiUrl(config, repositoryUrl).toString(); const host = getGitHost(repositoryUrl); - const options = getGithubRequestOptions(config, host); + + const integrationConfig = getGitHubIntegrationConfig(config, host); + const options = getGitHubRequestOptions(integrationConfig); try { const raw = await fetch(path, options); @@ -169,9 +126,10 @@ async function getGitlabDefaultBranch( config: Config, ): Promise { const path = getGitlabApiUrl(repositoryUrl).toString(); + const host = getGitHost(repositoryUrl); - const gitlabHost = getGitHost(repositoryUrl); - const options = getGitlabRequestOptions(config, gitlabHost); + const integrationConfig = getGitLabIntegrationConfig(config, host); + const options = getGitLabRequestOptions(integrationConfig); try { const raw = await fetch(path, options); @@ -203,7 +161,9 @@ async function getAzureDefaultBranch( ): Promise { const path = getAzureApiUrl(repositoryUrl).toString(); const host = getGitHost(repositoryUrl); - const options = getAzureRequestOptions(config, host); + + const integrationConfig = getAzureIntegrationConfig(config, host); + const options = getAzureRequestOptions(integrationConfig); try { const urlResponse = await fetch(path, options); @@ -235,12 +195,8 @@ async function getAzureDefaultBranch( export const getDefaultBranch = async ( repositoryUrl: string, + config: Config, ): Promise => { - // TODO(Rugvip): Config should not be loaded here, pass it in instead - const config = await loadBackendConfig({ - logger: getRootLogger(), - argv: process.argv, - }); const type = getGitRepoType(repositoryUrl); try { diff --git a/packages/techdocs-common/src/git-auth.ts b/packages/techdocs-common/src/git-auth.ts index b580a6f05b..41a33340af 100644 --- a/packages/techdocs-common/src/git-auth.ts +++ b/packages/techdocs-common/src/git-auth.ts @@ -15,7 +15,14 @@ */ import parseGitUrl from 'git-url-parse'; import { Config } from '@backstage/config'; -import { getRootLogger, loadBackendConfig } from '@backstage/backend-common'; +import { + readGitHubIntegrationConfigs, + readGitLabIntegrationConfigs, + readAzureIntegrationConfigs, + GitHubIntegrationConfig, + GitLabIntegrationConfig, + AzureIntegrationConfig, +} from '@backstage/integration'; export function getGitHost(url: string): string { const { resource } = parseGitUrl(url); @@ -34,85 +41,67 @@ export function getGitRepoType(url: string): string { return type; } -export function getGithubHostToken( +export const getGitHubIntegrationConfig = ( config: Config, host: string, -): string | undefined { - const providerConfigs = - config.getOptionalConfigArray('integrations.github') ?? []; - - const hostConfig = providerConfigs.filter( - providerConfig => providerConfig.getOptionalString('host') === host, +): GitHubIntegrationConfig => { + const allGitHubConfigs = readGitHubIntegrationConfigs( + config.getOptionalConfigArray('integrations.github') ?? [], ); - const token = - hostConfig[0]?.getOptionalString('token') ?? - config.getOptionalString('catalog.processors.github.privateToken') ?? - config.getOptionalString('catalog.processors.githubApi.privateToken') ?? - process.env.GITHUB_TOKEN; + const gitHubIntegrationConfig = allGitHubConfigs.find(v => v.host === host); + if (!gitHubIntegrationConfig) { + throw new Error(`Unable to locate GitHub integration for the host ${host}`); + } + return gitHubIntegrationConfig; +}; - return token; -} - -export function getGitlabHostToken( +export const getGitLabIntegrationConfig = ( config: Config, host: string, -): string | undefined { - const providerConfigs = - config.getOptionalConfigArray('integrations.gitlab') ?? []; - - const hostConfig = providerConfigs.filter( - providerConfig => providerConfig.getOptionalString('host') === host, +): GitLabIntegrationConfig => { + const allGitLabConfigs = readGitLabIntegrationConfigs( + config.getOptionalConfigArray('integrations.gitlab') ?? [], ); - const token = - hostConfig[0]?.getOptionalString('token') ?? - config.getOptionalString('catalog.processors.gitlab.privateToken') ?? - config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? - process.env.GITLAB_TOKEN; + const gitLabIntegrationConfig = allGitLabConfigs.find(v => v.host === host); + if (!gitLabIntegrationConfig) { + throw new Error(`Unable to locate GitLab integration for the host ${host}`); + } + return gitLabIntegrationConfig; +}; - return token; -} - -export function getAzureHostToken( +export const getAzureIntegrationConfig = ( config: Config, host: string, -): string | undefined { - const providerConfigs = - config.getOptionalConfigArray('integrations.azure') ?? []; - - const hostConfig = providerConfigs.filter( - providerConfig => providerConfig.getOptionalString('host') === host, +): AzureIntegrationConfig => { + const allAzureIntegrationConfig = readAzureIntegrationConfigs( + config.getOptionalConfigArray('integrations.azure') ?? [], ); - const token = - hostConfig[0]?.getOptionalString('token') ?? - config.getOptionalString('catalog.processors.azureApi.privateToken') ?? - process.env.AZURE_TOKEN; - - return token; -} + const azureIntegrationConfig = allAzureIntegrationConfig.find( + v => v.host === host, + ); + if (!azureIntegrationConfig) { + throw new Error(`Unable to locate Azure integration for the host ${host}`); + } + return azureIntegrationConfig; +}; export const getTokenForGitRepo = async ( repositoryUrl: string, + config: Config, ): Promise => { - // TODO(Rugvip): Config should not be loaded here, pass it in instead - const config = await loadBackendConfig({ - logger: getRootLogger(), - argv: process.argv, - }); - const host = getGitHost(repositoryUrl); const type = getGitRepoType(repositoryUrl); try { switch (type) { case 'github': - return getGithubHostToken(config, host); + return getGitHubIntegrationConfig(config, host).token; case 'gitlab': - return getGitlabHostToken(config, host); + return getGitLabIntegrationConfig(config, host).token; case 'azure/api': - return getAzureHostToken(config, host); - + return getAzureIntegrationConfig(config, host).token; default: - throw new Error('Failed to get repository type'); + throw new Error('Failed to get reository type'); } } catch (error) { throw error; diff --git a/packages/techdocs-common/src/helpers.ts b/packages/techdocs-common/src/helpers.ts index e08d58d58e..ab1e3e615f 100644 --- a/packages/techdocs-common/src/helpers.ts +++ b/packages/techdocs-common/src/helpers.ts @@ -19,10 +19,11 @@ import path from 'path'; import parseGitUrl from 'git-url-parse'; import NodeGit, { Clone, Repository } from 'nodegit'; import fs from 'fs-extra'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import { getDefaultBranch } from './default-branch'; import { getGitRepoType, getTokenForGitRepo } from './git-auth'; -import { Entity } from '@backstage/catalog-model'; -import { InputError, UrlReader } from '@backstage/backend-common'; import { RemoteProtocol } from './stages/prepare/types'; import { Logger } from 'winston'; @@ -94,6 +95,7 @@ export const getLocationForEntity = ( export const getGitRepositoryTempFolder = async ( repositoryUrl: string, + config: Config, ): Promise => { const parsedGitLocation = parseGitUrl(repositoryUrl); // removes .git from git location path @@ -102,6 +104,7 @@ export const getGitRepositoryTempFolder = async ( if (!parsedGitLocation.ref) { parsedGitLocation.ref = await getDefaultBranch( parsedGitLocation.toString('https'), + config, ); } @@ -118,11 +121,12 @@ export const getGitRepositoryTempFolder = async ( export const checkoutGitRepository = async ( repoUrl: string, + config: Config, logger: Logger, ): Promise => { const parsedGitLocation = parseGitUrl(repoUrl); - const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl); - const token = await getTokenForGitRepo(repoUrl); + const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl, config); + const token = await getTokenForGitRepo(repoUrl, config); if (fs.existsSync(repositoryTmpPath)) { try { @@ -170,9 +174,14 @@ export const checkoutGitRepository = async ( export const getLastCommitTimestamp = async ( repositoryUrl: string, + config: Config, logger: Logger, ): Promise => { - const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger); + const repositoryLocation = await checkoutGitRepository( + repositoryUrl, + config, + logger, + ); const repository = await Repository.open(repositoryLocation); const commit = await repository.getReferenceCommit('HEAD'); diff --git a/packages/techdocs-common/src/stages/prepare/commonGit.test.ts b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts index a4240514b5..704cd23daa 100644 --- a/packages/techdocs-common/src/stages/prepare/commonGit.test.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { CommonGitPreparer } from './commonGit'; import { checkoutGitRepository } from '../../helpers'; @@ -43,11 +44,13 @@ const createMockEntity = (annotations = {}) => { }; }; +const mockConfig = new ConfigReader({}); + const logger = getVoidLogger(); describe('commonGit preparer', () => { it('should prepare temp docs path from github repo', async () => { - const preparer = new CommonGitPreparer(logger); + const preparer = new CommonGitPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/techdocs-ref': @@ -62,7 +65,7 @@ describe('commonGit preparer', () => { }); it('should prepare temp docs path from gitlab repo', async () => { - const preparer = new CommonGitPreparer(logger); + const preparer = new CommonGitPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/techdocs-ref': @@ -77,7 +80,7 @@ describe('commonGit preparer', () => { }); it('should prepare temp docs path from azure repo', async () => { - const preparer = new CommonGitPreparer(logger); + const preparer = new CommonGitPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/techdocs-ref': diff --git a/packages/techdocs-common/src/stages/prepare/commonGit.ts b/packages/techdocs-common/src/stages/prepare/commonGit.ts index d79373fba3..7eac07d76f 100644 --- a/packages/techdocs-common/src/stages/prepare/commonGit.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.ts @@ -14,17 +14,20 @@ * limitations under the License. */ import path from 'path'; -import { Entity } from '@backstage/catalog-model'; -import { PreparerBase } from './types'; import parseGitUrl from 'git-url-parse'; +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { PreparerBase } from './types'; import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; import { Logger } from 'winston'; export class CommonGitPreparer implements PreparerBase { + private readonly config: Config; private readonly logger: Logger; - constructor(logger: Logger) { + constructor(config: Config, logger: Logger) { + this.config = config; this.logger = logger; } @@ -35,7 +38,11 @@ export class CommonGitPreparer implements PreparerBase { ); try { - const repoPath = await checkoutGitRepository(target, this.logger); + const repoPath = await checkoutGitRepository( + target, + this.config, + this.logger, + ); const parsedGitLocation = parseGitUrl(target); return path.join(repoPath, parsedGitLocation.filepath); diff --git a/packages/techdocs-common/src/stages/prepare/dir.test.ts b/packages/techdocs-common/src/stages/prepare/dir.test.ts index 5b51f1f46d..d082b1367b 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.test.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.test.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DirectoryPreparer } from './dir'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { DirectoryPreparer } from './dir'; import { checkoutGitRepository } from '../../helpers'; function normalizePath(path: string) { @@ -44,9 +45,11 @@ const createMockEntity = (annotations: {}) => { }; }; +const mockConfig = new ConfigReader({}); + describe('directory preparer', () => { it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => { - const directoryPreparer = new DirectoryPreparer(logger); + const directoryPreparer = new DirectoryPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/managed-by-location': @@ -60,7 +63,7 @@ describe('directory preparer', () => { }); it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => { - const directoryPreparer = new DirectoryPreparer(logger); + const directoryPreparer = new DirectoryPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/managed-by-location': @@ -74,7 +77,7 @@ describe('directory preparer', () => { }); it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => { - const directoryPreparer = new DirectoryPreparer(logger); + const directoryPreparer = new DirectoryPreparer(mockConfig, logger); const mockEntity = createMockEntity({ 'backstage.io/managed-by-location': diff --git a/packages/techdocs-common/src/stages/prepare/dir.ts b/packages/techdocs-common/src/stages/prepare/dir.ts index 3485ecb5dd..537f670edc 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.ts @@ -15,6 +15,7 @@ */ import { PreparerBase } from './types'; import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import path from 'path'; import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; import { InputError } from '@backstage/backend-common'; @@ -22,9 +23,11 @@ import parseGitUrl from 'git-url-parse'; import { Logger } from 'winston'; export class DirectoryPreparer implements PreparerBase { + private readonly config: Config; private readonly logger: Logger; - constructor(logger: Logger) { + constructor(config: Config, logger: Logger) { + this.config = config; this.logger = logger; } @@ -43,7 +46,11 @@ export class DirectoryPreparer implements PreparerBase { case 'url': case 'azure/api': { const parsedGitLocation = parseGitUrl(target); - const repoLocation = await checkoutGitRepository(target, this.logger); + const repoLocation = await checkoutGitRepository( + target, + this.config, + this.logger, + ); return path.dirname( path.join(repoLocation, parsedGitLocation.filepath), diff --git a/packages/techdocs-common/src/stages/prepare/preparers.ts b/packages/techdocs-common/src/stages/prepare/preparers.ts index 4a2d60eb31..4625d7b0e4 100644 --- a/packages/techdocs-common/src/stages/prepare/preparers.ts +++ b/packages/techdocs-common/src/stages/prepare/preparers.ts @@ -30,18 +30,15 @@ export class Preparers implements PreparerBuilder { private preparerMap = new Map(); static async fromConfig( - // @ts-ignore - // Config not used now, but will be used in urlPreparer when it starts using - // @backstage/integration to get the tokens for providers. config: Config, { logger, reader }: factoryOptions, ): Promise { const preparers = new Preparers(); - const directoryPreparer = new DirectoryPreparer(logger); + const directoryPreparer = new DirectoryPreparer(config, logger); preparers.register('dir', directoryPreparer); - const commonGitPreparer = new CommonGitPreparer(logger); + const commonGitPreparer = new CommonGitPreparer(config, logger); preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); preparers.register('azure/api', commonGitPreparer); diff --git a/packages/techdocs-common/src/stages/prepare/url.ts b/packages/techdocs-common/src/stages/prepare/url.ts index 3407684837..b938c6b5bc 100644 --- a/packages/techdocs-common/src/stages/prepare/url.ts +++ b/packages/techdocs-common/src/stages/prepare/url.ts @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; +import { UrlReader } from '@backstage/backend-common'; import { PreparerBase } from './types'; import { getDocFilesFromRepository } from '../../helpers'; -import { Logger } from 'winston'; -import { UrlReader } from '@backstage/backend-common'; export class UrlPreparer implements PreparerBase { private readonly logger: Logger; diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 67d0dfb89e..863f587a96 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -16,6 +16,7 @@ import Docker from 'dockerode'; import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import { PreparerBuilder, PublisherBase, @@ -40,6 +41,7 @@ type DocsBuilderArguments = { entity: Entity; logger: Logger; dockerClient: Docker; + config: Config; }; export class DocsBuilder { @@ -49,6 +51,7 @@ export class DocsBuilder { private entity: Entity; private logger: Logger; private dockerClient: Docker; + private config: Config; constructor({ preparers, @@ -57,6 +60,7 @@ export class DocsBuilder { entity, logger, dockerClient, + config, }: DocsBuilderArguments) { this.preparer = preparers.get(entity); this.generator = generators.get(entity); @@ -64,6 +68,7 @@ export class DocsBuilder { this.entity = entity; this.logger = logger; this.dockerClient = dockerClient; + this.config = config; } public async build() { @@ -109,7 +114,11 @@ export class DocsBuilder { // Unless docs are stored locally const nonAgeCheckTypes = ['dir', 'file', 'url']; if (!nonAgeCheckTypes.includes(type)) { - const lastCommit = await getLastCommitTimestamp(target, this.logger); + const lastCommit = await getLastCommitTimestamp( + target, + this.config, + this.logger, + ); const storageTimeStamp = buildMetadataStorage.getTimestamp(); // Check if documentation source is newer than what we have diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index a64f581a1e..37d5dd59ed 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -131,6 +131,7 @@ export async function createRouter({ dockerClient, logger, entity, + config, }); if (publisherType === 'local') { if (!(await docsBuilder.docsUpToDate())) { diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 65a286feef..721c10a1ec 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -52,7 +52,7 @@ export async function startStandaloneServer( logger.debug('Creating application...'); const preparers = new Preparers(); - const directoryPreparer = new DirectoryPreparer(logger); + const directoryPreparer = new DirectoryPreparer(config, logger); preparers.register('dir', directoryPreparer); const generators = new Generators(); From 8804e8981131f8755894baffa817f2168398e638 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Mon, 21 Dec 2020 23:15:21 +0100 Subject: [PATCH 6/6] techdocs: Add changeset for @backstage/integration changes --- .changeset/little-plums-beg.md | 20 ++++++++++++++++++++ packages/techdocs-common/src/git-auth.ts | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .changeset/little-plums-beg.md diff --git a/.changeset/little-plums-beg.md b/.changeset/little-plums-beg.md new file mode 100644 index 0000000000..351545fd6b --- /dev/null +++ b/.changeset/little-plums-beg.md @@ -0,0 +1,20 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Using @backstage/integration package for GitHub/GitLab/Azure tokens and request options. + +Most probably you do not have to make any changes in the app because of this change. +However, if you are using the `DirectoryPreparer` or `CommonGitPreparer` exported by +`@backstage/techdocs-common` package, you now need to add pass in a `config` (from `@backstage/config`) +instance as argument. + +``` + + const directoryPreparer = new DirectoryPreparer(logger); + const commonGitPreparer = new CommonGitPreparer(logger); + + const directoryPreparer = new DirectoryPreparer(config, logger); + const commonGitPreparer = new CommonGitPreparer(config, logger); +``` diff --git a/packages/techdocs-common/src/git-auth.ts b/packages/techdocs-common/src/git-auth.ts index 41a33340af..b58afae490 100644 --- a/packages/techdocs-common/src/git-auth.ts +++ b/packages/techdocs-common/src/git-auth.ts @@ -101,7 +101,7 @@ export const getTokenForGitRepo = async ( case 'azure/api': return getAzureIntegrationConfig(config, host).token; default: - throw new Error('Failed to get reository type'); + throw new Error('Failed to get repository type'); } } catch (error) { throw error;