From b46863b88b933b79ec74af2550a0b4d6e473a9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Fri, 9 Oct 2020 09:37:20 +0200 Subject: [PATCH] Techdocs: add Azure DevOps prepare support (#2748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update to latest git-url-parse * Add Azure DevOps preparer to techdocs * Upgraded git-url-parse in backend-common * Switch to AZURE_TOKEN Co-authored-by: Mattias Frinnström --- packages/backend-common/package.json | 2 +- packages/backend/src/plugins/techdocs.ts | 3 + plugins/catalog-backend/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- plugins/techdocs-backend/package.json | 2 +- .../techdocs-backend/src/default-branch.ts | 69 +++++++++++++++++++ plugins/techdocs-backend/src/helpers.ts | 7 +- .../src/techdocs/stages/prepare/azure.test.ts | 63 +++++++++++++++++ .../src/techdocs/stages/prepare/azure.ts | 55 +++++++++++++++ .../src/techdocs/stages/prepare/dir.ts | 3 +- .../src/techdocs/stages/prepare/index.ts | 1 + .../src/techdocs/stages/prepare/types.ts | 2 +- yarn.lock | 8 +-- 13 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts create mode 100644 plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 1c888ab8fe..b012b5205b 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -39,7 +39,7 @@ "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", - "git-url-parse": "^11.2.0", + "git-url-parse": "^11.3.0", "helmet": "^4.0.0", "knex": "^0.21.1", "lodash": "^4.17.15", diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index e04d2a2732..6e36c26f39 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -23,6 +23,7 @@ import { TechdocsGenerator, GithubPreparer, GitlabPreparer, + AzurePreparer, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -39,10 +40,12 @@ export default async function createPlugin({ const preparers = new Preparers(); const githubPreparer = new GithubPreparer(logger); const gitlabPreparer = new GitlabPreparer(logger); + const azurePreparer = new AzurePreparer(logger); const directoryPreparer = new DirectoryPreparer(logger); preparers.register('dir', directoryPreparer); preparers.register('github', githubPreparer); preparers.register('gitlab', gitlabPreparer); + preparers.register('azure/api', azurePreparer); const publisher = new LocalPublish(logger); diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f16f629c44..f2392571ce 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -31,7 +31,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.2.0", + "git-url-parse": "^11.3.0", "knex": "^0.21.1", "ldapjs": "^2.2.0", "lodash": "^4.17.15", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index ff584e7374..b146609d8d 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -36,7 +36,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.2.0", + "git-url-parse": "^11.3.0", "globby": "^11.0.0", "helmet": "^4.0.0", "jsonschema": "^1.2.6", diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 75f6071464..a9b6621a20 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -31,7 +31,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", - "git-url-parse": "^11.2.0", + "git-url-parse": "^11.3.0", "knex": "^0.21.1", "node-fetch": "^2.6.0", "nodegit": "^0.27.0", diff --git a/plugins/techdocs-backend/src/default-branch.ts b/plugins/techdocs-backend/src/default-branch.ts index a903b780d8..487f343d93 100644 --- a/plugins/techdocs-backend/src/default-branch.ts +++ b/plugins/techdocs-backend/src/default-branch.ts @@ -61,6 +61,16 @@ function getGitlabApiUrl(url: string): URL { ); } +function getAzureApiUrl(url: string): URL { + const { protocol, resource, organization, owner, name } = parseGitUrl(url); + const apiRepoPath = '_apis/git/repositories'; + const apiVersion = 'api-version=6.0'; + + return new URL( + `${protocol}://${resource}/${organization}/${owner}/${apiRepoPath}/${name}?${apiVersion}`, + ); +} + function getGithubRequestOptions(config: Config): RequestInit { const headers: HeadersInit = { Accept: 'application/vnd.github.v3.raw', @@ -99,6 +109,26 @@ function getGitlabRequestOptions(config: Config): RequestInit { }; } +function getAzureRequestOptions(config: Config): RequestInit { + const headers: HeadersInit = {}; + + const token = + config.getOptionalString('catalog.processors.azureApi.privateToken') ?? + process.env.AZURE_TOKEN; + + if (token !== '') { + headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString( + 'base64', + )}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; +} + async function getGithubDefaultBranch( repositoryUrl: string, config: Config, @@ -159,6 +189,42 @@ async function getGitlabDefaultBranch( } } +async function getAzureDefaultBranch( + repositoryUrl: string, + config: Config, +): Promise { + const path = getAzureApiUrl(repositoryUrl).toString(); + + const options = getAzureRequestOptions(config); + + try { + const urlResponse = await fetch(path, options); + if (!urlResponse.ok) { + throw new Error( + `Failed to load url: ${urlResponse.status} ${urlResponse.statusText}. Make sure you have permission to repository: ${repositoryUrl}`, + ); + } + const urlResult = await urlResponse.json(); + + const idResponse = await fetch(urlResult.url, options); + if (!idResponse.ok) { + throw new Error( + `Failed to load url: ${idResponse.status} ${idResponse.statusText}. Make sure you have permission to repository: ${urlResult.repository.url}`, + ); + } + const idResult = await idResponse.json(); + const name = idResult.defaultBranch; + + if (!name) { + throw new Error('Not found Azure DevOps default branch'); + } + + return name; + } catch (error) { + throw new Error(`Failed to get Azure DevOps default branch: ${error}`); + } +} + export const getDefaultBranch = async ( repositoryUrl: string, ): Promise => { @@ -166,6 +232,7 @@ export const getDefaultBranch = async ( const typeMapping = [ { url: /github*/g, type: 'github' }, { url: /gitlab*/g, type: 'gitlab' }, + { url: /azure*/g, type: 'azure/api' }, ]; const type = typeMapping.filter(item => item.url.test(repositoryUrl))[0] @@ -177,6 +244,8 @@ export const getDefaultBranch = async ( return await getGithubDefaultBranch(repositoryUrl, config); case 'gitlab': return await getGitlabDefaultBranch(repositoryUrl, config); + case 'azure/api': + return await getAzureDefaultBranch(repositoryUrl, config); default: throw new Error('Failed to get repository type'); diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index e79df12365..b2dcd1e8e8 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -76,6 +76,7 @@ export const getLocationForEntity = ( switch (type) { case 'github': case 'gitlab': + case 'azure/api': return { type, target }; case 'dir': if (path.isAbsolute(target)) return { type, target }; @@ -124,9 +125,13 @@ export const checkoutGitRepository = async ( const user = process.env.GITHUB_PRIVATE_TOKEN_USER || process.env.GITLAB_PRIVATE_TOKEN_USER || + process.env.AZURE_PRIVATE_TOKEN_USER || ''; const token = - process.env.GITHUB_TOKEN || process.env.GITLAB_PRIVATE_TOKEN_USER || ''; + process.env.GITHUB_TOKEN || + process.env.GITLAB_PRIVATE_TOKEN_USER || + process.env.AZURE_TOKEN || + ''; if (fs.existsSync(repositoryTmpPath)) { try { diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts new file mode 100644 index 0000000000..39a3d5c7fa --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { AzurePreparer } from './azure'; +import { checkoutGitRepository } from '../../../helpers'; + +function normalizePath(path: string) { + return path + .replace(/^[a-z]:/i, '') + .split('\\') + .join('/'); +} + +jest.mock('../../../helpers', () => ({ + ...jest.requireActual<{}>('../../../helpers'), + checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'), +})); + +const createMockEntity = (annotations = {}) => { + return { + apiVersion: 'version', + kind: 'TestKind', + metadata: { + name: 'test-component-name', + annotations: { + ...annotations, + }, + }, + }; +}; + +const logger = getVoidLogger(); + +describe('Azure DevOps preparer', () => { + it('should prepare temp docs path from Azure DevOps repo', async () => { + const preparer = new AzurePreparer(logger); + + const mockEntity = createMockEntity({ + 'backstage.io/techdocs-ref': + 'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml', + }); + + const tempDocsPath = await preparer.prepare(mockEntity); + expect(checkoutGitRepository).toHaveBeenCalledTimes(1); + expect(normalizePath(tempDocsPath)).toEqual( + '/tmp/backstage-repo/org/name/branch/template.yaml', + ); + }); +}); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts new file mode 100644 index 0000000000..18e6c3c755 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import { Entity } from '@backstage/catalog-model'; +import { InputError } from '@backstage/backend-common'; +import { PreparerBase } from './types'; +import parseGitUrl from 'git-url-parse'; +import { + parseReferenceAnnotation, + checkoutGitRepository, +} from '../../../helpers'; + +import { Logger } from 'winston'; + +export class AzurePreparer implements PreparerBase { + private readonly logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + + async prepare(entity: Entity): Promise { + const { type, target } = parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + entity, + ); + + if (type !== 'azure/api') { + throw new InputError(`Wrong target type: ${type}, should be 'azure/api'`); + } + + try { + const repoPath = await checkoutGitRepository(target, this.logger); + const parsedGitLocation = parseGitUrl(target); + + return path.join(repoPath, parsedGitLocation.filepath); + } catch (error) { + this.logger.debug(`Repo checkout failed with error ${error.message}`); + throw error; + } + } +} diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts index cad8a7effa..d76aee25bd 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts @@ -42,7 +42,8 @@ export class DirectoryPreparer implements PreparerBase { ); switch (type) { case 'github': - case 'gitlab': { + case 'gitlab': + case 'azure/api': { const parsedGitLocation = parseGitUrl(target); const repoLocation = await checkoutGitRepository(target, this.logger); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts index 9e91958bd7..249854f7ee 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts @@ -16,5 +16,6 @@ export { DirectoryPreparer } from './dir'; export { GithubPreparer } from './github'; export { GitlabPreparer } from './gitlab'; +export { AzurePreparer } from './azure'; export { Preparers } from './preparers'; export type { PreparerBuilder, PreparerBase } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts index 4cb1e1e006..1dd091e107 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts @@ -30,4 +30,4 @@ export type PreparerBuilder = { get(entity: Entity): PreparerBase; }; -export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file'; +export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api'; diff --git a/yarn.lock b/yarn.lock index f84330cc78..28f6f57cca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11985,10 +11985,10 @@ git-url-parse@^11.1.2: dependencies: git-up "^4.0.0" -git-url-parse@^11.2.0: - version "11.2.0" - resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.2.0.tgz#2955fd51befd6d96ea1389bbe2ef57e8e6042b04" - integrity sha512-KPoHZg8v+plarZvto4ruIzzJLFQoRx+sUs5DQSr07By9IBKguVd+e6jwrFR6/TP6xrCJlNV1tPqLO1aREc7O2g== +git-url-parse@^11.3.0: + version "11.3.0" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.3.0.tgz#1515b4574c4eb2efda7d25cc50b29ce8beaefaae" + integrity sha512-i3XNa8IKmqnUqWBcdWBjOcnyZYfN3C1WRvnKI6ouFWwsXCZEnlgbwbm55ZpJ3OJMhfEP/ryFhqW8bBhej3C5Ug== dependencies: git-up "^4.0.0"