diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 9f66cbe2ff..e04d2a2732 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -22,6 +22,7 @@ import { LocalPublish, TechdocsGenerator, GithubPreparer, + GitlabPreparer, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -37,9 +38,11 @@ export default async function createPlugin({ const preparers = new Preparers(); const githubPreparer = new GithubPreparer(logger); + const gitlabPreparer = new GitlabPreparer(logger); const directoryPreparer = new DirectoryPreparer(logger); preparers.register('dir', directoryPreparer); preparers.register('github', githubPreparer); + preparers.register('gitlab', gitlabPreparer); const publisher = new LocalPublish(logger); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts index 9f308ee184..c33b388e1a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -62,19 +62,15 @@ export class GitlabReaderProcessor implements LocationProcessor { try { const url = new URL(target); - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); + const [empty, userOrOrg, repoName, , ...restOfPath] = url.pathname + .split('/') + // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml + .filter(path => path !== '-'); if ( empty !== '' || userOrOrg === '' || repoName === '' || - blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { throw new Error('Wrong GitLab URL'); diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 0d90569574..933b67060a 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -86,7 +86,7 @@ export const RegisterComponentPage = ({ const { scmType, componentLocation: target } = formData; try { const typeMapping = [ - { url: /^https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, + { url: /^https:\/\/gitlab\.com\/.*/, type: 'gitlab/api' }, { url: /^https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, { url: /^https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, { url: /.*/, type: 'github' }, diff --git a/plugins/techdocs-backend/src/default-branch.ts b/plugins/techdocs-backend/src/default-branch.ts new file mode 100644 index 0000000000..05a8f666ce --- /dev/null +++ b/plugins/techdocs-backend/src/default-branch.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fetch, { RequestInit } from 'node-fetch'; +import parseGitUrl from 'git-url-parse'; +import { ConfigReader, Config } from '@backstage/config'; +import { loadBackendConfig } from '@backstage/backend-common'; + +interface IGitlabBranch { + name: string; + merged: boolean; + protected: boolean; + default: boolean; + developers_can_push: boolean; + developers_can_merge: boolean; + can_push: boolean; + web_url: string; + commit: { + author_email: string; + author_name: string; + authored_date: string; + committed_date: string; + committer_email: string; + committer_name: string; + id: string; + short_id: string; + title: string; + message: string; + parent_ids: string[]; + }; +} + +function getGithubApiUrl(url: string): URL { + const { protocol, owner, name } = parseGitUrl(url); + const apiBaseUrl = 'api.github.com'; + const apiRepos = 'repos'; + + return new URL(`${protocol}://${apiBaseUrl}/${apiRepos}/${owner}/${name}`); +} + +function getGitlabApiUrl(url: string): URL { + const { protocol, resource, full_name: fullName } = parseGitUrl(url); + const apiProjectsBasePath = 'api/v4/projects'; + const project = encodeURIComponent(fullName); + const branches = 'repository/branches'; + + return new URL( + `${protocol}://${resource}/${apiProjectsBasePath}/${project}/${branches}`, + ); +} + +function getGithubRequestOptions(config: Config): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + const token = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken') ?? + process.env.GITHUB_PRIVATE_TOKEN; + + if (token) { + headers.Authorization = `token ${token}`; + } + + return { + headers, + }; +} + +function getGitlabRequestOptions(config: Config): RequestInit { + const headers: HeadersInit = { + 'PRIVATE-TOKEN': '', + }; + + const token = + config.getOptionalString('catalog.processors.gitlab.privateToken') ?? + config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? + process.env.GITLAB_ACCESS_TOKEN; + + if (token) { + headers['PRIVATE-TOKEN'] = token; + } + + return { + headers, + }; +} + +async function getGithubDefaultBranch( + repositoryUrl: string, + config: Config, +): Promise { + const path = getGithubApiUrl(repositoryUrl).toString(); + const options = getGithubRequestOptions(config); + + try { + const raw = await fetch(path, options); + + if (!raw.ok) { + throw new Error( + `Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`, + ); + } + + const { default_branch: branch } = await raw.json(); + + if (!branch) { + throw new Error('Not found github default branch'); + } + + return branch; + } catch (error) { + throw new Error(`Failed to get github default branch: ${error}`); + } +} + +async function getGitlabDefaultBranch( + repositoryUrl: string, + config: Config, +): Promise { + const path = getGitlabApiUrl(repositoryUrl).toString(); + + const options = getGitlabRequestOptions(config); + + try { + const raw = await fetch(path, options); + + if (!raw.ok) { + throw new Error( + `Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`, + ); + } + + const result = await raw.json(); + const { name } = (result || []).find( + (branch: IGitlabBranch) => branch.default === true, + ); + + if (!name) { + throw new Error('Not found gitlab default branch'); + } + + return name; + } catch (error) { + throw new Error(`Failed to get gitlab default branch: ${error}`); + } +} + +export const getDefaultBranch = async ( + repositoryUrl: string, +): Promise => { + const config = ConfigReader.fromConfigs(await loadBackendConfig()); + const typeMapping = [ + { url: /github*/g, type: 'github' }, + { url: /gitlab*/g, type: 'gitlab' }, + ]; + + const type = typeMapping.filter(item => item.url.test(repositoryUrl))[0] + ?.type; + + try { + switch (type) { + case 'github': + return await getGithubDefaultBranch(repositoryUrl, config); + case 'gitlab': + return await getGitlabDefaultBranch(repositoryUrl, config); + + default: + throw new Error('Failed to get repository type'); + } + } catch (error) { + throw error; + } +}; diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index 8acd836cad..d6a275f183 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -19,8 +19,7 @@ import path from 'path'; import parseGitUrl from 'git-url-parse'; import NodeGit, { Clone, Repository } from 'nodegit'; import fs from 'fs-extra'; -// @ts-ignore -import defaultBranch from 'default-branch'; +import { getDefaultBranch } from './default-branch'; import { Entity } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; import { RemoteProtocol } from './techdocs/stages/prepare/types'; @@ -76,6 +75,7 @@ export const getLocationForEntity = ( switch (type) { case 'github': + case 'gitlab': return { type, target }; case 'dir': if (path.isAbsolute(target)) return { type, target }; @@ -89,7 +89,7 @@ export const getLocationForEntity = ( } }; -export const getGitHubRepositoryTempFolder = async ( +export const getGitRepositoryTempFolder = async ( repositoryUrl: string, ): Promise => { const parsedGitLocation = parseGitUrl(repositoryUrl); @@ -97,7 +97,7 @@ export const getGitHubRepositoryTempFolder = async ( parsedGitLocation.git_suffix = false; if (!parsedGitLocation.ref) { - parsedGitLocation.ref = await defaultBranch( + parsedGitLocation.ref = await getDefaultBranch( parsedGitLocation.toString('https'), ); } @@ -113,16 +113,22 @@ export const getGitHubRepositoryTempFolder = async ( ); }; -export const checkoutGithubRepository = async ( +export const checkoutGitRepository = async ( repoUrl: string, logger: Logger, ): Promise => { const parsedGitLocation = parseGitUrl(repoUrl); - const repositoryTmpPath = await getGitHubRepositoryTempFolder(repoUrl); + const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl); // TODO: Should propably not be hardcoded names of env variables, but seems too hard to access config down here - const user = process.env.GITHUB_PRIVATE_TOKEN_USER || ''; - const token = process.env.GITHUB_PRIVATE_TOKEN || ''; + const user = + process.env.GITHUB_PRIVATE_TOKEN_USER || + process.env.GITLAB_PRIVATE_TOKEN_USER || + ''; + const token = + process.env.GITHUB_PRIVATE_TOKEN || + process.env.GITLAB_PRIVATE_TOKEN_USER || + ''; if (fs.existsSync(repositoryTmpPath)) { try { @@ -160,10 +166,7 @@ export const getLastCommitTimestamp = async ( repositoryUrl: string, logger: Logger, ): Promise => { - const repositoryLocation = await checkoutGithubRepository( - repositoryUrl, - logger, - ); + const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger); const repository = await Repository.open(repositoryLocation); const commit = await repository.getReferenceCommit('HEAD'); diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 60c76e6384..6bf3626836 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -106,22 +106,16 @@ export class DocsBuilder { const buildMetadataStorage = new BuildMetadataStorage( this.entity.metadata.uid, ); - const { type, target } = getLocationForEntity(this.entity); + const { target } = getLocationForEntity(this.entity); + const lastCommit = await getLastCommitTimestamp(target, this.logger); + const storageTimeStamp = buildMetadataStorage.getTimestamp(); - // Should probably be broken out and handled per type later. Doing this for now since we only support github age checks - if (type === 'github') { - const lastCommit = await getLastCommitTimestamp(target, this.logger); - const storageTimeStamp = buildMetadataStorage.getTimestamp(); - - // Check if documentation source is newer than what we have - if (storageTimeStamp && storageTimeStamp >= lastCommit) { - this.logger.debug( - `[TechDocs] Docs for entity ${getEntityId( - this.entity, - )} is up to date.`, - ); - return true; - } + // Check if documentation source is newer than what we have + if (storageTimeStamp && storageTimeStamp >= lastCommit) { + this.logger.debug( + `[TechDocs] Docs for entity ${getEntityId(this.entity)} is up to date.`, + ); + return true; } this.logger.debug( diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts index e2b94d7e88..4412e6bf73 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts @@ -15,7 +15,7 @@ */ import { DirectoryPreparer } from './dir'; import { getVoidLogger } from '@backstage/backend-common'; -import { checkoutGithubRepository } from '../../../helpers'; +import { checkoutGitRepository } from '../../../helpers'; function normalizePath(path: string) { return path @@ -26,9 +26,7 @@ function normalizePath(path: string) { jest.mock('../../../helpers', () => ({ ...jest.requireActual<{}>('../../../helpers'), - checkoutGithubRepository: jest.fn( - () => '/tmp/backstage-repo/org/name/branch/', - ), + checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'), })); const logger = getVoidLogger(); @@ -87,6 +85,6 @@ describe('directory preparer', () => { expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual( '/tmp/backstage-repo/org/name/branch/docs', ); - expect(checkoutGithubRepository).toHaveBeenCalledTimes(1); + expect(checkoutGitRepository).toHaveBeenCalledTimes(1); }); }); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts index 5fd4abb4bd..cad8a7effa 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { parseReferenceAnnotation, - checkoutGithubRepository, + checkoutGitRepository, } from '../../../helpers'; import { InputError } from '@backstage/backend-common'; import parseGitUrl from 'git-url-parse'; @@ -41,17 +41,16 @@ export class DirectoryPreparer implements PreparerBase { `[TechDocs] Building docs for entity with type 'dir' and managed-by-location '${type}'`, ); switch (type) { - case 'github': { + case 'github': + case 'gitlab': { const parsedGitLocation = parseGitUrl(target); - const repoLocation = await checkoutGithubRepository( - target, - this.logger, - ); + const repoLocation = await checkoutGitRepository(target, this.logger); return path.dirname( path.join(repoLocation, parsedGitLocation.filepath), ); } + case 'file': return path.dirname(target); default: diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts index a188e1c986..a657cd1697 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts @@ -16,7 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GithubPreparer } from './github'; -import { checkoutGithubRepository } from '../../../helpers'; +import { checkoutGitRepository } from '../../../helpers'; function normalizePath(path: string) { return path @@ -27,9 +27,7 @@ function normalizePath(path: string) { jest.mock('../../../helpers', () => ({ ...jest.requireActual<{}>('../../../helpers'), - checkoutGithubRepository: jest.fn( - () => '/tmp/backstage-repo/org/name/branch', - ), + checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'), })); const createMockEntity = (annotations = {}) => { @@ -57,7 +55,7 @@ describe('github preparer', () => { }); const tempDocsPath = await preparer.prepare(mockEntity); - expect(checkoutGithubRepository).toHaveBeenCalledTimes(1); + expect(checkoutGitRepository).toHaveBeenCalledTimes(1); expect(normalizePath(tempDocsPath)).toEqual( '/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component', ); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts index 03b0adcb88..a0f91a375a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts @@ -20,7 +20,7 @@ import { PreparerBase } from './types'; import parseGitUrl from 'git-url-parse'; import { parseReferenceAnnotation, - checkoutGithubRepository, + checkoutGitRepository, } from '../../../helpers'; import { Logger } from 'winston'; @@ -43,7 +43,7 @@ export class GithubPreparer implements PreparerBase { } try { - const repoPath = await checkoutGithubRepository(target, this.logger); + const repoPath = await checkoutGitRepository(target, this.logger); const parsedGitLocation = parseGitUrl(target); return path.join(repoPath, parsedGitLocation.filepath); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.test.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.test.ts new file mode 100644 index 0000000000..75cad50224 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { GitlabPreparer } from './gitlab'; +import { checkoutGitRepository } from '../../../helpers'; + +function normalizePath(path: string) { + return path + .replace(/^[a-z]:/i, '') + .split('\\') + .join('/'); +} + +jest.mock('../../../helpers', () => ({ + ...jest.requireActual<{}>('../../../helpers'), + checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'), +})); + +const createMockEntity = (annotations = {}) => { + return { + apiVersion: 'version', + kind: 'TestKind', + metadata: { + name: 'test-component-name', + annotations: { + ...annotations, + }, + }, + }; +}; + +const logger = getVoidLogger(); + +describe('gitlab preparer', () => { + it('should prepare temp docs path from gitlab repo', async () => { + const preparer = new GitlabPreparer(logger); + + // TODO: fix url repo + const mockEntity = createMockEntity({ + 'backstage.io/techdocs-ref': + 'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml', + }); + + const tempDocsPath = await preparer.prepare(mockEntity); + expect(checkoutGitRepository).toHaveBeenCalledTimes(1); + expect(normalizePath(tempDocsPath)).toEqual( + '/tmp/backstage-repo/org/name/branch/catalog-info.yaml', + ); + }); +}); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.ts new file mode 100644 index 0000000000..56f3793fbb --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import { Entity } from '@backstage/catalog-model'; +import { InputError } from '@backstage/backend-common'; +import { PreparerBase } from './types'; +import parseGitUrl from 'git-url-parse'; +import { + parseReferenceAnnotation, + checkoutGitRepository, +} from '../../../helpers'; + +import { Logger } from 'winston'; + +export class GitlabPreparer implements PreparerBase { + private readonly logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + + async prepare(entity: Entity): Promise { + const { type, target } = parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + entity, + ); + + if (type !== 'gitlab') { + throw new InputError(`Wrong target type: ${type}, should be 'gitlab'`); + } + + try { + const repoPath = await checkoutGitRepository(target, this.logger); + const parsedGitLocation = parseGitUrl(target); + + return path.join(repoPath, parsedGitLocation.filepath); + } catch (error) { + this.logger.debug(`Repo checkout failed with error ${error.message}`); + throw error; + } + } +} diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts index 48ffa75903..9e91958bd7 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts @@ -15,5 +15,6 @@ */ export { DirectoryPreparer } from './dir'; export { GithubPreparer } from './github'; +export { GitlabPreparer } from './gitlab'; export { Preparers } from './preparers'; export type { PreparerBuilder, PreparerBase } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts index 1ceb884ea8..4cb1e1e006 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts @@ -30,4 +30,4 @@ export type PreparerBuilder = { get(entity: Entity): PreparerBase; }; -export type RemoteProtocol = 'dir' | 'github' | 'file'; +export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file';