paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 16 Packages Projects Releases Wiki Activity

TechDocs: add gitlab preparer & rewrite default-branch package (#2469)

Browse Source 
* add gitlab preparer & rewrite default-branch package

* feat(techdocs-backend): rewrite default-branch module
This commit is contained in:
Evgeny Vlasov
2020-10-02 12:10:02 +03:00
committed by GitHub
parent a825e4321a
commit e0dce26354
14 changed files with 353 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs-backend/src/default-branch.ts
+187
View File
@@ -0,0 +1,187 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fetch, { RequestInit } from 'node-fetch';
import parseGitUrl from 'git-url-parse';
import { ConfigReader, Config } from '@backstage/config';
import { loadBackendConfig } from '@backstage/backend-common';
interface IGitlabBranch {
name: string;
merged: boolean;
protected: boolean;
default: boolean;
developers_can_push: boolean;
developers_can_merge: boolean;
can_push: boolean;
web_url: string;
commit: {
author_email: string;
author_name: string;
authored_date: string;
committed_date: string;
committer_email: string;
committer_name: string;
id: string;
short_id: string;
title: string;
message: string;
parent_ids: string[];
};
}
function getGithubApiUrl(url: string): URL {
const { protocol, owner, name } = parseGitUrl(url);
const apiBaseUrl = 'api.github.com';
const apiRepos = 'repos';
return new URL(`${protocol}://${apiBaseUrl}/${apiRepos}/${owner}/${name}`);
}
function getGitlabApiUrl(url: string): URL {
const { protocol, resource, full_name: fullName } = parseGitUrl(url);
const apiProjectsBasePath = 'api/v4/projects';
const project = encodeURIComponent(fullName);
const branches = 'repository/branches';
return new URL(
`${protocol}://${resource}/${apiProjectsBasePath}/${project}/${branches}`,
);
}
function getGithubRequestOptions(config: Config): RequestInit {
const headers: HeadersInit = {
Accept: 'application/vnd.github.v3.raw',
};
const token =
config.getOptionalString('catalog.processors.github.privateToken') ??
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
process.env.GITHUB_PRIVATE_TOKEN;
if (token) {
headers.Authorization = `token ${token}`;
}
return {
headers,
};
}
function getGitlabRequestOptions(config: Config): RequestInit {
const headers: HeadersInit = {
'PRIVATE-TOKEN': '',
};
const token =
config.getOptionalString('catalog.processors.gitlab.privateToken') ??
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
process.env.GITLAB_ACCESS_TOKEN;
if (token) {
headers['PRIVATE-TOKEN'] = token;
}
return {
headers,
};
}
async function getGithubDefaultBranch(
repositoryUrl: string,
config: Config,
): Promise<string> {
const path = getGithubApiUrl(repositoryUrl).toString();
const options = getGithubRequestOptions(config);
try {
const raw = await fetch(path, options);
if (!raw.ok) {
throw new Error(
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
);
}
const { default_branch: branch } = await raw.json();
if (!branch) {
throw new Error('Not found github default branch');
}
return branch;
} catch (error) {
throw new Error(`Failed to get github default branch: ${error}`);
}
}
async function getGitlabDefaultBranch(
repositoryUrl: string,
config: Config,
): Promise<string> {
const path = getGitlabApiUrl(repositoryUrl).toString();
const options = getGitlabRequestOptions(config);
try {
const raw = await fetch(path, options);
if (!raw.ok) {
throw new Error(
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
);
}
const result = await raw.json();
const { name } = (result || []).find(
(branch: IGitlabBranch) => branch.default === true,
);
if (!name) {
throw new Error('Not found gitlab default branch');
}
return name;
} catch (error) {
throw new Error(`Failed to get gitlab default branch: ${error}`);
}
}
export const getDefaultBranch = async (
repositoryUrl: string,
): Promise<string> => {
const config = ConfigReader.fromConfigs(await loadBackendConfig());
const typeMapping = [
{ url: /github*/g, type: 'github' },
{ url: /gitlab*/g, type: 'gitlab' },
];
const type = typeMapping.filter(item => item.url.test(repositoryUrl))[0]
?.type;
try {
switch (type) {
case 'github':
return await getGithubDefaultBranch(repositoryUrl, config);
case 'gitlab':
return await getGitlabDefaultBranch(repositoryUrl, config);
default:
throw new Error('Failed to get repository type');
}
} catch (error) {
throw error;
}
};
plugins/techdocs-backend/src/helpers.ts
+15 -12
View File
@@ -19,8 +19,7 @@ import path from 'path';
import parseGitUrl from 'git-url-parse';
import NodeGit, { Clone, Repository } from 'nodegit';
import fs from 'fs-extra';
// @ts-ignore
import defaultBranch from 'default-branch';
import { getDefaultBranch } from './default-branch';
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/backend-common';
import { RemoteProtocol } from './techdocs/stages/prepare/types';
@@ -76,6 +75,7 @@ export const getLocationForEntity = (
switch (type) {
case 'github':
case 'gitlab':
return { type, target };
case 'dir':
if (path.isAbsolute(target)) return { type, target };
@@ -89,7 +89,7 @@ export const getLocationForEntity = (
}
};
export const getGitHubRepositoryTempFolder = async (
export const getGitRepositoryTempFolder = async (
repositoryUrl: string,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repositoryUrl);
@@ -97,7 +97,7 @@ export const getGitHubRepositoryTempFolder = async (
parsedGitLocation.git_suffix = false;
if (!parsedGitLocation.ref) {
parsedGitLocation.ref = await defaultBranch(
parsedGitLocation.ref = await getDefaultBranch(
parsedGitLocation.toString('https'),
);
}
@@ -113,16 +113,22 @@ export const getGitHubRepositoryTempFolder = async (
);
};
export const checkoutGithubRepository = async (
export const checkoutGitRepository = async (
repoUrl: string,
logger: Logger,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repoUrl);
const repositoryTmpPath = await getGitHubRepositoryTempFolder(repoUrl);
const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl);
// TODO: Should propably not be hardcoded names of env variables, but seems too hard to access config down here
const user = process.env.GITHUB_PRIVATE_TOKEN_USER || '';
const token = process.env.GITHUB_PRIVATE_TOKEN || '';
const user =
process.env.GITHUB_PRIVATE_TOKEN_USER ||
process.env.GITLAB_PRIVATE_TOKEN_USER ||
'';
const token =
process.env.GITHUB_PRIVATE_TOKEN ||
process.env.GITLAB_PRIVATE_TOKEN_USER ||
'';
if (fs.existsSync(repositoryTmpPath)) {
try {
@@ -160,10 +166,7 @@ export const getLastCommitTimestamp = async (
repositoryUrl: string,
logger: Logger,
): Promise<number> => {
const repositoryLocation = await checkoutGithubRepository(
repositoryUrl,
logger,
);
const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger);
const repository = await Repository.open(repositoryLocation);
const commit = await repository.getReferenceCommit('HEAD');
plugins/techdocs-backend/src/service/helpers.ts
+9 -15
View File
@@ -106,22 +106,16 @@ export class DocsBuilder {
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
const { type, target } = getLocationForEntity(this.entity);
const { target } = getLocationForEntity(this.entity);
const lastCommit = await getLastCommitTimestamp(target, this.logger);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Should probably be broken out and handled per type later. Doing this for now since we only support github age checks
if (type === 'github') {
const lastCommit = await getLastCommitTimestamp(target, this.logger);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`[TechDocs] Docs for entity ${getEntityId(
this.entity,
)} is up to date.`,
);
return true;
}
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`[TechDocs] Docs for entity ${getEntityId(this.entity)} is up to date.`,
);
return true;
}
this.logger.debug(
plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts
+3 -5
View File
@@ -15,7 +15,7 @@
*/
import { DirectoryPreparer } from './dir';
import { getVoidLogger } from '@backstage/backend-common';
import { checkoutGithubRepository } from '../../../helpers';
import { checkoutGitRepository } from '../../../helpers';
function normalizePath(path: string) {
return path
@@ -26,9 +26,7 @@ function normalizePath(path: string) {
jest.mock('../../../helpers', () => ({
...jest.requireActual<{}>('../../../helpers'),
checkoutGithubRepository: jest.fn(
() => '/tmp/backstage-repo/org/name/branch/',
),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
}));
const logger = getVoidLogger();
@@ -87,6 +85,6 @@ describe('directory preparer', () => {
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
'/tmp/backstage-repo/org/name/branch/docs',
);
expect(checkoutGithubRepository).toHaveBeenCalledTimes(1);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
});
});
plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts
+5 -6
View File
@@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model';
import path from 'path';
import {
parseReferenceAnnotation,
checkoutGithubRepository,
checkoutGitRepository,
} from '../../../helpers';
import { InputError } from '@backstage/backend-common';
import parseGitUrl from 'git-url-parse';
@@ -41,17 +41,16 @@ export class DirectoryPreparer implements PreparerBase {
`[TechDocs] Building docs for entity with type 'dir' and managed-by-location '${type}'`,
);
switch (type) {
case 'github': {
case 'github':
case 'gitlab': {
const parsedGitLocation = parseGitUrl(target);
const repoLocation = await checkoutGithubRepository(
target,
this.logger,
);
const repoLocation = await checkoutGitRepository(target, this.logger);
return path.dirname(
path.join(repoLocation, parsedGitLocation.filepath),
);
}
case 'file':
return path.dirname(target);
default:
plugins/techdocs-backend/src/techdocs/stages/prepare/github.test.ts
+3 -5
View File
@@ -16,7 +16,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { GithubPreparer } from './github';
import { checkoutGithubRepository } from '../../../helpers';
import { checkoutGitRepository } from '../../../helpers';
function normalizePath(path: string) {
return path
@@ -27,9 +27,7 @@ function normalizePath(path: string) {
jest.mock('../../../helpers', () => ({
...jest.requireActual<{}>('../../../helpers'),
checkoutGithubRepository: jest.fn(
() => '/tmp/backstage-repo/org/name/branch',
),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
}));
const createMockEntity = (annotations = {}) => {
@@ -57,7 +55,7 @@ describe('github preparer', () => {
});
const tempDocsPath = await preparer.prepare(mockEntity);
expect(checkoutGithubRepository).toHaveBeenCalledTimes(1);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
expect(normalizePath(tempDocsPath)).toEqual(
'/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component',
);
plugins/techdocs-backend/src/techdocs/stages/prepare/github.ts
+2 -2
View File
@@ -20,7 +20,7 @@ import { PreparerBase } from './types';
import parseGitUrl from 'git-url-parse';
import {
parseReferenceAnnotation,
checkoutGithubRepository,
checkoutGitRepository,
} from '../../../helpers';
import { Logger } from 'winston';
@@ -43,7 +43,7 @@ export class GithubPreparer implements PreparerBase {
}
try {
const repoPath = await checkoutGithubRepository(target, this.logger);
const repoPath = await checkoutGitRepository(target, this.logger);
const parsedGitLocation = parseGitUrl(target);
return path.join(repoPath, parsedGitLocation.filepath);
plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.test.ts
+64
View File
@@ -0,0 +1,64 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { GitlabPreparer } from './gitlab';
import { checkoutGitRepository } from '../../../helpers';
function normalizePath(path: string) {
return path
.replace(/^[a-z]:/i, '')
.split('\\')
.join('/');
}
jest.mock('../../../helpers', () => ({
...jest.requireActual<{}>('../../../helpers'),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
}));
const createMockEntity = (annotations = {}) => {
return {
apiVersion: 'version',
kind: 'TestKind',
metadata: {
name: 'test-component-name',
annotations: {
...annotations,
},
},
};
};
const logger = getVoidLogger();
describe('gitlab preparer', () => {
it('should prepare temp docs path from gitlab repo', async () => {
const preparer = new GitlabPreparer(logger);
// TODO: fix url repo
const mockEntity = createMockEntity({
'backstage.io/techdocs-ref':
'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml',
});
const tempDocsPath = await preparer.prepare(mockEntity);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
expect(normalizePath(tempDocsPath)).toEqual(
'/tmp/backstage-repo/org/name/branch/catalog-info.yaml',
);
});
});
plugins/techdocs-backend/src/techdocs/stages/prepare/gitlab.ts
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
import parseGitUrl from 'git-url-parse';
import {
parseReferenceAnnotation,
checkoutGitRepository,
} from '../../../helpers';
import { Logger } from 'winston';
export class GitlabPreparer implements PreparerBase {
private readonly logger: Logger;
constructor(logger: Logger) {
this.logger = logger;
}
async prepare(entity: Entity): Promise<string> {
const { type, target } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
if (type !== 'gitlab') {
throw new InputError(`Wrong target type: ${type}, should be 'gitlab'`);
}
try {
const repoPath = await checkoutGitRepository(target, this.logger);
const parsedGitLocation = parseGitUrl(target);
return path.join(repoPath, parsedGitLocation.filepath);
} catch (error) {
this.logger.debug(`Repo checkout failed with error ${error.message}`);
throw error;
}
}
}
plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts
+1
View File
@@ -15,5 +15,6 @@
*/
export { DirectoryPreparer } from './dir';
export { GithubPreparer } from './github';
export { GitlabPreparer } from './gitlab';
export { Preparers } from './preparers';
export type { PreparerBuilder, PreparerBase } from './types';
plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts
+1 -1
View File
@@ -30,4 +30,4 @@ export type PreparerBuilder = {
get(entity: Entity): PreparerBase;
};
export type RemoteProtocol = 'dir' | 'github' | 'file';
export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file';