techdocs-common: Initialize package and move common code from techdocs-backend
This commit is contained in:
Himanshu Mishra
@@ -1,261 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fetch from 'cross-fetch';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
|
||||
import {
|
||||
getAzureHostToken,
|
||||
getGitHost,
|
||||
getGithubHostToken,
|
||||
getGitlabHostToken,
|
||||
getGitRepoType,
|
||||
} from './git-auth';
|
||||
|
||||
interface IGitlabBranch {
|
||||
name: string;
|
||||
merged: boolean;
|
||||
protected: boolean;
|
||||
default: boolean;
|
||||
developers_can_push: boolean;
|
||||
developers_can_merge: boolean;
|
||||
can_push: boolean;
|
||||
web_url: string;
|
||||
commit: {
|
||||
author_email: string;
|
||||
author_name: string;
|
||||
authored_date: string;
|
||||
committed_date: string;
|
||||
committer_email: string;
|
||||
committer_name: string;
|
||||
id: string;
|
||||
short_id: string;
|
||||
title: string;
|
||||
message: string;
|
||||
parent_ids: string[];
|
||||
};
|
||||
}
|
||||
|
||||
function getGithubApiUrl(config: Config, url: string): URL {
|
||||
const { protocol, owner, name } = parseGitUrl(url);
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.github') ?? [];
|
||||
|
||||
// TODO: Maybe we need to filter by host in the array, not sure about GHE
|
||||
const targetProviderConfig = providerConfigs[0];
|
||||
|
||||
const apiBaseUrl =
|
||||
targetProviderConfig?.getOptionalString('integrations.github.apiBaseUrl') ??
|
||||
'api.github.com';
|
||||
const apiRepos = 'repos';
|
||||
|
||||
return new URL(`${protocol}://${apiBaseUrl}/${apiRepos}/${owner}/${name}`);
|
||||
}
|
||||
|
||||
function getGitlabApiUrl(url: string): URL {
|
||||
const { protocol, resource, full_name: fullName } = parseGitUrl(url);
|
||||
const apiProjectsBasePath = 'api/v4/projects';
|
||||
const project = encodeURIComponent(fullName);
|
||||
const branches = 'repository/branches';
|
||||
|
||||
return new URL(
|
||||
`${protocol}://${resource}/${apiProjectsBasePath}/${project}/${branches}`,
|
||||
);
|
||||
}
|
||||
|
||||
function getAzureApiUrl(url: string): URL {
|
||||
const { protocol, resource, organization, owner, name } = parseGitUrl(url);
|
||||
const apiRepoPath = '_apis/git/repositories';
|
||||
const apiVersion = 'api-version=6.0';
|
||||
|
||||
return new URL(
|
||||
`${protocol}://${resource}/${organization}/${owner}/${apiRepoPath}/${name}?${apiVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
function getGithubRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
const token = getGithubHostToken(config, host);
|
||||
|
||||
if (token) {
|
||||
headers.Authorization = `token ${token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
function getGitlabRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
'PRIVATE-TOKEN': '',
|
||||
};
|
||||
|
||||
const token = getGitlabHostToken(config, host);
|
||||
if (token) {
|
||||
headers['PRIVATE-TOKEN'] = token;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
function getAzureRequestOptions(config: Config, host: string): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
const token = getAzureHostToken(config, host);
|
||||
|
||||
if (token !== '') {
|
||||
headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString(
|
||||
'base64',
|
||||
)}`;
|
||||
}
|
||||
|
||||
const requestOptions: RequestInit = {
|
||||
headers,
|
||||
};
|
||||
|
||||
return requestOptions;
|
||||
}
|
||||
|
||||
async function getGithubDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getGithubApiUrl(config, repositoryUrl).toString();
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const options = getGithubRequestOptions(config, host);
|
||||
|
||||
try {
|
||||
const raw = await fetch(path, options);
|
||||
|
||||
if (!raw.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const { default_branch: branch } = await raw.json();
|
||||
|
||||
if (!branch) {
|
||||
throw new Error('Not found github default branch');
|
||||
}
|
||||
|
||||
return branch;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get github default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function getGitlabDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getGitlabApiUrl(repositoryUrl).toString();
|
||||
|
||||
const gitlabHost = getGitHost(repositoryUrl);
|
||||
const options = getGitlabRequestOptions(config, gitlabHost);
|
||||
|
||||
try {
|
||||
const raw = await fetch(path, options);
|
||||
|
||||
if (!raw.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${raw.status} ${raw.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await raw.json();
|
||||
const { name } = (result || []).find(
|
||||
(branch: IGitlabBranch) => branch.default === true,
|
||||
);
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Not found gitlab default branch');
|
||||
}
|
||||
|
||||
return name;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get gitlab default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function getAzureDefaultBranch(
|
||||
repositoryUrl: string,
|
||||
config: Config,
|
||||
): Promise<string> {
|
||||
const path = getAzureApiUrl(repositoryUrl).toString();
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const options = getAzureRequestOptions(config, host);
|
||||
|
||||
try {
|
||||
const urlResponse = await fetch(path, options);
|
||||
if (!urlResponse.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${urlResponse.status} ${urlResponse.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
|
||||
);
|
||||
}
|
||||
const urlResult = await urlResponse.json();
|
||||
|
||||
const idResponse = await fetch(urlResult.url, options);
|
||||
if (!idResponse.ok) {
|
||||
throw new Error(
|
||||
`Failed to load url: ${idResponse.status} ${idResponse.statusText}. Make sure you have permission to repository: ${urlResult.repository.url}`,
|
||||
);
|
||||
}
|
||||
const idResult = await idResponse.json();
|
||||
const name = idResult.defaultBranch;
|
||||
|
||||
if (!name) {
|
||||
throw new Error('Not found Azure DevOps default branch');
|
||||
}
|
||||
|
||||
return name;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get Azure DevOps default branch: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const getDefaultBranch = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string> => {
|
||||
// TODO(Rugvip): Config should not be loaded here, pass it in instead
|
||||
const config = await loadBackendConfig({
|
||||
logger: getRootLogger(),
|
||||
argv: process.argv,
|
||||
});
|
||||
const type = getGitRepoType(repositoryUrl);
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'github':
|
||||
return await getGithubDefaultBranch(repositoryUrl, config);
|
||||
case 'gitlab':
|
||||
return await getGitlabDefaultBranch(repositoryUrl, config);
|
||||
case 'azure/api':
|
||||
return await getAzureDefaultBranch(repositoryUrl, config);
|
||||
|
||||
default:
|
||||
throw new Error('Failed to get repository type');
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -1,120 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getRootLogger, loadBackendConfig } from '@backstage/backend-common';
|
||||
|
||||
export function getGitHost(url: string): string {
|
||||
const { resource } = parseGitUrl(url);
|
||||
return resource;
|
||||
}
|
||||
|
||||
export function getGitRepoType(url: string): string {
|
||||
const typeMapping = [
|
||||
{ url: /github*/g, type: 'github' },
|
||||
{ url: /gitlab*/g, type: 'gitlab' },
|
||||
{ url: /azure*/g, type: 'azure/api' },
|
||||
];
|
||||
|
||||
const type = typeMapping.filter(item => item.url.test(url))[0]?.type;
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
export function getGithubHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.github') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.github.privateToken') ??
|
||||
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
|
||||
process.env.GITHUB_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export function getGitlabHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.gitlab') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.gitlab.privateToken') ??
|
||||
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
|
||||
process.env.GITLAB_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export function getAzureHostToken(
|
||||
config: Config,
|
||||
host: string,
|
||||
): string | undefined {
|
||||
const providerConfigs =
|
||||
config.getOptionalConfigArray('integrations.azure') ?? [];
|
||||
|
||||
const hostConfig = providerConfigs.filter(
|
||||
providerConfig => providerConfig.getOptionalString('host') === host,
|
||||
);
|
||||
const token =
|
||||
hostConfig[0]?.getOptionalString('token') ??
|
||||
config.getOptionalString('catalog.processors.azureApi.privateToken') ??
|
||||
process.env.AZURE_TOKEN;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
export const getTokenForGitRepo = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string | undefined> => {
|
||||
// TODO(Rugvip): Config should not be loaded here, pass it in instead
|
||||
const config = await loadBackendConfig({
|
||||
logger: getRootLogger(),
|
||||
argv: process.argv,
|
||||
});
|
||||
|
||||
const host = getGitHost(repositoryUrl);
|
||||
const type = getGitRepoType(repositoryUrl);
|
||||
|
||||
try {
|
||||
switch (type) {
|
||||
case 'github':
|
||||
return getGithubHostToken(config, host);
|
||||
case 'gitlab':
|
||||
return getGitlabHostToken(config, host);
|
||||
case 'azure/api':
|
||||
return getAzureHostToken(config, host);
|
||||
|
||||
default:
|
||||
throw new Error('Failed to get repository type');
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -1,70 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Readable } from 'stream';
|
||||
import { getDocFilesFromRepository } from './helpers';
|
||||
import { UrlReader, ReadTreeResponse } from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
|
||||
describe('getDocFilesFromRepository', () => {
|
||||
it('should read a remote directory using UrlReader.readTree', async () => {
|
||||
class MockUrlReader implements UrlReader {
|
||||
async read() {
|
||||
return Buffer.from('mock');
|
||||
}
|
||||
|
||||
async readTree(): Promise<ReadTreeResponse> {
|
||||
return {
|
||||
dir: async () => {
|
||||
return '/tmp/testfolder';
|
||||
},
|
||||
files: async () => {
|
||||
return [];
|
||||
},
|
||||
archive: async () => {
|
||||
return Readable.from('');
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const mockEntity: Entity = {
|
||||
metadata: {
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
'backstage.io/techdocs-ref':
|
||||
'url:https://github.com/backstage/backstage/blob/master/subfolder/',
|
||||
},
|
||||
name: 'mytestcomponent',
|
||||
description: 'A component for testing',
|
||||
},
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
spec: {
|
||||
type: 'documentation',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'testuser',
|
||||
},
|
||||
};
|
||||
|
||||
const output = await getDocFilesFromRepository(
|
||||
new MockUrlReader(),
|
||||
mockEntity,
|
||||
);
|
||||
|
||||
expect(output).toBe('/tmp/testfolder');
|
||||
});
|
||||
});
|
||||
@@ -1,195 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import NodeGit, { Clone, Repository } from 'nodegit';
|
||||
import fs from 'fs-extra';
|
||||
import { getDefaultBranch } from './default-branch';
|
||||
import { getGitRepoType, getTokenForGitRepo } from './git-auth';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { InputError, UrlReader } from '@backstage/backend-common';
|
||||
import { RemoteProtocol } from './techdocs/stages/prepare/types';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
// Enables core.longpaths on windows to prevent crashing when checking out repos with long foldernames and/or deep nesting
|
||||
// @ts-ignore
|
||||
NodeGit.Libgit2.opts(28, 1);
|
||||
|
||||
export type ParsedLocationAnnotation = {
|
||||
type: RemoteProtocol;
|
||||
target: string;
|
||||
};
|
||||
|
||||
export const parseReferenceAnnotation = (
|
||||
annotationName: string,
|
||||
entity: Entity,
|
||||
): ParsedLocationAnnotation => {
|
||||
const annotation = entity.metadata.annotations?.[annotationName];
|
||||
|
||||
if (!annotation) {
|
||||
throw new InputError(
|
||||
`No location annotation provided in entity: ${entity.metadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
// split on the first colon for the protocol and the rest after the first split
|
||||
// is the location.
|
||||
const [type, target] = annotation.split(/:(.+)/) as [
|
||||
RemoteProtocol?,
|
||||
string?,
|
||||
];
|
||||
|
||||
if (!type || !target) {
|
||||
throw new InputError(
|
||||
`Failure to parse either protocol or location for entity: ${entity.metadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
type,
|
||||
target,
|
||||
};
|
||||
};
|
||||
|
||||
export const getLocationForEntity = (
|
||||
entity: Entity,
|
||||
): ParsedLocationAnnotation => {
|
||||
const { type, target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
switch (type) {
|
||||
case 'github':
|
||||
case 'gitlab':
|
||||
case 'azure/api':
|
||||
case 'url':
|
||||
return { type, target };
|
||||
case 'dir':
|
||||
if (path.isAbsolute(target)) return { type, target };
|
||||
|
||||
return parseReferenceAnnotation(
|
||||
'backstage.io/managed-by-location',
|
||||
entity,
|
||||
);
|
||||
default:
|
||||
throw new Error(`Invalid reference annotation ${type}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const getGitRepositoryTempFolder = async (
|
||||
repositoryUrl: string,
|
||||
): Promise<string> => {
|
||||
const parsedGitLocation = parseGitUrl(repositoryUrl);
|
||||
// removes .git from git location path
|
||||
parsedGitLocation.git_suffix = false;
|
||||
|
||||
if (!parsedGitLocation.ref) {
|
||||
parsedGitLocation.ref = await getDefaultBranch(
|
||||
parsedGitLocation.toString('https'),
|
||||
);
|
||||
}
|
||||
|
||||
return path.join(
|
||||
// fs.realpathSync fixes a problem with macOS returning a path that is a symlink
|
||||
fs.realpathSync(os.tmpdir()),
|
||||
'backstage-repo',
|
||||
parsedGitLocation.source,
|
||||
parsedGitLocation.owner,
|
||||
parsedGitLocation.name,
|
||||
parsedGitLocation.ref,
|
||||
);
|
||||
};
|
||||
|
||||
export const checkoutGitRepository = async (
|
||||
repoUrl: string,
|
||||
logger: Logger,
|
||||
): Promise<string> => {
|
||||
const parsedGitLocation = parseGitUrl(repoUrl);
|
||||
const repositoryTmpPath = await getGitRepositoryTempFolder(repoUrl);
|
||||
const token = await getTokenForGitRepo(repoUrl);
|
||||
|
||||
if (fs.existsSync(repositoryTmpPath)) {
|
||||
try {
|
||||
const repository = await Repository.open(repositoryTmpPath);
|
||||
const currentBranchName = (
|
||||
await repository.getCurrentBranch()
|
||||
).shorthand();
|
||||
await repository.fetch('origin');
|
||||
await repository.mergeBranches(
|
||||
currentBranchName,
|
||||
`origin/${currentBranchName}`,
|
||||
);
|
||||
return repositoryTmpPath;
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Found error "${e.message}" in cached repository "${repoUrl}" when getting latest changes. Removing cached repository.`,
|
||||
);
|
||||
fs.removeSync(repositoryTmpPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (token) {
|
||||
const type = getGitRepoType(repoUrl);
|
||||
switch (type) {
|
||||
case 'gitlab':
|
||||
// Personal Access Token
|
||||
parsedGitLocation.token = `dummyUsername:${token}`;
|
||||
parsedGitLocation.git_suffix = true;
|
||||
break;
|
||||
case 'github':
|
||||
parsedGitLocation.token = `${token}:x-oauth-basic`;
|
||||
break;
|
||||
default:
|
||||
parsedGitLocation.token = `:${token}`;
|
||||
}
|
||||
}
|
||||
|
||||
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
|
||||
|
||||
fs.mkdirSync(repositoryTmpPath, { recursive: true });
|
||||
await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath);
|
||||
|
||||
return repositoryTmpPath;
|
||||
};
|
||||
|
||||
export const getLastCommitTimestamp = async (
|
||||
repositoryUrl: string,
|
||||
logger: Logger,
|
||||
): Promise<number> => {
|
||||
const repositoryLocation = await checkoutGitRepository(repositoryUrl, logger);
|
||||
|
||||
const repository = await Repository.open(repositoryLocation);
|
||||
const commit = await repository.getReferenceCommit('HEAD');
|
||||
|
||||
return commit.date().getTime();
|
||||
};
|
||||
|
||||
export const getDocFilesFromRepository = async (
|
||||
reader: UrlReader,
|
||||
entity: Entity,
|
||||
): Promise<any> => {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
const response = await reader.readTree(target);
|
||||
|
||||
return await response.dir();
|
||||
};
|
||||
@@ -15,4 +15,3 @@
|
||||
*/
|
||||
|
||||
export * from './service/router';
|
||||
export * from './techdocs';
|
||||
|
||||
@@ -22,9 +22,10 @@ import {
|
||||
GeneratorBuilder,
|
||||
PreparerBase,
|
||||
GeneratorBase,
|
||||
} from '../techdocs';
|
||||
getLocationForEntity,
|
||||
getLastCommitTimestamp,
|
||||
} from '@backstage/techdocs-common';
|
||||
import { BuildMetadataStorage } from '../storage';
|
||||
import { getLocationForEntity, getLastCommitTimestamp } from '../helpers';
|
||||
|
||||
const getEntityId = (entity: Entity) => {
|
||||
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
|
||||
|
||||
@@ -20,14 +20,18 @@ import Knex from 'knex';
|
||||
import fetch from 'cross-fetch';
|
||||
import { Config } from '@backstage/config';
|
||||
import Docker from 'dockerode';
|
||||
import { GeneratorBuilder, PreparerBuilder, PublisherBase } from '../techdocs';
|
||||
import {
|
||||
GeneratorBuilder,
|
||||
PreparerBuilder,
|
||||
PublisherBase,
|
||||
getLocationForEntity,
|
||||
} from '@backstage/techdocs-common';
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
resolvePackagePath,
|
||||
} from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { DocsBuilder } from './helpers';
|
||||
import { getLocationForEntity } from '../helpers';
|
||||
|
||||
type RouterOptions = {
|
||||
preparers: PreparerBuilder;
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
Generators,
|
||||
TechdocsGenerator,
|
||||
Publisher,
|
||||
} from '../techdocs';
|
||||
} from '@backstage/techdocs-common';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
|
||||
export interface ServerOptions {
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export * from './stages';
|
||||
@@ -1,2 +0,0 @@
|
||||
site_name: Test site name
|
||||
site_description: Test site description
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
site_name: Test site name
|
||||
site_description: Test site description
|
||||
|
||||
repo_url: https://github.com/backstage/backstage
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Generators, TechdocsGenerator } from './';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const mockEntity = {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
},
|
||||
};
|
||||
|
||||
describe('generators', () => {
|
||||
it('should return error if no generator is registered', async () => {
|
||||
const generators = new Generators();
|
||||
|
||||
expect(() => generators.get(mockEntity)).toThrowError(
|
||||
'No generator registered for entity: "techdocs"',
|
||||
);
|
||||
});
|
||||
|
||||
it('should return correct registered generator', async () => {
|
||||
const generators = new Generators();
|
||||
const techdocs = new TechdocsGenerator(
|
||||
logger,
|
||||
ConfigReader.fromConfigs([]),
|
||||
);
|
||||
|
||||
generators.register('techdocs', techdocs);
|
||||
|
||||
expect(generators.get(mockEntity)).toBe(techdocs);
|
||||
});
|
||||
});
|
||||
@@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
GeneratorBase,
|
||||
SupportedGeneratorKey,
|
||||
GeneratorBuilder,
|
||||
} from './types';
|
||||
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { getGeneratorKey } from './helpers';
|
||||
|
||||
export class Generators implements GeneratorBuilder {
|
||||
private generatorMap = new Map<SupportedGeneratorKey, GeneratorBase>();
|
||||
|
||||
register(generatorKey: SupportedGeneratorKey, generator: GeneratorBase) {
|
||||
this.generatorMap.set(generatorKey, generator);
|
||||
}
|
||||
|
||||
get(entity: Entity): GeneratorBase {
|
||||
const generatorKey = getGeneratorKey(entity);
|
||||
const generator = this.generatorMap.get(generatorKey);
|
||||
|
||||
if (!generator) {
|
||||
throw new Error(`No generator registered for entity: "${generatorKey}"`);
|
||||
}
|
||||
|
||||
return generator;
|
||||
}
|
||||
}
|
||||
@@ -1,332 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import Stream, { PassThrough } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import mockFs from 'mock-fs';
|
||||
import * as winston from 'winston';
|
||||
import {
|
||||
runDockerContainer,
|
||||
getGeneratorKey,
|
||||
isValidRepoUrlForMkdocs,
|
||||
getRepoUrlFromLocationAnnotation,
|
||||
patchMkdocsYmlPreBuild,
|
||||
} from './helpers';
|
||||
import { RemoteProtocol } from '../prepare/types';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
|
||||
const mockEntity = {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
},
|
||||
};
|
||||
|
||||
const mockDocker = new Docker() as jest.Mocked<Docker>;
|
||||
|
||||
const mkdocsYml = fs.readFileSync(
|
||||
resolvePath(__filename, '../__fixtures__/mkdocs.yml'),
|
||||
);
|
||||
const mkdocsYmlWithRepoUrl = fs.readFileSync(
|
||||
resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'),
|
||||
);
|
||||
const mockLogger = winston.createLogger();
|
||||
|
||||
describe('helpers', () => {
|
||||
describe('getGeneratorKey', () => {
|
||||
it('should return techdocs as the only generator key', () => {
|
||||
const key = getGeneratorKey(mockEntity);
|
||||
expect(key).toBe('techdocs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runDockerContainer', () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(mockDocker, 'pull').mockImplementation((async (
|
||||
_image: string,
|
||||
_something: any,
|
||||
handler: (err: Error | undefined, stream: PassThrough) => void,
|
||||
) => {
|
||||
const mockStream = new PassThrough();
|
||||
handler(undefined, mockStream);
|
||||
mockStream.end();
|
||||
}) as any);
|
||||
|
||||
jest
|
||||
.spyOn(mockDocker, 'run')
|
||||
.mockResolvedValue([{ Error: null, StatusCode: 0 }]);
|
||||
|
||||
jest
|
||||
.spyOn(mockDocker, 'ping')
|
||||
.mockResolvedValue(Buffer.from('OK', 'utf-8'));
|
||||
});
|
||||
|
||||
const imageName = 'spotify/techdocs';
|
||||
const args = ['build', '-d', '/result'];
|
||||
const docsDir = os.tmpdir();
|
||||
const resultDir = os.tmpdir();
|
||||
|
||||
it('should pull the techdocs docker container', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.pull).toHaveBeenCalledWith(
|
||||
imageName,
|
||||
{},
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it('should run the techdocs docker container', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.run).toHaveBeenCalledWith(
|
||||
imageName,
|
||||
args,
|
||||
expect.any(Stream),
|
||||
{
|
||||
Volumes: {
|
||||
'/content': {},
|
||||
'/result': {},
|
||||
},
|
||||
WorkingDir: '/content',
|
||||
HostConfig: {
|
||||
Binds: [`${docsDir}:/content`, `${resultDir}:/result`],
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('should ping docker to test availability', async () => {
|
||||
await runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
});
|
||||
|
||||
expect(mockDocker.ping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('where docker is unavailable', () => {
|
||||
const dockerError = 'a docker error';
|
||||
|
||||
beforeEach(() => {
|
||||
jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => {
|
||||
throw new Error(dockerError);
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw with a descriptive error message including the docker error message', async () => {
|
||||
await expect(
|
||||
runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient: mockDocker,
|
||||
}),
|
||||
).rejects.toThrow(new RegExp(`.+: ${dockerError}`));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidRepoUrlForMkdocs', () => {
|
||||
it('should return true for valid repo_url values for mkdocs', () => {
|
||||
const validRepoUrls = [
|
||||
'https://github.com/org/repo',
|
||||
'https://github.com/backstage/backstage/',
|
||||
'https://github.com/org123/repo1-2-3/',
|
||||
'http://github.com/insecureOrg/insecureRepo',
|
||||
'https://gitlab.com/org/repo',
|
||||
'https://gitlab.com/backstage/backstage/',
|
||||
'https://gitlab.com/org123/repo1-2-3/',
|
||||
'http://gitlab.com/insecureOrg/insecureRepo',
|
||||
];
|
||||
|
||||
const validRemoteProtocols = ['github', 'gitlab'];
|
||||
|
||||
validRepoUrls.forEach(url => {
|
||||
validRemoteProtocols.forEach(targetType => {
|
||||
expect(
|
||||
isValidRepoUrlForMkdocs(url, targetType as RemoteProtocol),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should return false for invalid repo_urls values for mkdocs', () => {
|
||||
const invalidRepoUrls = [
|
||||
'git@github.com:org/repo',
|
||||
'https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend',
|
||||
];
|
||||
|
||||
invalidRepoUrls.forEach(url => {
|
||||
expect(isValidRepoUrlForMkdocs(url, 'github')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it('should return false for unsupported remote protocols', () => {
|
||||
const validRepoUrl = 'https://github.com/backstage/backstage';
|
||||
|
||||
const unsupportedRemoteProtocols = ['dir', 'file', 'url'];
|
||||
|
||||
unsupportedRemoteProtocols.forEach(targetType => {
|
||||
expect(
|
||||
isValidRepoUrlForMkdocs(validRepoUrl, targetType as RemoteProtocol),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRepoUrlFromLocationAnnotation', () => {
|
||||
it('should return undefined for unsupported location type', () => {
|
||||
const parsedLocationAnnotation1: ParsedLocationAnnotation = {
|
||||
type: 'dir',
|
||||
target: '/home/user/workspace/docs-repository',
|
||||
};
|
||||
|
||||
const parsedLocationAnnotation2: ParsedLocationAnnotation = {
|
||||
type: 'file',
|
||||
target: '/home/user/workspace/docs-repository/catalog-info.yaml',
|
||||
};
|
||||
|
||||
const parsedLocationAnnotation3: ParsedLocationAnnotation = {
|
||||
type: 'url',
|
||||
target: 'https://my-website.com/storage/this/docs/repository',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
|
||||
undefined,
|
||||
);
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return correct target url for supported hosts', () => {
|
||||
const parsedLocationAnnotation1: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/backstage/backstage.git',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe(
|
||||
'https://github.com/backstage/backstage',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation2: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/org/repo',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe(
|
||||
'https://github.com/org/repo',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation3: ParsedLocationAnnotation = {
|
||||
type: 'gitlab',
|
||||
target: 'https://gitlab.com/org/repo',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe(
|
||||
'https://gitlab.com/org/repo',
|
||||
);
|
||||
|
||||
const parsedLocationAnnotation4: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target:
|
||||
'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
};
|
||||
|
||||
expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation4)).toBe(
|
||||
'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pathMkdocsPreBuild', () => {
|
||||
beforeEach(() => {
|
||||
mockFs({
|
||||
'/mkdocs.yml': mkdocsYml,
|
||||
'/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mockFs.restore();
|
||||
});
|
||||
|
||||
it('should add repo_url to mkdocs.yml', async () => {
|
||||
const parsedLocationAnnotation: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/backstage/backstage',
|
||||
};
|
||||
|
||||
await patchMkdocsYmlPreBuild(
|
||||
'/mkdocs.yml',
|
||||
mockLogger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
const updatedMkdocsYml = await fs.readFile('/mkdocs.yml');
|
||||
|
||||
expect(updatedMkdocsYml.toString()).toContain(
|
||||
"repo_url: 'https://github.com/backstage/backstage'",
|
||||
);
|
||||
});
|
||||
|
||||
it('should not override existing repo_url in mkdocs.yml', async () => {
|
||||
const parsedLocationAnnotation: ParsedLocationAnnotation = {
|
||||
type: 'github',
|
||||
target: 'https://github.com/neworg/newrepo',
|
||||
};
|
||||
|
||||
await patchMkdocsYmlPreBuild(
|
||||
'/mkdocs_with_repo_url.yml',
|
||||
mockLogger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
const updatedMkdocsYml = await fs.readFile('/mkdocs_with_repo_url.yml');
|
||||
|
||||
expect(updatedMkdocsYml.toString()).toContain(
|
||||
"repo_url: 'https://github.com/backstage/backstage'",
|
||||
);
|
||||
expect(updatedMkdocsYml.toString()).not.toContain(
|
||||
"repo_url: 'https://github.com/neworg/newrepo'",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,277 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import { spawn } from 'child_process';
|
||||
import { Writable, PassThrough } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import yaml from 'js-yaml';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { SupportedGeneratorKey } from './types';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
import { RemoteProtocol } from '../prepare/types';
|
||||
|
||||
// TODO: Implement proper support for more generators.
|
||||
export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
|
||||
if (!entity) {
|
||||
throw new Error('No entity provided');
|
||||
}
|
||||
|
||||
return 'techdocs';
|
||||
}
|
||||
|
||||
type RunDockerContainerOptions = {
|
||||
imageName: string;
|
||||
args: string[];
|
||||
logStream?: Writable;
|
||||
docsDir: string;
|
||||
resultDir: string;
|
||||
dockerClient: Docker;
|
||||
createOptions?: Docker.ContainerCreateOptions;
|
||||
};
|
||||
|
||||
export type RunCommandOptions = {
|
||||
command: string;
|
||||
args: string[];
|
||||
options: object;
|
||||
logStream?: Writable;
|
||||
};
|
||||
|
||||
export async function runDockerContainer({
|
||||
imageName,
|
||||
args,
|
||||
logStream = new PassThrough(),
|
||||
docsDir,
|
||||
resultDir,
|
||||
dockerClient,
|
||||
createOptions,
|
||||
}: RunDockerContainerOptions) {
|
||||
try {
|
||||
await dockerClient.ping();
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
dockerClient.pull(imageName, {}, (err, stream) => {
|
||||
if (err) return reject(err);
|
||||
stream.pipe(logStream, { end: false });
|
||||
stream.on('end', () => resolve());
|
||||
stream.on('error', (error: Error) => reject(error));
|
||||
return undefined;
|
||||
});
|
||||
});
|
||||
|
||||
const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
|
||||
imageName,
|
||||
args,
|
||||
logStream,
|
||||
{
|
||||
Volumes: {
|
||||
'/content': {},
|
||||
'/result': {},
|
||||
},
|
||||
WorkingDir: '/content',
|
||||
HostConfig: {
|
||||
Binds: [`${docsDir}:/content`, `${resultDir}:/result`],
|
||||
},
|
||||
...createOptions,
|
||||
},
|
||||
);
|
||||
|
||||
if (error) {
|
||||
throw new Error(
|
||||
`Docker failed to run with the following error message: ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (statusCode !== 0) {
|
||||
throw new Error(
|
||||
`Docker container returned a non-zero exit code (${statusCode})`,
|
||||
);
|
||||
}
|
||||
|
||||
return { error, statusCode };
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param options the options object
|
||||
* @param options.command the command to run
|
||||
* @param options.args the arguments to pass the command
|
||||
* @param options.options options used in spawn
|
||||
* @param options.logStream the log streamer to capture log messages
|
||||
*/
|
||||
export const runCommand = async ({
|
||||
command,
|
||||
args,
|
||||
options,
|
||||
logStream = new PassThrough(),
|
||||
}: RunCommandOptions) => {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const process = spawn(command, args, options);
|
||||
|
||||
process.stdout.on('data', stream => {
|
||||
logStream.write(stream);
|
||||
});
|
||||
|
||||
process.stderr.on('data', stream => {
|
||||
logStream.write(stream);
|
||||
});
|
||||
|
||||
process.on('error', error => {
|
||||
return reject(error);
|
||||
});
|
||||
|
||||
process.on('close', code => {
|
||||
if (code !== 0) {
|
||||
return reject(`Command ${command} failed, exit code: ${code}`);
|
||||
}
|
||||
return resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Return true if mkdocs can compile docs with provided repo_url
|
||||
*
|
||||
* Valid repo_url examples in mkdocs.yml
|
||||
* - https://github.com/backstage/backstage
|
||||
* - https://gitlab.com/org/repo/
|
||||
* - http://github.com/backstage/backstage
|
||||
* - A http(s) protocol URL to the root of the repository
|
||||
*
|
||||
* Invalid repo_url examples in mkdocs.yml
|
||||
* - https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component
|
||||
* - (anything that is not valid as described above)
|
||||
*
|
||||
* @param {string} repoUrl URL supposed to be used as repo_url in mkdocs.yml
|
||||
* @param {RemoteProtocol} locationType Type of source code host - github, gitlab, dir, url, etc.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export const isValidRepoUrlForMkdocs = (
|
||||
repoUrl: string,
|
||||
locationType: RemoteProtocol,
|
||||
): boolean => {
|
||||
// Trim trailing slash
|
||||
const cleanRepoUrl = repoUrl.replace(/\/$/, '');
|
||||
|
||||
if (locationType === 'github' || locationType === 'gitlab') {
|
||||
// A valid repoUrl to the root of the repository will be split into 5 strings if split using the / delimiter.
|
||||
// We do not want URLs which have more than that number of forward slashes since they will signify a non-root location
|
||||
// Note: This is not the best possible implementation but will work most of the times.. Feel free to improve or
|
||||
// highlight edge cases.
|
||||
return cleanRepoUrl.split('/').length === 5;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a valid URL of the repository used in backstage.io/techdocs-ref annotation.
|
||||
* Return undefined if the `target` is not valid in context of repo_url in mkdocs.yml
|
||||
* Alter URL so that it is a valid repo_url config in mkdocs.yml
|
||||
*
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
|
||||
* @returns {string | undefined}
|
||||
*/
|
||||
export const getRepoUrlFromLocationAnnotation = (
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation,
|
||||
): string | undefined => {
|
||||
const { type: locationType, target } = parsedLocationAnnotation;
|
||||
|
||||
// Add more options from the RemoteProtocol type of parsedLocationAnnotation.type here
|
||||
// when TechDocs supports more hosts and if mkdocs can generated an Edit URL for them.
|
||||
const supportedHosts = ['github', 'gitlab'];
|
||||
|
||||
if (supportedHosts.includes(locationType)) {
|
||||
// Trim .git or .git/ from the end of repository url
|
||||
return target.replace(/.git\/*$/, '');
|
||||
}
|
||||
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the mkdocs.yml file before TechDocs generator uses it to build docs site.
|
||||
*
|
||||
* List of tasks:
|
||||
* - Add repo_url if it does not exists
|
||||
* If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default.
|
||||
* If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get
|
||||
* the repository URL.
|
||||
*
|
||||
* This function will not throw an error since this is not critical to the whole TechDocs pipeline.
|
||||
* Instead it will log warnings if there are any errors in reading, parsing or writing YAML.
|
||||
*
|
||||
* @param {string} mkdocsYmlPath Absolute path to mkdocs.yml or equivalent of a docs site
|
||||
* @param {Logger} logger
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type
|
||||
*/
|
||||
export const patchMkdocsYmlPreBuild = async (
|
||||
mkdocsYmlPath: string,
|
||||
logger: Logger,
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation,
|
||||
) => {
|
||||
let mkdocsYmlFileString;
|
||||
try {
|
||||
mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8');
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Could not read file ${mkdocsYmlPath} before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mkdocsYml: any;
|
||||
try {
|
||||
mkdocsYml = yaml.safeLoad(mkdocsYmlFileString);
|
||||
|
||||
// mkdocsYml should be an object type after successful parsing.
|
||||
// But based on its type definition, it can also be a string or undefined, which we don't want.
|
||||
if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') {
|
||||
throw new Error('Bad YAML format.');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add repo_url to mkdocs.yml if it is missing. This will enable the Page edit button generated by MkDocs.
|
||||
if (!('repo_url' in mkdocsYml)) {
|
||||
const repoUrl = getRepoUrlFromLocationAnnotation(parsedLocationAnnotation);
|
||||
if (repoUrl !== undefined) {
|
||||
// mkdocs.yml will not build with invalid repo_url. So, make sure it is valid.
|
||||
if (isValidRepoUrlForMkdocs(repoUrl, parsedLocationAnnotation.type)) {
|
||||
mkdocsYml.repo_url = repoUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.writeFile(mkdocsYmlPath, yaml.safeDump(mkdocsYml), 'utf8');
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -1,18 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { TechdocsGenerator } from './techdocs';
|
||||
export { Generators } from './generators';
|
||||
export type { GeneratorBuilder, GeneratorBase } from './types';
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { Logger } from 'winston';
|
||||
import { PassThrough } from 'stream';
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
import {
|
||||
GeneratorBase,
|
||||
GeneratorRunOptions,
|
||||
GeneratorRunResult,
|
||||
} from './types';
|
||||
import {
|
||||
runDockerContainer,
|
||||
runCommand,
|
||||
patchMkdocsYmlPreBuild,
|
||||
} from './helpers';
|
||||
|
||||
type TechdocsGeneratorOptions = {
|
||||
// This option enables users to configure if they want to use TechDocs container
|
||||
// or generate without the container.
|
||||
// This is used to avoid running into Docker in Docker environment.
|
||||
runGeneratorIn: string;
|
||||
};
|
||||
|
||||
const createStream = (): [string[], PassThrough] => {
|
||||
const log = [] as Array<string>;
|
||||
|
||||
const stream = new PassThrough();
|
||||
stream.on('data', chunk => {
|
||||
const textValue = chunk.toString().trim();
|
||||
if (textValue?.length > 1) log.push(textValue);
|
||||
});
|
||||
|
||||
return [log, stream];
|
||||
};
|
||||
|
||||
export class TechdocsGenerator implements GeneratorBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly options: TechdocsGeneratorOptions;
|
||||
|
||||
constructor(logger: Logger, config: Config) {
|
||||
this.logger = logger;
|
||||
this.options = {
|
||||
runGeneratorIn:
|
||||
config.getOptionalString('techdocs.generators.techdocs') ?? 'docker',
|
||||
};
|
||||
}
|
||||
|
||||
public async run({
|
||||
directory,
|
||||
dockerClient,
|
||||
parsedLocationAnnotation,
|
||||
}: GeneratorRunOptions): Promise<GeneratorRunResult> {
|
||||
const tmpdirPath = os.tmpdir();
|
||||
// Fixes a problem with macOS returning a path that is a symlink
|
||||
const tmpdirResolvedPath = fs.realpathSync(tmpdirPath);
|
||||
const resultDir = fs.mkdtempSync(
|
||||
path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
|
||||
);
|
||||
const [log, logStream] = createStream();
|
||||
|
||||
// TODO: In future mkdocs.yml can be mkdocs.yaml. So, use a config variable here to find out
|
||||
// the correct file name.
|
||||
// Do some updates to mkdocs.yml before generating docs e.g. adding repo_url
|
||||
await patchMkdocsYmlPreBuild(
|
||||
path.join(directory, 'mkdocs.yml'),
|
||||
this.logger,
|
||||
parsedLocationAnnotation,
|
||||
);
|
||||
|
||||
try {
|
||||
switch (this.options.runGeneratorIn) {
|
||||
case 'local':
|
||||
await runCommand({
|
||||
command: 'mkdocs',
|
||||
args: ['build', '-d', resultDir, '-v'],
|
||||
options: {
|
||||
cwd: directory,
|
||||
},
|
||||
logStream,
|
||||
});
|
||||
this.logger.info(
|
||||
`Successfully generated docs from ${directory} into ${resultDir} using local mkdocs`,
|
||||
);
|
||||
break;
|
||||
case 'docker':
|
||||
await runDockerContainer({
|
||||
imageName: 'spotify/techdocs',
|
||||
args: ['build', '-d', '/result'],
|
||||
logStream,
|
||||
docsDir: directory,
|
||||
resultDir,
|
||||
dockerClient,
|
||||
});
|
||||
this.logger.info(
|
||||
`Successfully generated docs from ${directory} into ${resultDir} using techdocs-container`,
|
||||
);
|
||||
break;
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.debug(
|
||||
`Failed to generate docs from ${directory} into ${resultDir}`,
|
||||
);
|
||||
this.logger.debug(`Build failed with error: ${log}`);
|
||||
throw new Error(
|
||||
`Failed to generate docs from ${directory} into ${resultDir} with error ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
return { resultDir };
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Writable } from 'stream';
|
||||
import Docker from 'dockerode';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ParsedLocationAnnotation } from '../../../helpers';
|
||||
|
||||
/**
|
||||
* The returned directory from the generator which is ready
|
||||
* to pass to the next stage of the TechDocs which is publishing
|
||||
*/
|
||||
export type GeneratorRunResult = {
|
||||
resultDir: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The values that the generator will receive.
|
||||
*
|
||||
* @param {string} directory The directory of the uncompiled documentation, with the values from the frontend
|
||||
* @param {Docker} dockerClient A docker client to run any generator on top of your directory
|
||||
* @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity
|
||||
* @param {Writable} [logStream] A dedicated log stream
|
||||
*/
|
||||
export type GeneratorRunOptions = {
|
||||
directory: string;
|
||||
dockerClient: Docker;
|
||||
parsedLocationAnnotation: ParsedLocationAnnotation;
|
||||
logStream?: Writable;
|
||||
};
|
||||
|
||||
export type GeneratorBase = {
|
||||
// runs the generator with the values and returns the directory to be published
|
||||
run(opts: GeneratorRunOptions): Promise<GeneratorRunResult>;
|
||||
};
|
||||
|
||||
/**
|
||||
* List of supported generator options
|
||||
*/
|
||||
export type SupportedGeneratorKey = 'techdocs' | string;
|
||||
|
||||
/**
|
||||
* The generator builder holds the generator ready for run time
|
||||
*/
|
||||
export type GeneratorBuilder = {
|
||||
register(protocol: SupportedGeneratorKey, generator: GeneratorBase): void;
|
||||
get(entity: Entity): GeneratorBase;
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export * from './generate';
|
||||
export * from './prepare';
|
||||
export * from './publish';
|
||||
@@ -1,93 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { CommonGitPreparer } from './commonGit';
|
||||
import { checkoutGitRepository } from '../../../helpers';
|
||||
|
||||
function normalizePath(path: string) {
|
||||
return path
|
||||
.replace(/^[a-z]:/i, '')
|
||||
.split('\\')
|
||||
.join('/');
|
||||
}
|
||||
|
||||
jest.mock('../../../helpers', () => ({
|
||||
...jest.requireActual<{}>('../../../helpers'),
|
||||
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
|
||||
}));
|
||||
|
||||
const createMockEntity = (annotations = {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'test-component-name',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('commonGit preparer', () => {
|
||||
it('should prepare temp docs path from github repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'github:https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component',
|
||||
);
|
||||
});
|
||||
|
||||
it('should prepare temp docs path from gitlab repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(2);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/catalog-info.yaml',
|
||||
);
|
||||
});
|
||||
|
||||
it('should prepare temp docs path from azure repo', async () => {
|
||||
const preparer = new CommonGitPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/techdocs-ref':
|
||||
'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
|
||||
});
|
||||
|
||||
const tempDocsPath = await preparer.prepare(mockEntity);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(3);
|
||||
expect(normalizePath(tempDocsPath)).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/template.yaml',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,50 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PreparerBase } from './types';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import {
|
||||
parseReferenceAnnotation,
|
||||
checkoutGitRepository,
|
||||
} from '../../../helpers';
|
||||
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export class CommonGitPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
try {
|
||||
const repoPath = await checkoutGitRepository(target, this.logger);
|
||||
const parsedGitLocation = parseGitUrl(target);
|
||||
|
||||
return path.join(repoPath, parsedGitLocation.filepath);
|
||||
} catch (error) {
|
||||
this.logger.debug(`Repo checkout failed with error ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { DirectoryPreparer } from './dir';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { checkoutGitRepository } from '../../../helpers';
|
||||
|
||||
function normalizePath(path: string) {
|
||||
return path
|
||||
.replace(/^[a-z]:/i, '')
|
||||
.split('\\')
|
||||
.join('/');
|
||||
}
|
||||
|
||||
jest.mock('../../../helpers', () => ({
|
||||
...jest.requireActual<{}>('../../../helpers'),
|
||||
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
|
||||
}));
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const createMockEntity = (annotations: {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'testName',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
describe('directory preparer', () => {
|
||||
it('should merge managed-by-location and techdocs-ref when techdocs-ref is relative', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'file:/directory/documented-component.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:./our-documentation',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/directory/our-documentation',
|
||||
);
|
||||
});
|
||||
|
||||
it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'file:/directory/documented-component.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/our-documentation/techdocs',
|
||||
);
|
||||
});
|
||||
|
||||
it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
|
||||
const directoryPreparer = new DirectoryPreparer(logger);
|
||||
|
||||
const mockEntity = createMockEntity({
|
||||
'backstage.io/managed-by-location':
|
||||
'github:https://github.com/backstage/backstage/blob/master/catalog-info.yaml',
|
||||
'backstage.io/techdocs-ref': 'dir:./docs',
|
||||
});
|
||||
|
||||
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
|
||||
'/tmp/backstage-repo/org/name/branch/docs',
|
||||
);
|
||||
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { PreparerBase } from './types';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import path from 'path';
|
||||
import {
|
||||
parseReferenceAnnotation,
|
||||
checkoutGitRepository,
|
||||
} from '../../../helpers';
|
||||
import { InputError } from '@backstage/backend-common';
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export class DirectoryPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
private async resolveManagedByLocationToDir(entity: Entity) {
|
||||
const { type, target } = parseReferenceAnnotation(
|
||||
'backstage.io/managed-by-location',
|
||||
entity,
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Building docs for entity with type 'dir' and managed-by-location '${type}'`,
|
||||
);
|
||||
switch (type) {
|
||||
case 'github':
|
||||
case 'gitlab':
|
||||
case 'azure/api': {
|
||||
const parsedGitLocation = parseGitUrl(target);
|
||||
const repoLocation = await checkoutGitRepository(target, this.logger);
|
||||
|
||||
return path.dirname(
|
||||
path.join(repoLocation, parsedGitLocation.filepath),
|
||||
);
|
||||
}
|
||||
|
||||
case 'file':
|
||||
return path.dirname(target);
|
||||
default:
|
||||
throw new InputError(`Unable to resolve location type ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
const { target } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
|
||||
const managedByLocationDirectory = await this.resolveManagedByLocationToDir(
|
||||
entity,
|
||||
);
|
||||
|
||||
return new Promise(resolve => {
|
||||
resolve(path.resolve(managedByLocationDirectory, target));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { DirectoryPreparer } from './dir';
|
||||
export { CommonGitPreparer } from './commonGit';
|
||||
export { UrlPreparer } from './url';
|
||||
export { Preparers } from './preparers';
|
||||
export type { PreparerBuilder, PreparerBase } from './types';
|
||||
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { parseReferenceAnnotation } from '../../../helpers';
|
||||
|
||||
export class Preparers implements PreparerBuilder {
|
||||
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
|
||||
|
||||
register(protocol: RemoteProtocol, preparer: PreparerBase) {
|
||||
this.preparerMap.set(protocol, preparer);
|
||||
}
|
||||
|
||||
get(entity: Entity): PreparerBase {
|
||||
const { type } = parseReferenceAnnotation(
|
||||
'backstage.io/techdocs-ref',
|
||||
entity,
|
||||
);
|
||||
const preparer = this.preparerMap.get(type);
|
||||
|
||||
if (!preparer) {
|
||||
throw new Error(`No preparer registered for type: "${type}"`);
|
||||
}
|
||||
|
||||
return preparer;
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import type { Entity } from '@backstage/catalog-model';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
export type PreparerBase = {
|
||||
/**
|
||||
* Given an Entity definition from the Service Catalog, go and prepare a directory
|
||||
* with contents from the location in temporary storage and return the path
|
||||
* @param entity The entity from the Service Catalog
|
||||
*/
|
||||
prepare(entity: Entity, opts?: { logger: Logger }): Promise<string>;
|
||||
};
|
||||
|
||||
export type PreparerBuilder = {
|
||||
register(protocol: RemoteProtocol, preparer: PreparerBase): void;
|
||||
get(entity: Entity): PreparerBase;
|
||||
};
|
||||
|
||||
export type RemoteProtocol =
|
||||
| 'dir'
|
||||
| 'github'
|
||||
| 'gitlab'
|
||||
| 'file'
|
||||
| 'azure/api'
|
||||
| 'url';
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PreparerBase } from './types';
|
||||
import { getDocFilesFromRepository } from '../../../helpers';
|
||||
|
||||
import { Logger } from 'winston';
|
||||
import { UrlReader } from '@backstage/backend-common';
|
||||
|
||||
export class UrlPreparer implements PreparerBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly reader: UrlReader;
|
||||
|
||||
constructor(reader: UrlReader, logger: Logger) {
|
||||
this.logger = logger;
|
||||
this.reader = reader;
|
||||
}
|
||||
|
||||
async prepare(entity: Entity): Promise<string> {
|
||||
try {
|
||||
return getDocFilesFromRepository(this.reader, entity);
|
||||
} catch (error) {
|
||||
this.logger.debug(
|
||||
`Unable to fetch files for building docs ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { Publisher } from './publish';
|
||||
export type { PublisherBase } from './publish';
|
||||
export type { PublisherType } from './types';
|
||||
@@ -1,75 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-restricted-syntax */
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
import {
|
||||
getVoidLogger,
|
||||
PluginEndpointDiscovery,
|
||||
} from '@backstage/backend-common';
|
||||
import { LocalPublish } from './local';
|
||||
|
||||
const createMockEntity = (annotations = {}) => {
|
||||
return {
|
||||
apiVersion: 'version',
|
||||
kind: 'TestKind',
|
||||
metadata: {
|
||||
name: 'test-component-name',
|
||||
annotations: {
|
||||
...annotations,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('local publisher', () => {
|
||||
it('should publish generated documentation dir', async () => {
|
||||
const testDiscovery: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7000'),
|
||||
getExternalBaseUrl: jest.fn(),
|
||||
};
|
||||
|
||||
const publisher = new LocalPublish(logger, testDiscovery);
|
||||
|
||||
const mockEntity = createMockEntity();
|
||||
|
||||
const tempDir = fs.mkdtempSync(`${__dirname}/test-component-folder-`);
|
||||
|
||||
expect(tempDir).toBeTruthy();
|
||||
|
||||
fs.closeSync(fs.openSync(path.join(tempDir, '/mock-file'), 'w'));
|
||||
|
||||
await publisher.publish({ entity: mockEntity, directory: tempDir });
|
||||
const publishDir = path.resolve(
|
||||
__dirname,
|
||||
`../../../../static/docs/${mockEntity.metadata.name}`,
|
||||
);
|
||||
|
||||
const resultDir = path.resolve(
|
||||
__dirname,
|
||||
`../../../../static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`,
|
||||
);
|
||||
|
||||
expect(fs.existsSync(resultDir)).toBeTruthy();
|
||||
expect(fs.existsSync(path.join(resultDir, '/mock-file'))).toBeTruthy();
|
||||
|
||||
fs.removeSync(publishDir);
|
||||
fs.removeSync(tempDir);
|
||||
});
|
||||
});
|
||||
@@ -1,97 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import fs from 'fs-extra';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import {
|
||||
resolvePackagePath,
|
||||
PluginEndpointDiscovery,
|
||||
} from '@backstage/backend-common';
|
||||
|
||||
export type LocalPublishParams = {
|
||||
entity: Entity;
|
||||
directory: string;
|
||||
};
|
||||
|
||||
export type LocalPublishReturn =
|
||||
| Promise<{ remoteUrl: string }>
|
||||
| { remoteUrl: string };
|
||||
|
||||
/**
|
||||
* Type for the local publisher which uses local filesystem to store the generated static files.
|
||||
*
|
||||
* It uses a directory called "static" at the root of techdocs-backend plugin.
|
||||
*/
|
||||
export interface LocalPublisher {
|
||||
/**
|
||||
* Store the generated files inside a static folder in local filesystem.
|
||||
*
|
||||
* @param {LocalPublishParams} opts Object containing the entity from the service
|
||||
* catalog, and the directory that contains the generated static files from TechDocs.
|
||||
* @returns {LocalPublishReturn} Either a promise or an object with `remoteUrl` which is the URL
|
||||
* which serves files from the local publisher's static directory.
|
||||
*/
|
||||
publish(opts: LocalPublishParams): LocalPublishReturn;
|
||||
}
|
||||
|
||||
export class LocalPublish {
|
||||
private readonly logger: Logger;
|
||||
private readonly discovery: PluginEndpointDiscovery;
|
||||
|
||||
constructor(logger: Logger, discovery: PluginEndpointDiscovery) {
|
||||
this.logger = logger;
|
||||
this.discovery = discovery;
|
||||
}
|
||||
|
||||
publish({ entity, directory }: LocalPublishParams): LocalPublishReturn {
|
||||
const entityNamespace = entity.metadata.namespace ?? 'default';
|
||||
|
||||
const publishDir = resolvePackagePath(
|
||||
'@backstage/plugin-techdocs-backend',
|
||||
'static/docs',
|
||||
entityNamespace,
|
||||
entity.kind,
|
||||
entity.metadata.name,
|
||||
);
|
||||
|
||||
if (!fs.existsSync(publishDir)) {
|
||||
this.logger.info(`Could not find ${publishDir}, creating the directory.`);
|
||||
fs.mkdirSync(publishDir, { recursive: true });
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.copy(directory, publishDir, err => {
|
||||
if (err) {
|
||||
this.logger.debug(
|
||||
`Failed to copy docs from ${directory} to ${publishDir}`,
|
||||
);
|
||||
reject(err);
|
||||
}
|
||||
|
||||
this.discovery
|
||||
.getBaseUrl('techdocs')
|
||||
.then(techdocsApiUrl => {
|
||||
resolve({
|
||||
remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
|
||||
});
|
||||
})
|
||||
.catch(reason => {
|
||||
reject(reason);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Logger } from 'winston';
|
||||
import { Config } from '@backstage/config';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { PublisherType } from './types';
|
||||
import { LocalPublish } from './local';
|
||||
|
||||
export type PublisherBaseParams = {
|
||||
entity: Entity;
|
||||
directory: string;
|
||||
};
|
||||
|
||||
export type PublisherBaseReturn =
|
||||
| Promise<{ remoteUrl: string }>
|
||||
| { remoteUrl: string };
|
||||
|
||||
/**
|
||||
* Type for the publisher instance registered with backend which manages publishing of the
|
||||
* generated static files after the prepare and generate steps of TechDocs.
|
||||
*
|
||||
* Depending upon the value of techdocs.publisher.type in app config, this instance creates
|
||||
* and uses a publisher of the particular type i.e. local, google_gcs, aws_s3, etc. It defaults
|
||||
* to use the local publisher.
|
||||
*/
|
||||
export interface PublisherBase {
|
||||
/**
|
||||
* Invoke the app's configured publisher's publish method.
|
||||
*
|
||||
* @param {PublisherBaseParams} opts Object containing the entity from the service
|
||||
* catalog, and the directory that contains the generated static files from TechDocs.
|
||||
*/
|
||||
publish(opts: PublisherBaseParams): PublisherBaseReturn;
|
||||
|
||||
/**
|
||||
* Return true if local filesystem is being used to store generated files for TechDocs.
|
||||
*/
|
||||
isLocalPublisher(): boolean;
|
||||
|
||||
/**
|
||||
* Return true if an external cloud storage (GCS, S3, SFTP server, etc.) is being used to
|
||||
* store generated files for TechDocs.
|
||||
*/
|
||||
isExternalPublisher(): boolean;
|
||||
}
|
||||
|
||||
export class Publisher implements PublisherBase {
|
||||
private readonly logger: Logger;
|
||||
private readonly config: Config;
|
||||
private readonly discovery: PluginEndpointDiscovery;
|
||||
private readonly publisherType: PublisherType;
|
||||
private readonly publisher: any;
|
||||
|
||||
constructor(
|
||||
logger: Logger,
|
||||
config: Config,
|
||||
discovery: PluginEndpointDiscovery,
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.config = config;
|
||||
this.discovery = discovery;
|
||||
|
||||
this.publisherType =
|
||||
(this.config.getOptionalString(
|
||||
'techdocs.publisher.type',
|
||||
) as PublisherType) ?? 'local';
|
||||
|
||||
switch (this.publisherType) {
|
||||
case 'google_gcs':
|
||||
this.logger.info(
|
||||
'Creating Google Storage Bucket publisher for TechDocs',
|
||||
);
|
||||
this.publisher = new LocalPublish(this.logger, this.discovery);
|
||||
break;
|
||||
case 'local':
|
||||
this.logger.info('Creating Local publisher for TechDocs');
|
||||
this.publisher = new LocalPublish(this.logger, this.discovery);
|
||||
break;
|
||||
default:
|
||||
this.logger.info('Creating Local publisher for TechDocs');
|
||||
this.publisher = new LocalPublish(this.logger, this.discovery);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
publish({ entity, directory }: PublisherBaseParams): PublisherBaseReturn {
|
||||
return this.publisher.publish({ entity, directory });
|
||||
}
|
||||
|
||||
isLocalPublisher(): boolean {
|
||||
return this.publisherType === 'local';
|
||||
}
|
||||
|
||||
isExternalPublisher(): boolean {
|
||||
return this.publisherType !== 'local';
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Key for all the different types of TechDocs publishers available.
|
||||
*/
|
||||
export type PublisherType = 'local' | 'google_gcs';
|
||||
|
||||
export interface ExternalPublisher {}
|
||||
Reference in New Issue
Block a user