diff --git a/.changeset/eleven-lobsters-train.md b/.changeset/eleven-lobsters-train.md new file mode 100644 index 0000000000..95b9145ee6 --- /dev/null +++ b/.changeset/eleven-lobsters-train.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-techdocs': minor +'@backstage/plugin-techdocs-backend': minor +--- + +- Use techdocs annotation to add repo_url if missing in mkdocs.yml. Having repo_url creates a Edit button on techdocs pages. +- techdocs-backend: API endpoint `/metadata/mkdocs/*` renamed to `/metadata/techdocs/*` diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 1d9f3ea7c1..a09adb7380 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -32,7 +32,9 @@ "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", + "js-yaml": "^3.14.0", "knex": "^0.21.6", + "mock-fs": "^4.13.0", "nodegit": "^0.27.0", "winston": "^3.2.1" }, diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 3017a68461..4a736f7388 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -69,10 +69,13 @@ export class DocsBuilder { this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`); const preparedDir = await this.preparer.prepare(this.entity); + const parsedLocationAnnotation = getLocationForEntity(this.entity); + this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`); const { resultDir } = await this.generator.run({ directory: preparedDir, dockerClient: this.dockerClient, + parsedLocationAnnotation, }); this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`); diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml b/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml new file mode 100644 index 0000000000..1d1186840f --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml @@ -0,0 +1,2 @@ +site_name: Test site name +site_description: Test site description diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml b/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml new file mode 100644 index 0000000000..37b004bfe4 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml @@ -0,0 +1,4 @@ +site_name: Test site name +site_description: Test site description + +repo_url: https://github.com/backstage/backstage diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts index 283560f0ce..4938739fdf 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts @@ -13,10 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import Stream, { PassThrough } from 'stream'; +import fs from 'fs'; import os from 'os'; +import { resolve as resolvePath } from 'path'; +import Stream, { PassThrough } from 'stream'; import Docker from 'dockerode'; -import { runDockerContainer, getGeneratorKey } from './helpers'; +import mockFs from 'mock-fs'; +import * as winston from 'winston'; +import { + runDockerContainer, + getGeneratorKey, + isValidRepoUrlForMkdocs, + getRepoUrlFromLocationAnnotation, + patchMkdocsYmlPreBuild, +} from './helpers'; +import { RemoteProtocol } from '../prepare/types'; +import { ParsedLocationAnnotation } from '../../../helpers'; const mockEntity = { apiVersion: 'version', @@ -28,6 +40,14 @@ const mockEntity = { const mockDocker = new Docker() as jest.Mocked; +const mkdocsYml = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs.yml'), +); +const mkdocsYmlWithRepoUrl = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_repo_url.yml'), +); +const mockLogger = winston.createLogger(); + describe('helpers', () => { describe('getGeneratorKey', () => { it('should return techdocs as the only generator key', () => { @@ -138,4 +158,177 @@ describe('helpers', () => { }); }); }); + + describe('isValidRepoUrlForMkdocs', () => { + it('should return true for valid repo_url values for mkdocs', () => { + const validRepoUrls = [ + 'https://github.com/org/repo', + 'https://github.com/backstage/backstage/', + 'https://github.com/org123/repo1-2-3/', + 'http://github.com/insecureOrg/insecureRepo', + 'https://gitlab.com/org/repo', + 'https://gitlab.com/backstage/backstage/', + 'https://gitlab.com/org123/repo1-2-3/', + 'http://gitlab.com/insecureOrg/insecureRepo', + ]; + + const validRemoteProtocols = ['github', 'gitlab']; + + validRepoUrls.forEach(url => { + validRemoteProtocols.forEach(targetType => { + expect( + isValidRepoUrlForMkdocs(url, targetType as RemoteProtocol), + ).toBe(true); + }); + }); + }); + + it('should return false for invalid repo_urls values for mkdocs', () => { + const invalidRepoUrls = [ + 'git@github.com:org/repo', + 'https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend', + ]; + + invalidRepoUrls.forEach(url => { + expect(isValidRepoUrlForMkdocs(url, 'github')).toBe(false); + }); + }); + + it('should return false for unsupported remote protocols', () => { + const validRepoUrl = 'https://github.com/backstage/backstage'; + + const unsupportedRemoteProtocols = ['dir', 'file', 'url']; + + unsupportedRemoteProtocols.forEach(targetType => { + expect( + isValidRepoUrlForMkdocs(validRepoUrl, targetType as RemoteProtocol), + ).toBe(false); + }); + }); + }); + + describe('getRepoUrlFromLocationAnnotation', () => { + it('should return undefined for unsupported location type', () => { + const parsedLocationAnnotation1: ParsedLocationAnnotation = { + type: 'dir', + target: '/home/user/workspace/docs-repository', + }; + + const parsedLocationAnnotation2: ParsedLocationAnnotation = { + type: 'file', + target: '/home/user/workspace/docs-repository/catalog-info.yaml', + }; + + const parsedLocationAnnotation3: ParsedLocationAnnotation = { + type: 'url', + target: 'https://my-website.com/storage/this/docs/repository', + }; + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe( + undefined, + ); + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe( + undefined, + ); + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe( + undefined, + ); + }); + + it('should return correct target url for supported hosts', () => { + const parsedLocationAnnotation1: ParsedLocationAnnotation = { + type: 'github', + target: 'https://github.com/backstage/backstage.git', + }; + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation1)).toBe( + 'https://github.com/backstage/backstage', + ); + + const parsedLocationAnnotation2: ParsedLocationAnnotation = { + type: 'github', + target: 'https://github.com/org/repo', + }; + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation2)).toBe( + 'https://github.com/org/repo', + ); + + const parsedLocationAnnotation3: ParsedLocationAnnotation = { + type: 'gitlab', + target: 'https://gitlab.com/org/repo', + }; + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation3)).toBe( + 'https://gitlab.com/org/repo', + ); + + const parsedLocationAnnotation4: ParsedLocationAnnotation = { + type: 'github', + target: + 'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component', + }; + + expect(getRepoUrlFromLocationAnnotation(parsedLocationAnnotation4)).toBe( + 'github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component', + ); + }); + }); + + describe('pathMkdocsPreBuild', () => { + beforeEach(() => { + mockFs({ + '/mkdocs.yml': mkdocsYml, + '/mkdocs_with_repo_url.yml': mkdocsYmlWithRepoUrl, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should add repo_url to mkdocs.yml', () => { + const parsedLocationAnnotation: ParsedLocationAnnotation = { + type: 'github', + target: 'https://github.com/backstage/backstage', + }; + + patchMkdocsYmlPreBuild( + '/mkdocs.yml', + mockLogger, + parsedLocationAnnotation, + ); + + const updatedMkdocsYml = fs.readFileSync('/mkdocs.yml').toString(); + + expect(updatedMkdocsYml).toContain( + "repo_url: 'https://github.com/backstage/backstage'", + ); + }); + + it('should not override existing repo_url in mkdocs.yml', () => { + const parsedLocationAnnotation: ParsedLocationAnnotation = { + type: 'github', + target: 'https://github.com/neworg/newrepo', + }; + + patchMkdocsYmlPreBuild( + '/mkdocs_with_repo_url.yml', + mockLogger, + parsedLocationAnnotation, + ); + + const updatedMkdocsYml = fs + .readFileSync('/mkdocs_with_repo_url.yml') + .toString(); + + expect(updatedMkdocsYml).toContain( + "repo_url: 'https://github.com/backstage/backstage'", + ); + expect(updatedMkdocsYml).not.toContain( + "repo_url: 'https://github.com/neworg/newrepo'", + ); + }); + }); }); diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts index d0ffa72f1c..254186b484 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts @@ -14,11 +14,16 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import fs from 'fs'; +import { spawn } from 'child_process'; import { Writable, PassThrough } from 'stream'; import Docker from 'dockerode'; +import yaml from 'js-yaml'; +import { Logger } from 'winston'; +import { Entity } from '@backstage/catalog-model'; import { SupportedGeneratorKey } from './types'; -import { spawn } from 'child_process'; +import { ParsedLocationAnnotation } from '../../../helpers'; +import { RemoteProtocol } from '../prepare/types'; // TODO: Implement proper support for more generators. export function getGeneratorKey(entity: Entity): SupportedGeneratorKey { @@ -142,3 +147,131 @@ export const runCommand = async ({ }); }); }; + +/** + * Return true if mkdocs can compile docs with provided repo_url + * + * Valid repo_url examples in mkdocs.yml + * - https://github.com/backstage/backstage + * - https://gitlab.com/org/repo/ + * - http://github.com/backstage/backstage + * - A http(s) protocol URL to the root of the repository + * + * Invalid repo_url examples in mkdocs.yml + * - https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component + * - (anything that is not valid as described above) + * + * @param {string} repoUrl URL supposed to be used as repo_url in mkdocs.yml + * @param {RemoteProtocol} locationType Type of source code host - github, gitlab, dir, url, etc. + * @returns {boolean} + */ +export const isValidRepoUrlForMkdocs = ( + repoUrl: string, + locationType: RemoteProtocol, +): boolean => { + // Trim trailing slash + const cleanRepoUrl = repoUrl.replace(/\/$/, ''); + + if (locationType === 'github' || locationType === 'gitlab') { + // A valid repoUrl to the root of the repository will be split into 5 strings if split using the / delimiter. + // We do not want URLs which have more than that number of forward slashes since they will signify a non-root location + // Note: This is not the best possible implementation but will work most of the times.. Feel free to improve or + // highlight edge cases. + return cleanRepoUrl.split('/').length === 5; + } + + return false; +}; + +/** + * Return a valid URL of the repository used in backstage.io/techdocs-ref annotation. + * Return undefined if the `target` is not valid in context of repo_url in mkdocs.yml + * Alter URL so that it is a valid repo_url config in mkdocs.yml + * + * @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type + * @returns {string | undefined} + */ +export const getRepoUrlFromLocationAnnotation = ( + parsedLocationAnnotation: ParsedLocationAnnotation, +): string | undefined => { + const { type: locationType, target } = parsedLocationAnnotation; + + // Add more options from the RemoteProtocol type of parsedLocationAnnotation.type here + // when TechDocs supports more hosts and if mkdocs can generated an Edit URL for them. + const supportedHosts = ['github', 'gitlab']; + + if (supportedHosts.includes(locationType)) { + // Trim .git or .git/ from the end of repository url + return target.replace(/.git\/*$/, ''); + } + + return undefined; +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to build docs site. + * + * List of tasks: + * - Add repo_url if it does not exists + * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. + * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get + * the repository URL. + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param {string} mkdocsYmlPath Absolute path to mkdocs.yml or equivalent of a docs site + * @param {Logger} logger + * @param {ParsedLocationAnnotation} parsedLocationAnnotation Object with location url and type + */ +export const patchMkdocsYmlPreBuild = async ( + mkdocsYmlPath: string, + logger: Logger, + parsedLocationAnnotation: ParsedLocationAnnotation, +) => { + let mkdocsYmlFileString; + try { + mkdocsYmlFileString = fs.readFileSync(mkdocsYmlPath, 'utf8'); + } catch (error) { + logger.warn( + `Could not read file ${mkdocsYmlPath} before running the generator. ${error.message}`, + ); + return; + } + + let mkdocsYml: any; + try { + mkdocsYml = yaml.safeLoad(mkdocsYmlFileString); + + // mkdocsYml should be an object type after successful parsing. + // But based on its type definition, it can also be a string or undefined, which we don't want. + if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { + throw new Error('Bad YAML format.'); + } + } catch (error) { + logger.warn( + `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, + ); + return; + } + + // Add repo_url to mkdocs.yml if it is missing. This will enable the Page edit button generated by MkDocs. + if (!('repo_url' in mkdocsYml)) { + const repoUrl = getRepoUrlFromLocationAnnotation(parsedLocationAnnotation); + if (repoUrl !== undefined) { + // mkdocs.yml will not build with invalid repo_url. So, make sure it is valid. + if (isValidRepoUrlForMkdocs(repoUrl, parsedLocationAnnotation.type)) { + mkdocsYml.repo_url = repoUrl; + } + } + } + + try { + fs.writeFileSync(mkdocsYmlPath, yaml.safeDump(mkdocsYml), 'utf8'); + } catch (error) { + logger.warn( + `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, + ); + return; + } +}; diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts index 1160ff1ade..faefd8ac16 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts @@ -26,7 +26,11 @@ import { GeneratorRunOptions, GeneratorRunResult, } from './types'; -import { runDockerContainer, runCommand } from './helpers'; +import { + runDockerContainer, + runCommand, + patchMkdocsYmlPreBuild, +} from './helpers'; type TechdocsGeneratorOptions = { // This option enables users to configure if they want to use TechDocs container @@ -62,6 +66,7 @@ export class TechdocsGenerator implements GeneratorBase { public async run({ directory, dockerClient, + parsedLocationAnnotation, }: GeneratorRunOptions): Promise { const tmpdirPath = os.tmpdir(); // Fixes a problem with macOS returning a path that is a symlink @@ -71,6 +76,15 @@ export class TechdocsGenerator implements GeneratorBase { ); const [log, logStream] = createStream(); + // TODO: In future mkdocs.yml can be mkdocs.yaml. So, use a config variable here to find out + // the correct file name. + // Do some updates to mkdocs.yml before generating docs e.g. adding repo_url + await patchMkdocsYmlPreBuild( + path.join(directory, 'mkdocs.yml'), + this.logger, + parsedLocationAnnotation, + ); + try { switch (this.options.runGeneratorIn) { case 'local': diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts index 398e4dcdd0..6d9ce5afca 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { Writable } from 'stream'; +import { Writable } from 'stream'; import Docker from 'dockerode'; import { Entity } from '@backstage/catalog-model'; +import { ParsedLocationAnnotation } from '../../../helpers'; /** * The returned directory from the generator which is ready @@ -26,14 +27,18 @@ export type GeneratorRunResult = { }; /** - * The values that the generator will receive. The directory of the - * uncompiled documentation, with the values from the frontend. A dedicated log stream and a docker - * client to run any generator on top of your directory. + * The values that the generator will receive. + * + * @param {string} directory The directory of the uncompiled documentation, with the values from the frontend + * @param {Docker} dockerClient A docker client to run any generator on top of your directory + * @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity + * @param {Writable} [logStream] A dedicated log stream */ export type GeneratorRunOptions = { directory: string; - logStream?: Writable; dockerClient: Docker; + parsedLocationAnnotation: ParsedLocationAnnotation; + logStream?: Writable; }; export type GeneratorBase = {