[TechDocs] Rebuild docs if the github source is newer than last docs build (#2353)

* Added age check for documentation in github to make sure it's up to date

* Updated failing tests

* Updated docs to explain requestUrl and storageUrl

* Updated failing tests

* Update packages/create-app/templates/default-app/app-config.yaml.hbs

* Update docs/features/techdocs/getting-started.md

Co-authored-by: Emma Indal <emmai@spotify.com>
This commit is contained in:
Sebastian Qvarfordt
2020-09-09 16:05:28 +02:00
committed by GitHub
parent bf1f645dd6
commit d865f88344
20 changed files with 313 additions and 172 deletions
+1
View File
@@ -34,6 +34,7 @@ organization:
techdocs:
storageUrl: http://localhost:7000/techdocs/static/docs
requestUrl: http://localhost:7000/techdocs/docs
sentry:
organization: spotify
+15 -4
View File
@@ -74,17 +74,28 @@ export { plugin as TechDocs } from '@backstage/plugin-techdocs';
### Setting the configuration
TechDocs allows for configuration of the docs storage URL through your
`app-config` file.
`app-config` file. We provide two different values to be configured,
`requestUrl` and `storageUrl`. The `requestUrl` is what the reader will request
its data from, and `storageUrl` is where the backend can find the stored
documentation.
The default storage URL:
The default storage and request URLs:
```yaml
techdocs:
storageUrl: http://localhost:7000/techdocs/static/docs
requestUrl: http://localhost:7000/techdocs/docs
```
If you want to configure this to point to another storage URL, change the value
of `storageUrl`.
If you want `techdocs-backend` to manage building and publishing you want
`requestUrl` to point to the default value (or wherever `techdocs-backend` is
hosted). `storageUrl` should be where your publisher publishes your docs. Using
the default `LocalPublish` that is the default value.
If you have a setup where you are not using `techdocs-backend` for managing
building and publishing of your documentation you want to change the
`requestUrl` to point to your storage. In this case `storageUrl` is not
required.
## Run Backstage locally
@@ -39,6 +39,7 @@ proxy:
techdocs:
storageUrl: http://localhost:7000/techdocs/static/docs
requestUrl: http://localhost:7000/techdocs/docs
lighthouse:
baseUrl: http://localhost:3003
+1
View File
@@ -26,6 +26,7 @@
"@backstage/config": "^0.1.1-alpha.21",
"@types/dockerode": "^2.5.34",
"@types/express": "^4.17.6",
"default-branch": "^1.0.8",
"dockerode": "^3.2.1",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
@@ -13,14 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/backend-common';
import { RemoteProtocol } from './types';
import os from 'os';
import path from 'path';
import parseGitUrl from 'git-url-parse';
import { Clone, Repository } from 'nodegit';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
// @ts-ignore
import defaultBranch from 'default-branch';
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/backend-common';
import { RemoteProtocol } from './techdocs/stages/prepare/types';
export type ParsedLocationAnnotation = {
type: RemoteProtocol;
@@ -58,16 +61,43 @@ export const parseReferenceAnnotation = (
};
};
export const clearGithubRepositoryCache = () => {
fs.removeSync(path.join(fs.realpathSync(os.tmpdir()), 'backstage-repo'));
export const getLocationForEntity = (
entity: Entity,
): ParsedLocationAnnotation => {
const { type, target } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
switch (type) {
case 'github':
return { type, target };
case 'dir':
if (path.isAbsolute(target)) return { type, target };
return parseReferenceAnnotation(
'backstage.io/managed-by-location',
entity,
);
default:
throw new Error(`Invalid reference annotation ${type}`);
}
};
export const checkoutGitRepository = async (
repoUrl: string,
export const getGitHubRepositoryTempFolder = async (
repositoryUrl: string,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repoUrl);
const parsedGitLocation = parseGitUrl(repositoryUrl);
// removes .git from git location path
parsedGitLocation.git_suffix = false;
const repositoryTmpPath = path.join(
if (!parsedGitLocation.ref) {
parsedGitLocation.ref = await defaultBranch(
parsedGitLocation.toString('https'),
);
}
return path.join(
// fs.realpathSync fixes a problem with macOS returning a path that is a symlink
fs.realpathSync(os.tmpdir()),
'backstage-repo',
@@ -76,48 +106,22 @@ export const checkoutGitRepository = async (
parsedGitLocation.name,
parsedGitLocation.ref,
);
if (fs.existsSync(repositoryTmpPath)) {
const repository = await Repository.open(repositoryTmpPath);
const currentBranchName = (await repository.getCurrentBranch()).shorthand();
await repository.mergeBranches(
currentBranchName,
`origin/${currentBranchName}`,
);
return repositoryTmpPath;
}
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
fs.mkdirSync(repositoryTmpPath, { recursive: true });
await Clone.clone(repositoryCheckoutUrl, repositoryTmpPath, {});
return repositoryTmpPath;
};
// Could be merged with checkoutGitRepository
export const checkoutGithubRepository = async (
repoUrl: string,
): Promise<string> => {
const parsedGitLocation = parseGitUrl(repoUrl);
const repositoryTmpPath = await getGitHubRepositoryTempFolder(repoUrl);
// Should propably not be hardcoded names of env variables, but seems too hard to access config down here
// TODO: Should propably not be hardcoded names of env variables, but seems too hard to access config down here
const user = process.env.GITHUB_PRIVATE_TOKEN_USER || '';
const token = process.env.GITHUB_PRIVATE_TOKEN || '';
const repositoryTmpPath = path.join(
// fs.realpathSync fixes a problem with macOS returning a path that is a symlink
fs.realpathSync(os.tmpdir()),
'backstage-repo',
parsedGitLocation.source,
parsedGitLocation.owner,
parsedGitLocation.name,
parsedGitLocation.ref,
);
if (fs.existsSync(repositoryTmpPath)) {
const repository = await Repository.open(repositoryTmpPath);
const currentBranchName = (await repository.getCurrentBranch()).shorthand();
await repository.fetch('origin');
await repository.mergeBranches(
currentBranchName,
`origin/${currentBranchName}`,
@@ -136,3 +140,14 @@ export const checkoutGithubRepository = async (
return repositoryTmpPath;
};
export const getLastCommitTimestamp = async (
repositoryUrl: string,
): Promise<number> => {
const repositoryLocation = await checkoutGithubRepository(repositoryUrl);
const repository = await Repository.open(repositoryLocation);
const commit = await repository.getReferenceCommit('HEAD');
return commit.date().getTime();
};
@@ -0,0 +1,132 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Docker from 'dockerode';
import { Logger } from 'winston';
import { Entity } from '@backstage/catalog-model';
import {
PreparerBuilder,
PublisherBase,
GeneratorBuilder,
PreparerBase,
GeneratorBase,
} from '../techdocs';
import { BuildMetadataStorage } from '../storage';
import { getLocationForEntity, getLastCommitTimestamp } from '../helpers';
const getEntityId = (entity: Entity) => {
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
entity.metadata.name
}`;
};
type DocsBuilderArguments = {
preparers: PreparerBuilder;
generators: GeneratorBuilder;
publisher: PublisherBase;
entity: Entity;
logger: Logger;
dockerClient: Docker;
};
export class DocsBuilder {
private preparer: PreparerBase;
private generator: GeneratorBase;
private publisher: PublisherBase;
private entity: Entity;
private logger: Logger;
private dockerClient: Docker;
constructor({
preparers,
generators,
publisher,
entity,
logger,
dockerClient,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
this.publisher = publisher;
this.entity = entity;
this.logger = logger;
this.dockerClient = dockerClient;
}
public async build() {
this.logger.info(
`[TechDocs] Running preparer on entity ${getEntityId(this.entity)}`,
);
const preparedDir = await this.preparer.prepare(this.entity);
this.logger.info(
`[TechDocs] Running generator on entity ${getEntityId(this.entity)}`,
);
const { resultDir } = await this.generator.run({
directory: preparedDir,
dockerClient: this.dockerClient,
});
this.logger.info(
`[TechDocs] Running publisher on entity ${getEntityId(this.entity)}`,
);
await this.publisher.publish({
entity: this.entity,
directory: resultDir,
});
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
}
public async docsUpToDate() {
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
const { type, target } = getLocationForEntity(this.entity);
// Should probably be broken out and handled per type later. Doing this for now since we only support github age checks
if (type === 'github') {
const lastCommit = await getLastCommitTimestamp(target);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`[TechDocs] Docs for entity ${getEntityId(
this.entity,
)} is up to date.`,
);
return true;
}
}
this.logger.debug(
`[TechDocs] Docs for entity ${getEntityId(this.entity)} was outdated.`,
);
return false;
}
}
@@ -1,49 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Preparers, LocalPublish, Generators } from '../techdocs';
import { ConfigReader } from '@backstage/config';
import Docker from 'dockerode';
import express from 'express';
import request from 'supertest';
import { createRouter } from './router';
describe('createRouter', () => {
let app: express.Express;
const logger = getVoidLogger();
beforeAll(async () => {
const router = await createRouter({
preparers: new Preparers(),
generators: new Generators(),
publisher: new LocalPublish(logger),
logger: getVoidLogger(),
dockerClient: new Docker(),
config: ConfigReader.fromConfigs([]),
});
app = express().use(router);
});
describe('GET /', () => {
it('does not explode', async () => {
const response = await request(app).get('/');
expect(response.status).toEqual(200);
expect(response.text).toEqual('Hello TechDocs Backend');
});
});
});
+23 -63
View File
@@ -28,6 +28,7 @@ import {
} from '../techdocs';
import { resolvePackagePath } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { DocsBuilder } from './helpers';
type RouterOptions = {
preparers: PreparerBuilder;
@@ -54,77 +55,36 @@ export async function createRouter({
}: RouterOptions): Promise<express.Router> {
const router = Router();
const getEntityId = (entity: Entity) => {
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
entity.metadata.name
}`;
};
const buildDocsForEntity = async (entity: Entity) => {
const preparer = preparers.get(entity);
const generator = generators.get(entity);
logger.info(`[TechDocs] Running preparer on entity ${getEntityId(entity)}`);
const preparedDir = await preparer.prepare(entity);
logger.info(
`[TechDocs] Running generator on entity ${getEntityId(entity)}`,
);
const { resultDir } = await generator.run({
directory: preparedDir,
dockerClient,
});
logger.info(
`[TechDocs] Running publisher on entity ${getEntityId(entity)}`,
);
await publisher.publish({
entity,
directory: resultDir,
});
};
router.get('/', async (_, res) => {
res.status(200).send('Hello TechDocs Backend');
});
// TODO: This route should not exist in the future
router.get('/buildall', async (_, res) => {
router.get('/docs/:kind/:namespace/:name/*', async (req, res) => {
const baseUrl = config.getString('backend.baseUrl');
const entitiesResponse = (await (
await fetch(`${baseUrl}/catalog/entities`)
).json()) as Entity[];
const storageUrl = config.getString('techdocs.storageUrl');
const entitiesWithDocs = entitiesResponse.filter(
entity => entity.metadata.annotations?.['backstage.io/techdocs-ref'],
);
const { kind, namespace, name } = req.params;
entitiesWithDocs.forEach(async entity => {
await buildDocsForEntity(entity);
const entity = (await (
await fetch(
`${baseUrl}/catalog/entities/by-name/${kind}/${namespace}/${name}`,
)
).json()) as Entity;
const docsBuilder = new DocsBuilder({
preparers,
generators,
publisher,
dockerClient,
logger,
entity,
});
res.send('Successfully generated documentation');
if (!(await docsBuilder.docsUpToDate())) {
await docsBuilder.build();
}
return res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`);
});
if (publisher instanceof LocalPublish) {
router.use('/static/docs/', express.static(staticDocsDir));
router.use(
'/static/docs/:kind/:namespace/:name',
async (req, res, next) => {
const baseUrl = config.getString('backend.baseUrl');
const { kind, namespace, name } = req.params;
const entityResponse = await fetch(
`${baseUrl}/catalog/entities/by-name/${kind}/${namespace}/${name}`,
);
if (!entityResponse.ok) next();
const entity = (await entityResponse.json()) as Entity;
await buildDocsForEntity(entity);
res.redirect(req.originalUrl);
},
);
router.use('/static/docs', express.static(staticDocsDir));
}
return router;
@@ -0,0 +1,39 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
type buildInfo = {
// uid: timestamp
[key: string]: number;
};
const builds = {} as buildInfo;
export class BuildMetadataStorage {
public entityUid: string;
private builds: buildInfo;
constructor(entityUid: string) {
this.entityUid = entityUid;
this.builds = builds;
}
storeBuildTimestamp() {
this.builds[this.entityUid] = Date.now();
}
getTimestamp() {
return this.builds[this.entityUid];
}
}
@@ -0,0 +1,16 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './BuildMetadataStorage';
@@ -15,4 +15,4 @@
*/
export { TechdocsGenerator } from './techdocs';
export { Generators } from './generators';
export type { GeneratorBuilder } from './types';
export type { GeneratorBuilder, GeneratorBase } from './types';
@@ -15,7 +15,7 @@
*/
import { DirectoryPreparer } from './dir';
import { getVoidLogger } from '@backstage/backend-common';
import { checkoutGitRepository } from './helpers';
import { checkoutGithubRepository } from '../../../helpers';
function normalizePath(path: string) {
return path
@@ -24,9 +24,11 @@ function normalizePath(path: string) {
.join('/');
}
jest.mock('./helpers', () => ({
...jest.requireActual<{}>('./helpers'),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
jest.mock('../../../helpers', () => ({
...jest.requireActual<{}>('../../../helpers'),
checkoutGithubRepository: jest.fn(
() => '/tmp/backstage-repo/org/name/branch/',
),
}));
const logger = getVoidLogger();
@@ -85,6 +87,6 @@ describe('directory preparer', () => {
expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual(
'/tmp/backstage-repo/org/name/branch/docs',
);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
expect(checkoutGithubRepository).toHaveBeenCalledTimes(1);
});
});
@@ -16,7 +16,10 @@
import { PreparerBase } from './types';
import { Entity } from '@backstage/catalog-model';
import path from 'path';
import { parseReferenceAnnotation, checkoutGitRepository } from './helpers';
import {
parseReferenceAnnotation,
checkoutGithubRepository,
} from '../../../helpers';
import { InputError } from '@backstage/backend-common';
import parseGitUrl from 'git-url-parse';
import { Logger } from 'winston';
@@ -40,7 +43,7 @@ export class DirectoryPreparer implements PreparerBase {
switch (type) {
case 'github': {
const parsedGitLocation = parseGitUrl(target);
const repoLocation = await checkoutGitRepository(target);
const repoLocation = await checkoutGithubRepository(target);
return path.dirname(
path.join(repoLocation, parsedGitLocation.filepath),
@@ -16,7 +16,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { GithubPreparer } from './github';
import { checkoutGithubRepository } from './helpers';
import { checkoutGithubRepository } from '../../../helpers';
function normalizePath(path: string) {
return path
@@ -25,8 +25,8 @@ function normalizePath(path: string) {
.join('/');
}
jest.mock('./helpers', () => ({
...jest.requireActual<{}>('./helpers'),
jest.mock('../../../helpers', () => ({
...jest.requireActual<{}>('../../../helpers'),
checkoutGithubRepository: jest.fn(
() => '/tmp/backstage-repo/org/name/branch',
),
@@ -18,7 +18,11 @@ import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
import parseGitUrl from 'git-url-parse';
import { parseReferenceAnnotation, checkoutGithubRepository } from './helpers';
import {
parseReferenceAnnotation,
checkoutGithubRepository,
} from '../../../helpers';
import { Logger } from 'winston';
export class GithubPreparer implements PreparerBase {
@@ -16,4 +16,4 @@
export { DirectoryPreparer } from './dir';
export { GithubPreparer } from './github';
export { Preparers } from './preparers';
export type { PreparerBuilder } from './types';
export type { PreparerBuilder, PreparerBase } from './types';
@@ -16,7 +16,7 @@
import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types';
import { Entity } from '@backstage/catalog-model';
import { parseReferenceAnnotation } from './helpers';
import { parseReferenceAnnotation } from '../../../helpers';
export class Preparers implements PreparerBuilder {
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
@@ -30,4 +30,4 @@ export type PreparerBuilder = {
get(entity: Entity): PreparerBase;
};
export type RemoteProtocol = 'dir' | string;
export type RemoteProtocol = 'dir' | 'github' | 'file';
+1 -1
View File
@@ -60,7 +60,7 @@ export const plugin = createPlugin({
deps: { configApi: configApiRef },
factory: ({ configApi }) =>
new TechDocsStorageApi({
apiOrigin: configApi.getString('techdocs.storageUrl'),
apiOrigin: configApi.getString('techdocs.requestUrl'),
}),
}),
],
+5
View File
@@ -8522,6 +8522,11 @@ deepmerge@^4.2.2:
resolved "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
default-branch@^1.0.8:
version "1.0.8"
resolved "https://registry.npmjs.org/default-branch/-/default-branch-1.0.8.tgz#0e2f36a90e3b0d9f73cdf8e02841364ed35b8b54"
integrity sha512-pViUZEnaxd/Hbu880MEXF7XqV8RJ1ssDlkvzx+woQhcKW8px3BrVDvwBGn09zRiKZ+gOLipK7ft5x3no+3vb8A==
default-gateway@^4.2.0:
version "4.2.0"
resolved "https://registry.npmjs.org/default-gateway/-/default-gateway-4.2.0.tgz#167104c7500c2115f6dd69b0a536bb8ed720552b"