paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Google Cloud Storage working with TechDocs (publish and fetch)

Browse Source 
Plus
1. Introduce ncessary configs to connect with storage
2. Introduce config to prefer CI build of docs, or techdocs-backend to build and publish them.
3. Write guide how to use Google Cloud Storage with TechDocs
4. Add a TechDocs Configuration reference page in docucmentation
This commit is contained in:
Himanshu Mishra
2020-12-03 22:40:15 +01:00
parent e09cff1682
commit 084670b37a
20 changed files with 971 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts → plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.test.ts
View File
plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts → plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts
View File
plugins/techdocs-backend/src/DocsBuilder/builder.ts
+129
View File
@@ -0,0 +1,129 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Docker from 'dockerode';
import { Logger } from 'winston';
import { Entity } from '@backstage/catalog-model';
import {
PreparerBuilder,
PublisherBase,
GeneratorBuilder,
PreparerBase,
GeneratorBase,
getLocationForEntity,
getLastCommitTimestamp,
} from '@backstage/techdocs-common';
import { BuildMetadataStorage } from '.';
const getEntityId = (entity: Entity) => {
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
entity.metadata.name
}`;
};
type DocsBuilderArguments = {
preparers: PreparerBuilder;
generators: GeneratorBuilder;
publisher: PublisherBase;
entity: Entity;
logger: Logger;
dockerClient: Docker;
};
export class DocsBuilder {
private preparer: PreparerBase;
private generator: GeneratorBase;
private publisher: PublisherBase;
private entity: Entity;
private logger: Logger;
private dockerClient: Docker;
constructor({
preparers,
generators,
publisher,
entity,
logger,
dockerClient,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
this.publisher = publisher;
this.entity = entity;
this.logger = logger;
this.dockerClient = dockerClient;
}
public async build() {
this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
const preparedDir = await this.preparer.prepare(this.entity);
const parsedLocationAnnotation = getLocationForEntity(this.entity);
this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
const { resultDir } = await this.generator.run({
directory: preparedDir,
dockerClient: this.dockerClient,
parsedLocationAnnotation,
});
this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);
await this.publisher.publish({
entity: this.entity,
directory: resultDir,
});
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
}
public async docsUpToDate() {
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
const { type, target } = getLocationForEntity(this.entity);
// Unless docs are stored locally
const nonAgeCheckTypes = ['dir', 'file', 'url'];
if (!nonAgeCheckTypes.includes(type)) {
const lastCommit = await getLastCommitTimestamp(target, this.logger);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} is up to date.`,
);
return true;
}
}
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} was outdated.`,
);
return false;
}
}
plugins/techdocs-backend/src/storage/index.ts → plugins/techdocs-backend/src/DocsBuilder/index.ts
+1
View File
@@ -14,3 +14,4 @@
* limitations under the License.
*/
export * from './BuildMetadataStorage';
export * from './builder';
plugins/techdocs-backend/src/index.ts
+1
View File
@@ -15,4 +15,5 @@
*/
export * from './service/router';
// TODO: Do named exports here e.g. publishers, generators.
export * from '@backstage/techdocs-common';
plugins/techdocs-backend/src/service/helpers.test.ts
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getEntityNameFromUrlPath } from './helpers';
describe('getEntityNameFromUrlPath', () => {
it('should parse correctly', () => {
const path = 'default/Component/documented-component';
const parsedEntity = getEntityNameFromUrlPath(path);
expect(parsedEntity).toHaveProperty('namespace', 'default');
expect(parsedEntity).toHaveProperty('kind', 'Component');
expect(parsedEntity).toHaveProperty('name', 'documented-component');
});
});
plugins/techdocs-backend/src/service/helpers.ts
+3 -117
View File
@@ -13,132 +13,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Docker from 'dockerode';
import { Logger } from 'winston';
import { Entity, EntityName } from '@backstage/catalog-model';
import {
PreparerBuilder,
PublisherBase,
GeneratorBuilder,
PreparerBase,
GeneratorBase,
getLocationForEntity,
getLastCommitTimestamp,
} from '@backstage/techdocs-common';
import { BuildMetadataStorage } from '../storage';
const getEntityId = (entity: Entity) => {
return `${entity.kind}:${entity.metadata.namespace ?? ''}:${
entity.metadata.name
}`;
};
type DocsBuilderArguments = {
preparers: PreparerBuilder;
generators: GeneratorBuilder;
publisher: PublisherBase;
entity: Entity;
logger: Logger;
dockerClient: Docker;
};
export class DocsBuilder {
private preparer: PreparerBase;
private generator: GeneratorBase;
private publisher: PublisherBase;
private entity: Entity;
private logger: Logger;
private dockerClient: Docker;
constructor({
preparers,
generators,
publisher,
entity,
logger,
dockerClient,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
this.publisher = publisher;
this.entity = entity;
this.logger = logger;
this.dockerClient = dockerClient;
}
public async build() {
this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
const preparedDir = await this.preparer.prepare(this.entity);
const parsedLocationAnnotation = getLocationForEntity(this.entity);
this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
const { resultDir } = await this.generator.run({
directory: preparedDir,
dockerClient: this.dockerClient,
parsedLocationAnnotation,
});
this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);
await this.publisher.publish({
entity: this.entity,
directory: resultDir,
});
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
}
public async docsUpToDate() {
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
const { type, target } = getLocationForEntity(this.entity);
// Unless docs are stored locally
const nonAgeCheckTypes = ['dir', 'file', 'url'];
if (!nonAgeCheckTypes.includes(type)) {
const lastCommit = await getLastCommitTimestamp(target, this.logger);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} is up to date.`,
);
return true;
}
}
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} was outdated.`,
);
return false;
}
}
import { EntityName } from '@backstage/catalog-model';
/**
* Using the path of the TechDocs page URL, return a structured EntityName type object with namespace,
* kind and name of the Entity.
* @param {string} path Example: default/Component/documented-component
*/
export const getEntityNameFromUrlPath = (path: string): EntityName => {
const [kind, namespace, name] = path.split('/');
const [namespace, kind, name] = path.split('/');
return {
kind,
namespace,
kind,
name,
};
};
plugins/techdocs-backend/src/service/router.ts
+46 -13
View File
@@ -28,7 +28,8 @@ import {
} from '@backstage/techdocs-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { DocsBuilder, getEntityNameFromUrlPath } from './helpers';
import { getEntityNameFromUrlPath } from './helpers';
import { DocsBuilder } from '../DocsBuilder';
type RouterOptions = {
preparers: PreparerBuilder;
@@ -92,9 +93,8 @@ export async function createRouter({
});
router.get('/docs/:namespace/:kind/:name/*', async (req, res) => {
const storageUrl = config.getString('techdocs.storageUrl');
const { kind, namespace, name } = req.params;
const storageUrl = config.getString('techdocs.storageUrl');
const catalogUrl = await discovery.getBaseUrl('catalog');
const triple = [kind, namespace, name].map(encodeURIComponent).join('/');
@@ -109,22 +109,55 @@ export async function createRouter({
const entity: Entity = await catalogRes.json();
const docsBuilder = new DocsBuilder({
preparers,
generators,
publisher,
dockerClient,
logger,
entity,
});
let publisherType = '';
try {
publisherType = config.getString('techdocs.publisher.type');
} catch (err) {
throw new Error(
'Unable to get techdocs.publisher.type in your app config. Set it to either ' +
"'local', 'google_gcs' or other support storage providers. Read more here " +
'https://backstage.io/docs/features/techdocs/architecture',
);
}
if (!(await docsBuilder.docsUpToDate())) {
await docsBuilder.build();
// techdocs-backend will only try to build documentation for an entity if techdocs.docsBuilder is set to 'local'
// If set to 'ci', it will only try to fetch and assume that that CI/CD pipeline of the repository hosting the
// entity's documentation is responsible for building and publishing documentation to the storage provider.
const shouldBuildDocs =
config.getString('techdocs.docsBuilder') === 'local';
if (shouldBuildDocs) {
const docsBuilder = new DocsBuilder({
preparers,
generators,
publisher,
dockerClient,
logger,
entity,
});
if (publisherType === 'local') {
if (!(await docsBuilder.docsUpToDate())) {
await docsBuilder.build();
}
} else if (publisherType === 'google_gcs') {
if (!(await publisher.hasDocsBeenGenerated(entity))) {
logger.info(
'Did not find generated docs files for the entity. Building docs.',
);
await docsBuilder.build();
} else {
logger.info(
'Found pre-generated docs for this entity. Serving them.',
);
// TODO: When to re-trigger build for cache invalidation?
}
}
}
res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`);
});
// Route middleware which serves files from the storage set in the publisher.
router.use('/static/docs', publisher.docsRouter());
return router;