paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

upstream master

Browse Source
This commit is contained in:
NHI TRAN
2021-02-21 13:21:37 -05:00
parent f43192207a 1add977bae
commit e6fe13c675
711 changed files with 28616 additions and 8710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.test.ts
+4 -4
View File
@@ -17,11 +17,11 @@ import { BuildMetadataStorage } from './BuildMetadataStorage';
describe('BuildMetadataStorage', () => {
it('should return build timestamp', () => {
const newMetadataStorage = new BuildMetadataStorage('123abc');
newMetadataStorage.storeBuildTimestamp();
const newMetadataStorage = new BuildMetadataStorage('entityID123abc');
newMetadataStorage.setEtag('etag123abc');
const timestamp = newMetadataStorage.getTimestamp();
const timestamp = newMetadataStorage.getEtag();
expect(timestamp).toBeLessThanOrEqual(Date.now());
expect(timestamp).toBe('etag123abc');
});
});
plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts
+10 -5
View File
@@ -14,10 +14,15 @@
* limitations under the License.
*/
type buildInfo = {
// uid: timestamp
[key: string]: number;
// Entity uid: etag
[key: string]: string;
};
// TODO: Build info should be part of TechDocs storage, inside `techdocs_metadata.json`
// instead of in-memory storage of the Backstage instance.
// In case of multi-region Backstage deployments, or even using multiple Kubernetes pods,
// if each instance creates its separate build info in-memory, it will result in duplicate
// builds per instance. Also if the pod restarts, all the sites will have to be re-built.
const builds = {} as buildInfo;
/**
@@ -34,11 +39,11 @@ export class BuildMetadataStorage {
this.builds = builds;
}
storeBuildTimestamp() {
this.builds[this.entityUid] = Date.now();
setEtag(etag: string): void {
this.builds[this.entityUid] = etag;
}
getTimestamp() {
getEtag(): string | undefined {
return this.builds[this.entityUid];
}
}
plugins/techdocs-backend/src/DocsBuilder/builder.ts
+88 -78
View File
@@ -13,22 +13,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotModifiedError } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import {
GeneratorBase,
GeneratorBuilder,
getLocationForEntity,
PreparerBase,
PreparerBuilder,
PublisherBase,
UrlPreparer,
} from '@backstage/techdocs-common';
import Docker from 'dockerode';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import Docker from 'dockerode';
import { Logger } from 'winston';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
PreparerBuilder,
PublisherBase,
GeneratorBuilder,
PreparerBase,
GeneratorBase,
getLocationForEntity,
getLastCommitTimestamp,
} from '@backstage/techdocs-common';
import { BuildMetadataStorage } from '.';
const getEntityId = (entity: Entity) => {
@@ -44,7 +44,6 @@ type DocsBuilderArguments = {
entity: Entity;
logger: Logger;
dockerClient: Docker;
config: Config;
};
export class DocsBuilder {
@@ -54,7 +53,6 @@ export class DocsBuilder {
private entity: Entity;
private logger: Logger;
private dockerClient: Docker;
private config: Config;
constructor({
preparers,
@@ -63,7 +61,6 @@ export class DocsBuilder {
entity,
logger,
dockerClient,
config,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
@@ -71,14 +68,53 @@ export class DocsBuilder {
this.entity = entity;
this.logger = logger;
this.dockerClient = dockerClient;
this.config = config;
}
public async build() {
this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`);
const preparedDir = await this.preparer.prepare(this.entity);
public async build(): Promise<void> {
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
const parsedLocationAnnotation = getLocationForEntity(this.entity);
/**
* Prepare and cache check
*/
// Use the in-memory storage for setting and getting etag for this entity.
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
// TODO: As of now, this happens on each and every request to TechDocs.
// In a high traffic environment, this will cause a lot of requests to the source code provider.
// After Async build is implemented https://github.com/backstage/backstage/issues/3717,
// make sure to limit checking for cache invalidation to once per minute or so.
let preparedDir: string;
let etag: string;
try {
const preparerResponse = await this.preparer.prepare(this.entity, {
etag: buildMetadataStorage.getEtag(),
});
preparedDir = preparerResponse.preparedDir;
etag = preparerResponse.etag;
} catch (err) {
if (err instanceof NotModifiedError) {
// No need to prepare anymore since cache is valid.
return;
}
throw new Error(err.message);
}
this.logger.info(
`TechDocs prepare step completed for entity ${getEntityId(this.entity)}.`,
);
this.logger.debug(`Prepared files temporarily stored at ${preparedDir}`);
/**
* Generate
*/
this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`);
// Create a temporary directory to store the generated files in.
@@ -88,77 +124,51 @@ export class DocsBuilder {
const outputDir = await fs.mkdtemp(
path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
);
const parsedLocationAnnotation = getLocationForEntity(this.entity);
await this.generator.run({
inputDir: preparedDir,
outputDir,
dockerClient: this.dockerClient,
parsedLocationAnnotation,
etag,
});
this.logger.debug(`Generated files temporarily stored at ${outputDir}`);
// Remove Prepared directory since it is no longer needed.
// Caveat: Can not remove prepared directory in case of git preparer since the
// local git repository is used to get etag on subsequent requests.
if (this.preparer instanceof UrlPreparer) {
this.logger.debug(
`Removing prepared directory ${preparedDir} since the site has been generated.`,
);
try {
// Not a blocker hence no need to await this.
fs.remove(preparedDir);
} catch (error) {
this.logger.debug(`Error removing prepared directory ${error.message}`);
}
}
/**
* Publish
*/
this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`);
await this.publisher.publish({
entity: this.entity,
directory: outputDir,
});
// TODO: Remove the generated directory once published.
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp();
}
public async docsUpToDate() {
if (!this.entity.metadata.uid) {
throw new Error(
'Trying to build documentation for entity not in service catalog',
);
}
const buildMetadataStorage = new BuildMetadataStorage(
this.entity.metadata.uid,
);
const { type, target } = getLocationForEntity(this.entity);
// Unless docs are stored locally
const nonAgeCheckTypes = ['dir', 'file', 'url'];
if (!nonAgeCheckTypes.includes(type)) {
const lastCommit = await getLastCommitTimestamp(
target,
this.config,
this.logger,
);
const storageTimeStamp = buildMetadataStorage.getTimestamp();
// Check if documentation source is newer than what we have
if (storageTimeStamp && storageTimeStamp >= lastCommit) {
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} is up to date.`,
);
return true;
}
}
// Cache downloaded source files for 30 minutes.
// TODO: When urlReader/readTree supports some way to get latest commit timestamp,
// it should be used to invalidate cache.
if (type === 'url') {
const builtAt = buildMetadataStorage.getTimestamp();
const now = Date.now();
if (builtAt > now - 1800000) {
return true;
}
}
this.logger.debug(
`Docs for entity ${getEntityId(this.entity)} was outdated.`,
`Removing generated directory ${outputDir} since the site has been published`,
);
return false;
try {
// Not a blocker hence no need to await this.
fs.remove(outputDir);
} catch (error) {
this.logger.debug(`Error removing generated directory ${error.message}`);
}
// Store the latest build etag for the entity
new BuildMetadataStorage(this.entity.metadata.uid).setEtag(etag);
}
}
plugins/techdocs-backend/src/service/router.ts
+19 -20
View File
@@ -13,23 +13,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Logger } from 'winston';
import Router from 'express-promise-router';
import express from 'express';
import Knex from 'knex';
import fetch from 'cross-fetch';
import { Config } from '@backstage/config';
import Docker from 'dockerode';
import {
GeneratorBuilder,
PreparerBuilder,
PublisherBase,
getLocationForEntity,
} from '@backstage/techdocs-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { getEntityNameFromUrlPath } from './helpers';
import { Config } from '@backstage/config';
import {
GeneratorBuilder,
getLocationForEntity,
PreparerBuilder,
PublisherBase,
} from '@backstage/techdocs-common';
import fetch from 'cross-fetch';
import Docker from 'dockerode';
import express from 'express';
import Router from 'express-promise-router';
import Knex from 'knex';
import { Logger } from 'winston';
import { DocsBuilder } from '../DocsBuilder';
import { getEntityNameFromUrlPath } from './helpers';
type RouterOptions = {
preparers: PreparerBuilder;
@@ -140,13 +140,10 @@ export async function createRouter({
dockerClient,
logger,
entity,
config,
});
switch (publisherType) {
case 'local':
if (!(await docsBuilder.docsUpToDate())) {
await docsBuilder.build();
}
await docsBuilder.build();
break;
case 'awsS3':
case 'azureBlobStorage':
@@ -185,9 +182,11 @@ export async function createRouter({
'Found pre-generated docs for this entity. Serving them.',
);
// TODO: re-trigger build for cache invalidation.
// Compare the date modified of the requested file on storage and compare it against
// the last modified or last commit timestamp in the repository.
// Add build info in techdocs_metadata.json and compare it against
// the etag/commit in the repository.
// Without this, docs will not be re-built once they have been generated.
// Although it is unconventional that anyone will face this issue - because
// if you have an external storage, you should be using CI/CD to build and publish docs.
}
break;
default:
plugins/techdocs-backend/src/service/standaloneServer.ts
+1
View File
@@ -53,6 +53,7 @@ export async function startStandaloneServer(
const mockUrlReader: jest.Mocked<UrlReader> = {
read: jest.fn(),
readTree: jest.fn(),
search: jest.fn(),
};
logger.debug('Creating application...');