paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Merge pull request #6335 from backstage/iameap/techdocs-cache

Browse Source 
[TechDocs] Optional static resource caching
This commit is contained in:
Otto Sichert
2021-12-06 10:38:30 +01:00
committed by GitHub
parent 963d69b30b e3e6e9a86d
commit 731e9293bd
33 changed files with 1232 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/api-report.md
+2
View File
@@ -287,6 +287,8 @@ export type TechDocsMetadata = {
site_name: string;
site_description: string;
etag: string;
build_timestamp: number;
files?: string[];
};
// Warning: (ae-missing-release-tag) "transformDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
packages/techdocs-common/src/stages/generate/helpers.test.ts
+20 -8
View File
@@ -23,7 +23,7 @@ import os from 'os';
import path, { resolve as resolvePath } from 'path';
import { ParsedLocationAnnotation } from '../../helpers';
import {
addBuildTimestampMetadata,
createOrUpdateMetadata,
getGeneratorKey,
getMkdocsYml,
getRepoUrlFromLocationAnnotation,
@@ -369,13 +369,15 @@ describe('helpers', () => {
});
describe('addBuildTimestampMetadata', () => {
const mockFiles = {
'invalid_techdocs_metadata.json': 'dsds',
'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
};
beforeEach(() => {
mockFs.restore();
mockFs({
[rootDir]: {
'invalid_techdocs_metadata.json': 'dsds',
'techdocs_metadata.json': '{"site_name": "Tech Docs"}',
},
[rootDir]: mockFiles,
});
});
@@ -385,7 +387,7 @@ describe('helpers', () => {
it('should create the file if it does not exist', async () => {
const filePath = path.join(rootDir, 'wrong_techdocs_metadata.json');
await addBuildTimestampMetadata(filePath, mockLogger);
await createOrUpdateMetadata(filePath, mockLogger);
// Check if the file exists
await expect(
@@ -397,18 +399,28 @@ describe('helpers', () => {
const filePath = path.join(rootDir, 'invalid_techdocs_metadata.json');
await expect(
addBuildTimestampMetadata(filePath, mockLogger),
createOrUpdateMetadata(filePath, mockLogger),
).rejects.toThrowError('Unexpected token d in JSON at position 0');
});
it('should add build timestamp to the metadata json', async () => {
const filePath = path.join(rootDir, 'techdocs_metadata.json');
await addBuildTimestampMetadata(filePath, mockLogger);
await createOrUpdateMetadata(filePath, mockLogger);
const json = await fs.readJson(filePath);
expect(json.build_timestamp).toBeLessThanOrEqual(Date.now());
});
it('should add list of files to the metadata json', async () => {
const filePath = path.join(rootDir, 'techdocs_metadata.json');
await createOrUpdateMetadata(filePath, mockLogger);
const json = await fs.readJson(filePath);
expect(json.files[0]).toEqual(Object.keys(mockFiles)[0]);
expect(json.files[1]).toEqual(Object.keys(mockFiles)[1]);
});
});
describe('storeEtagMetadata', () => {
packages/techdocs-common/src/stages/generate/helpers.ts
+22 -2
View File
@@ -27,6 +27,7 @@ import { PassThrough, Writable } from 'stream';
import { Logger } from 'winston';
import { ParsedLocationAnnotation } from '../../helpers';
import { SupportedGeneratorKey } from './types';
import { getFileTreeRecursively } from '../publish/helpers';
// TODO: Implement proper support for more generators.
export function getGeneratorKey(entity: Entity): SupportedGeneratorKey {
@@ -345,14 +346,20 @@ export const patchIndexPreBuild = async ({
};
/**
* Update the techdocs_metadata.json to add a new build timestamp metadata. Create the .json file if it doesn't exist.
* Create or update the techdocs_metadata.json. Values initialized/updated are:
* - The build_timestamp (now)
* - The list of files generated
*
* @param {string} techdocsMetadataPath File path to techdocs_metadata.json
*/
export const addBuildTimestampMetadata = async (
export const createOrUpdateMetadata = async (
techdocsMetadataPath: string,
logger: Logger,
): Promise<void> => {
const techdocsMetadataDir = techdocsMetadataPath
.split(path.sep)
.slice(0, -1)
.join(path.sep);
// check if file exists, create if it does not.
try {
await fs.access(techdocsMetadataPath, fs.constants.F_OK);
@@ -372,6 +379,19 @@ export const addBuildTimestampMetadata = async (
}
json.build_timestamp = Date.now();
// Get and write generated files to the metadata JSON. Each file string is in
// a form appropriate for invalidating the associated object from cache.
try {
json.files = (await getFileTreeRecursively(techdocsMetadataDir)).map(file =>
file.replace(`${techdocsMetadataDir}/`, ''),
);
} catch (err) {
assertError(err);
json.files = [];
logger.warn(`Unable to add files list to metadata: ${err.message}`);
}
await fs.writeJson(techdocsMetadataPath, json);
return;
};
packages/techdocs-common/src/stages/generate/techdocs.ts
+3 -3
View File
@@ -23,7 +23,7 @@ import {
ScmIntegrations,
} from '@backstage/integration';
import {
addBuildTimestampMetadata,
createOrUpdateMetadata,
getMkdocsYml,
patchIndexPreBuild,
patchMkdocsYmlPreBuild,
@@ -164,9 +164,9 @@ export class TechdocsGenerator implements GeneratorBase {
* Post Generate steps
*/
// Add build timestamp to techdocs_metadata.json
// Add build timestamp and files to techdocs_metadata.json
// Creates techdocs_metadata.json if file does not exist.
await addBuildTimestampMetadata(
await createOrUpdateMetadata(
path.join(outputDir, 'techdocs_metadata.json'),
childLogger,
);
packages/techdocs-common/src/stages/publish/awsS3.test.ts
+36 -5
View File
@@ -95,6 +95,7 @@ describe('AwsS3Publish', () => {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
build_timestamp: 612741599,
};
const directory = getEntityRootDir(entity);
@@ -149,21 +150,39 @@ describe('AwsS3Publish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const publisher = createPublisherFromConfig();
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/component/backstage/404.html',
`default/component/backstage/index.html`,
`default/component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory as well when legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/Component/backstage/404.html',
`default/Component/backstage/index.html`,
`default/Component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory when root path is specified', async () => {
const publisher = createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'backstage-data/techdocs/default/component/backstage/404.html',
`backstage-data/techdocs/default/component/backstage/index.html`,
`backstage-data/techdocs/default/component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory when root path is specified and legacy casing is used', async () => {
@@ -171,14 +190,26 @@ describe('AwsS3Publish', () => {
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'backstage-data/techdocs/default/Component/backstage/404.html',
`backstage-data/techdocs/default/Component/backstage/index.html`,
`backstage-data/techdocs/default/Component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory when sse is specified', async () => {
const publisher = createPublisherFromConfig({
sse: 'aws:kms',
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/component/backstage/404.html',
'default/component/backstage/index.html',
'default/component/backstage/assets/main.css',
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/awsS3.ts
+8 -1
View File
@@ -39,6 +39,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -214,7 +215,11 @@ export class AwsS3Publish implements PublisherBase {
* Upload all the files from the generated `directory` to the S3 bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
const bucketRootPath = this.bucketRootPath;
const sse = this.sse;
@@ -263,6 +268,7 @@ export class AwsS3Publish implements PublisherBase {
...(sse && { ServerSideEncryption: sse }),
} as aws.S3.PutObjectRequest;
objects.push(params.Key);
return this.storageClient.upload(params).promise();
},
absoluteFilesToUpload,
@@ -311,6 +317,7 @@ export class AwsS3Publish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
async fetchTechDocsMetadata(
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+15 -2
View File
@@ -89,6 +89,7 @@ describe('AzureBlobStoragePublish', () => {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
build_timestamp: 612741599,
};
const directory = getEntityRootDir(entity);
@@ -154,14 +155,26 @@ describe('AzureBlobStoragePublish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const publisher = createPublisherFromConfig();
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/component/backstage/404.html',
`default/component/backstage/index.html`,
`default/component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory as well when legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/Component/backstage/404.html',
`default/Component/backstage/index.html`,
`default/Component/backstage/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+15 -8
View File
@@ -39,6 +39,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -156,7 +157,11 @@ export class AzureBlobStoragePublish implements PublisherBase {
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
// First, try to retrieve a list of all individual files currently existing
@@ -194,14 +199,14 @@ export class AzureBlobStoragePublish implements PublisherBase {
const relativeFilePath = path.normalize(
path.relative(directory, absoluteFilePath),
);
const remotePath = getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
);
objects.push(remotePath);
const response = await container
.getBlockBlobClient(
getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
),
)
.getBlockBlobClient(remotePath)
.uploadFile(absoluteFilePath);
if (response._response.status >= 400) {
@@ -264,6 +269,8 @@ export class AzureBlobStoragePublish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from Azure. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
private download(containerName: string, blobPath: string): Promise<Buffer> {
packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+29 -4
View File
@@ -88,6 +88,7 @@ describe('GoogleGCSPublish', () => {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
build_timestamp: 612741599,
};
const directory = getEntityRootDir(entity);
@@ -142,21 +143,39 @@ describe('GoogleGCSPublish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const publisher = createPublisherFromConfig();
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/component/backstage/404.html',
`default/component/backstage/index.html`,
`default/component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory as well when legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/Component/backstage/404.html',
`default/Component/backstage/index.html`,
`default/Component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory when root path is specified', async () => {
const publisher = createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'backstage-data/techdocs/default/component/backstage/404.html',
`backstage-data/techdocs/default/component/backstage/index.html`,
`backstage-data/techdocs/default/component/backstage/assets/main.css`,
]),
});
});
it('should publish a directory when root path is specified and legacy casing is used', async () => {
@@ -164,7 +183,13 @@ describe('GoogleGCSPublish', () => {
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'backstage-data/techdocs/default/Component/backstage/404.html',
`backstage-data/techdocs/default/Component/backstage/index.html`,
`backstage-data/techdocs/default/Component/backstage/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/googleStorage.ts
+16 -9
View File
@@ -36,6 +36,7 @@ import { MigrateWriteStream } from './migrations';
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -145,7 +146,11 @@ export class GoogleGCSPublish implements PublisherBase {
* Upload all the files from the generated `directory` to the GCS bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
const bucket = this.storageClient.bucket(this.bucketName);
const bucketRootPath = this.bucketRootPath;
@@ -178,14 +183,14 @@ export class GoogleGCSPublish implements PublisherBase {
await bulkStorageOperation(
async absoluteFilePath => {
const relativeFilePath = path.relative(directory, absoluteFilePath);
return await bucket.upload(absoluteFilePath, {
destination: getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
bucketRootPath,
),
});
const destination = getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
bucketRootPath,
);
objects.push(destination);
return await bucket.upload(absoluteFilePath, { destination });
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 },
@@ -228,6 +233,8 @@ export class GoogleGCSPublish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
packages/techdocs-common/src/stages/publish/local.ts
+28 -22
View File
@@ -98,7 +98,10 @@ export class LocalPublish implements PublisherBase {
};
}
publish({ entity, directory }: PublishRequest): Promise<PublishResponse> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const entityNamespace = entity.metadata.namespace ?? 'default';
const publishDir = this.staticEntityPathJoin(
@@ -112,27 +115,30 @@ export class LocalPublish implements PublisherBase {
fs.mkdirSync(publishDir, { recursive: true });
}
return new Promise((resolve, reject) => {
fs.copy(directory, publishDir, err => {
if (err) {
this.logger.debug(
`Failed to copy docs from ${directory} to ${publishDir}`,
);
reject(err);
}
this.logger.info(`Published site stored at ${publishDir}`);
this.discovery
.getBaseUrl('techdocs')
.then(techdocsApiUrl => {
resolve({
remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
});
})
.catch(reason => {
reject(reason);
});
});
});
try {
await fs.copy(directory, publishDir);
this.logger.info(`Published site stored at ${publishDir}`);
} catch (error) {
this.logger.debug(
`Failed to copy docs from ${directory} to ${publishDir}`,
);
throw error;
}
// Generate publish response.
const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs');
const publishedFilePaths = (await getFileTreeRecursively(publishDir)).map(
abs => {
return abs.split(`${staticDocsDir}/`)[1];
},
);
return {
remoteUrl: `${techdocsApiUrl}/static/docs/${encodeURIComponent(
entity.metadata.name,
)}`,
objects: publishedFilePaths,
};
}
async fetchTechDocsMetadata(
packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
+11 -3
View File
@@ -170,7 +170,13 @@ describe('OpenStackSwiftPublish', () => {
entity,
directory: entityRootDir,
}),
).toBeUndefined();
).toMatchObject({
objects: expect.arrayContaining([
'test-namespace/TestKind/test-component-name/404.html',
`test-namespace/TestKind/test-component-name/index.html`,
`test-namespace/TestKind/test-component-name/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
@@ -241,7 +247,7 @@ describe('OpenStackSwiftPublish', () => {
mockFs({
[entityRootDir]: {
'techdocs_metadata.json':
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
'{"site_name": "backstage", "site_description": "site_content", "etag": "etag", "build_timestamp": 612741599}',
},
});
@@ -249,6 +255,7 @@ describe('OpenStackSwiftPublish', () => {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
build_timestamp: 612741599,
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
@@ -263,7 +270,7 @@ describe('OpenStackSwiftPublish', () => {
mockFs({
[entityRootDir]: {
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag', 'build_timestamp': 612741599}`,
},
});
@@ -271,6 +278,7 @@ describe('OpenStackSwiftPublish', () => {
site_name: 'backstage',
site_description: 'site_content',
etag: 'etag',
build_timestamp: 612741599,
};
expect(
await publisher.fetchTechDocsMetadata(entityNameMock),
packages/techdocs-common/src/stages/publish/openStackSwift.ts
+9 -2
View File
@@ -32,6 +32,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -139,8 +140,13 @@ export class OpenStackSwiftPublish implements PublisherBase {
* Upload all the files from the generated `directory` to the OpenStack Swift container.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
try {
const objects: string[] = [];
// Note: OpenStack Swift manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
@@ -161,6 +167,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
objects.push(destination);
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(async () => {
@@ -178,7 +185,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
return;
return { objects };
} catch (e) {
const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`;
this.logger.error(errorMessage);
packages/techdocs-common/src/stages/publish/types.ts
+15 -1
View File
@@ -32,9 +32,21 @@ export type PublishRequest = {
directory: string;
};
/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */
/**
* Response containing metadata about where files were published and what may
* have been published or updated.
*/
export type PublishResponse = {
/**
* The URL which serves files from the local publisher's static directory.
*/
remoteUrl?: string;
/**
* The list of objects (specifically their paths) that were published.
* Objects do not have a preceding slash, and match how one would load the
* object over the `/static/docs/*` TechDocs Backend Plugin endpoint.
*/
objects?: string[];
} | void;
/**
@@ -53,6 +65,8 @@ export type TechDocsMetadata = {
site_name: string;
site_description: string;
etag: string;
build_timestamp: number;
files?: string[];
};
export type MigrateRequest = {